CLAW Publications

Reasoning about the Unseen for Efficient Outdoor Object Navigation

ArXiv (ArXiv)
Quanting Xie, Tianyi Zhang, Kedi Xu, Matthew Johnson-Roberson,

Yonatan Bisk

Paper BibTeX


@article{Xie2023,
    author    = {Quanting Xie and Tianyi Zhang and Kedi Xu and Matthew Johnson-Roberson and Yonatan Bisk},
    title     = {{Reasoning about the Unseen for Efficient Outdoor Object Navigation}},
    journal = {ArXiv},
    year      = {2023},
    url       = {https://arxiv.org/abs/2309.10103},
}

MOSAIC: Learning Unified Multi-Sensory Object Property Representations for Robot Perception

ArXiv (ArXiv)
Gyan Tatiya, Jonathan Francis, Ho-Hsiang Wu,

Yonatan Bisk

, Jivko Sinapov
Paper BibTeX


@article{Tatiya2023,
    author    = {Gyan Tatiya and Jonathan Francis and Ho-Hsiang Wu and Yonatan Bisk and Jivko Sinapov},
    title     = {{MOSAIC: Learning Unified Multi-Sensory Object Property Representations for Robot Perception}},
    journal = {ArXiv},
    year      = {2023},
    url       = {https://arxiv.org/abs/2309.08508},
}

WebArena: A Realistic Web Environment for Building Autonomous Agents

ArXiv (ArXiv)
Shuyan Zhou, Frank F. Xu,

Hao Zhu

, Xuhui Zhou, Robert Lo, Abishek Sridhar, Xianyi Cheng,

Yonatan Bisk

, Daniel Fried, Uri Alon, Graham Neubig
Paper BibTeX


@article{Zhou2023,
    author    = {Shuyan Zhou and Frank F. Xu and Hao Zhu and Xuhui Zhou and Robert Lo and Abishek Sridhar and Xianyi Cheng and Yonatan Bisk and Daniel Fried and Uri Alon and Graham Neubig},
    title     = {{WebArena: A Realistic Web Environment for Building Autonomous Agents}},
    journal = {ArXiv},
    year      = {2023},
    url       = {https://arxiv.org/abs/2307.13854},
}

Spatial-Language Attention Policies for Efficient Robot Learning

Conference on Robot Learning (CoRL)
Priyam Parashar,

Vidhi Jain

, Xiaohan Zhang, Jay Vakil, Sam Powers,

Yonatan Bisk

, Chris Paxton
Paper BibTeX Website


@inproceedings{Parashar2023,
    author    = {Priyam Parashar and Vidhi Jain and Xiaohan Zhang and Jay Vakil and Sam Powers and Yonatan Bisk and Chris Paxton},
    title     = {{Spatial-Language Attention Policies for Efficient Robot Learning}},
    booktitle = {Conference on Robot Learning},
    year      = {2023},
    url       = {https://arxiv.org/abs/2304.11235},
}

SPAE: Semantic Pyramid AutoEncoder for Multimodal Generation with Frozen LLMs

Thirty-seventh Conference on Neural Information Processing Systems (NeurIPS)
Lijun Yu, Yong Cheng, Zhiruo Wang, Vivek Kumar, Wolfgang Macherey, Yanping Huang, David A. Ross, Irfan Essa,

Yonatan Bisk

, Ming-Hsuan Yang, Kevin Murphy, Alexander G. Hauptmann, Lu Jiang
Paper BibTeX


@inproceedings{Yu2023,
    author    = {Lijun Yu and Yong Cheng and Zhiruo Wang and Vivek Kumar and Wolfgang Macherey and Yanping Huang and David A. Ross and Irfan Essa and Yonatan Bisk and Ming-Hsuan Yang and Kevin Murphy and Alexander G. Hauptmann and Lu Jiang},
    title     = {{SPAE: Semantic Pyramid AutoEncoder for Multimodal Generation with Frozen LLMs}},
    booktitle = {Thirty-seventh Conference on Neural Information Processing Systems },
    year      = {2023},
    url       = {https://arxiv.org/abs/2306.17842},
}

HomeRobot: Open-Vocabulary Mobile Manipulation

Conference on Robot Learning (CoRL)
Sriram Yenamandra, Arun Ramachandran, Karmesh Yadav, Austin Wang, Mukul Khanna, Theophile Gervet, Tsung-Yen Yang,

Vidhi Jain

, Alexander William Clegg, John Turner, Zsolt Kira, Manolis Savva, Angel Chang, Devendra Singh Chaplot, Dhruv Batra, Roozbeh Mottaghi,

Yonatan Bisk

, Chris Paxton
Paper BibTeX Video


@inproceedings{Yenamandra2023,
    author    = {Sriram Yenamandra and Arun Ramachandran and Karmesh Yadav and Austin Wang and Mukul Khanna and Theophile Gervet and Tsung-Yen Yang and Vidhi Jain and Alexander William Clegg and John Turner and Zsolt Kira and Manolis Savva and Angel Chang and Devendra Singh Chaplot and Dhruv Batra and Roozbeh Mottaghi and Yonatan Bisk and Chris Paxton},
    title     = {{HomeRobot: Open-Vocabulary Mobile Manipulation}},
    booktitle = {Conference on Robot Learning},
    year      = {2023},
    url       = {https://arxiv.org/abs/2306.11565},
}

SPRING: GPT-4 Out-performs RL Algorithms by Studying Papers and Reasoning

Thirty-seventh Conference on Neural Information Processing Systems (NeurIPS)
Yue Wu, Shrimai Prabhumoye,

So Yeon Min

,

Yonatan Bisk

, Ruslan Salakhutdinov, Amos Azaria, Tom Mitchell, Yuanzhi Li
Paper BibTeX


@inproceedings{Wu2023,
    author    = {Yue Wu and Shrimai Prabhumoye and So Yeon Min and Yonatan Bisk and Ruslan Salakhutdinov and Amos Azaria and Tom Mitchell and Yuanzhi Li},
    title     = {{SPRING: GPT-4 Out-performs RL Algorithms by Studying Papers and Reasoning}},
    booktitle = {Thirty-seventh Conference on Neural Information Processing Systems },
    year      = {2023},
    url       = {https://arxiv.org/abs/2305.15486},
}

Plan, Eliminate, and Track -- Language Models are Good Teachers for Embodied Agents

ArXiv (ArXiv)
Yue Wu,

So Yeon Min

,

Yonatan Bisk

, Ruslan Salakhutdinov, Amos Azaria, Yuanzhi Li, Tom Mitchell, Shrimai Prabhumoye
Paper BibTeX


@article{Wu2023,
    author    = {Yue Wu and So Yeon Min and Yonatan Bisk and Ruslan Salakhutdinov and Amos Azaria and Yuanzhi Li and Tom Mitchell and Shrimai Prabhumoye},
    title     = {{Plan, Eliminate, and Track -- Language Models are Good Teachers for Embodied Agents}},
    journal = {ArXiv},
    year      = {2023},
    url       = {https://arxiv.org/abs/2305.02412},
}

HomeRobot Open Vocab Mobile Manipulation Challenge

Thirty-seventh Conference on Neural Information Processing Systems: Competition Track (NeurIPS Challenge)
Sriram Yenamandra, Arun Ramachandran, Mukul Khanna, Karmesh Yadav, Devendra Singh Chaplot, Gunjan Chhablani, Alexander Clegg, Theophile Gervet,

Vidhi Jain

, Ruslan Partsey, Ram Ramrakhya, Andrew Szot, Tsung-Yen Yang, Aaron Edsinger, Charlie Kemp, Binit Shah, Zsolt Kira, Dhruv Batra, Roozbeh Mottaghi,

Yonatan Bisk

, Chris Paxton
Paper BibTeX


@inproceedings{Yenamandra2023,
    author    = {Sriram Yenamandra and Arun Ramachandran and Mukul Khanna and Karmesh Yadav and Devendra Singh Chaplot and Gunjan Chhablani and Alexander Clegg and Theophile Gervet and Vidhi Jain and Ruslan Partsey and Ram Ramrakhya and Andrew Szot and Tsung-Yen Yang and Aaron Edsinger and Charlie Kemp and Binit Shah and Zsolt Kira and Dhruv Batra and Roozbeh Mottaghi and Yonatan Bisk and Chris Paxton},
    title     = {{HomeRobot Open Vocab Mobile Manipulation Challenge}},
    booktitle = {Thirty-seventh Conference on Neural Information Processing Systems: Competition Track},
    year      = {2023},
    url       = {https://aihabitat.org/challenge/2023_homerobot_ovmm/},
}

EXCALIBUR: Encouraging and Evaluating Embodied Exploration

Conference on Computer Vision and Pattern Recognition (CVPR)

Hao Zhu

, Raghav Kapoor,

So Yeon Min

, Winson Han, Jiatai Li, Kaiwen Geng, Graham Neubig,

Yonatan Bisk

, Aniruddha Kembhavi, Luca Weihs
Paper BibTeX Video Tasks


@inproceedings{Zhu2023,
    author    = {Hao Zhu and Raghav Kapoor and So Yeon Min and Winson Han and Jiatai Li and Kaiwen Geng and Graham Neubig and Yonatan Bisk and Aniruddha Kembhavi and Luca Weihs},
    title     = {{EXCALIBUR: Encouraging and Evaluating Embodied Exploration}},
    booktitle = {Conference on Computer Vision and Pattern Recognition},
    year      = {2023},
    url       = {https://openaccess.thecvf.com/content/CVPR2023/html/Zhu_EXCALIBUR_Encouraging_and_Evaluating_Embodied_Exploration_CVPR_2023_paper.html},
}

The Framework Tax: Disparities Between Inference Efficiency in Research and Deployment

ArXiv (ArXiv)

Jared Fernandez

, Jacob Kahn, Clara Na,

Yonatan Bisk

, Emma Strubell
Paper BibTeX


@article{Fernandez2023,
    author    = {Jared Fernandez and Jacob Kahn and Clara Na and Yonatan Bisk and Emma Strubell},
    title     = {{The Framework Tax: Disparities Between Inference Efficiency in Research and Deployment}},
    journal = {ArXiv},
    year      = {2023},
    url       = {https://arxiv.org/abs/2302.06117},
}

Computational Language Acquisition with Theory of Mind

The Eleventh International Conference on Learning Representations (ICLR)
Andy Liu,

Hao Zhu

, Emmy Liu,

Yonatan Bisk

, Graham Neubig
Paper BibTeX Code


@inproceedings{Liu2023,
    author    = {Andy Liu and Hao Zhu and Emmy Liu and Yonatan Bisk and Graham Neubig},
    title     = {{Computational Language Acquisition with Theory of Mind}},
    booktitle = {The Eleventh International Conference on Learning Representations},
    year      = {2023},
    url       = {https://openreview.net/forum?id=C2ulri4duIs},
}

Self-Supervised Object Goal Navigation with In-Situ Finetuning

IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)

So Yeon Min

, Yao-Hung Hubert Tsai, Wei Ding, Ali Farhadi, Ruslan Salakhutdinov,

Yonatan Bisk

, Jian Zhang
Paper BibTeX Video


@inproceedings{Min2023,
    author    = {So Yeon Min and Yao-Hung Hubert Tsai and Wei Ding and Ali Farhadi and Ruslan Salakhutdinov and Yonatan Bisk and Jian Zhang},
    title     = {{Self-Supervised Object Goal Navigation with In-Situ Finetuning}},
    booktitle = {IEEE/RSJ International Conference on Intelligent Robots and Systems},
    year      = {2023},
    url       = {https://arxiv.org/abs/2212.05923},
}

MAEA: Multimodal Attribution for Embodied AI

Progress and Challenges in Building Trustworthy Embodied AI at NeurIPS 2022 (TEA)

Vidhi Jain

, Jayant Sravan Tamarapalli, Sahiti Yerramilli,

Yonatan Bisk

Paper BibTeX


@inproceedings{Jain2022,
    author    = {Vidhi Jain and Jayant Sravan Tamarapalli and Sahiti Yerramilli and Yonatan Bisk},
    title     = {{MAEA: Multimodal Attribution for Embodied AI}},
    booktitle = {Progress and Challenges in Building Trustworthy Embodied AI at NeurIPS 2022},
    year      = {2022},
    url       = {https://openreview.net/pdf?id=OUs2us5Xhx},
}

Tackling AlfWorld with Action Attention and Common Sense from Language Models

Language and Reinforcement Learning Workshop at NeurIPS 2022 (LaReL)
Yue Wu,

So Yeon Min

,

Yonatan Bisk

, Ruslan Salakhutdinov, Shrimai Prabhumoye
Paper BibTeX


@inproceedings{Wu2022,
    author    = {Yue Wu and So Yeon Min and Yonatan Bisk and Ruslan Salakhutdinov and Shrimai Prabhumoye},
    title     = {{Tackling AlfWorld with Action Attention and Common Sense from Language Models}},
    booktitle = {Language and Reinforcement Learning Workshop at NeurIPS 2022},
    year      = {2022},
    url       = {https://openreview.net/forum?id=AqkPRUZ-YkO},
}

Retrospectives on the Embodied AI Workshop

ArXiv (ArXiv)
Matt Deitke, Dhruv Batra,

Yonatan Bisk

, Tommaso Campari, Angel X. Chang, Devendra Singh Chaplot, Changan Chen, Claudia Pérez D'Arpino, Kiana Ehsani, Ali Farhadi, Li Fei-Fei, Anthony Francis, Chuang Gan, Kristen Grauman, David Hall, Winson Han, Unnat Jain, Aniruddha Kembhavi, Jacob Krantz, Stefan Lee, Chengshu Li, Sagnik Majumder, Oleksandr Maksymets, Roberto Martín-Martín, Roozbeh Mottaghi, Sonia Raychaudhuri, Mike Roberts, Silvio Savarese, Manolis Savva, Mohit Shridhar, Niko Sünderhauf, Andrew Szot, Ben Talbot, Joshua B. Tenenbaum, Jesse Thomason, Alexander Toshev, Joanne Truong, Luca Weihs, Jiajun Wu
Paper BibTeX


@article{Deitke2022,
    author    = {Matt Deitke and Dhruv Batra and Yonatan Bisk and Tommaso Campari and Angel X. Chang and Devendra Singh Chaplot and Changan Chen and Claudia Pérez D'Arpino and Kiana Ehsani and Ali Farhadi and Li Fei-Fei and Anthony Francis and Chuang Gan and Kristen Grauman and David Hall and Winson Han and Unnat Jain and Aniruddha Kembhavi and Jacob Krantz and Stefan Lee and Chengshu Li and Sagnik Majumder and Oleksandr Maksymets and Roberto Martín-Martín and Roozbeh Mottaghi and Sonia Raychaudhuri and Mike Roberts and Silvio Savarese and Manolis Savva and Mohit Shridhar and Niko Sünderhauf and Andrew Szot and Ben Talbot and Joshua B. Tenenbaum and Jesse Thomason and Alexander Toshev and Joanne Truong and Luca Weihs and Jiajun Wu},
    title     = {{Retrospectives on the Embodied AI Workshop}},
    journal = {ArXiv},
    year      = {2022},
    url       = {https://arxiv.org/abs/2210.06849},
}

Don't Copy the Teacher: Data and Model Challenges in Embodied Dialogue

Conference on Empirical Methods in Natural Language Processing (EMNLP)

So Yeon Min

,

Hao Zhu

, Ruslan Salakhutdinov,

Yonatan Bisk

Paper BibTeX Video


@inproceedings{Min2022,
    author    = {So Yeon Min and Hao Zhu and Ruslan Salakhutdinov and Yonatan Bisk},
    title     = {{Don't Copy the Teacher: Data and Model Challenges in Embodied Dialogue}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing},
    year      = {2022},
    url       = {https://arxiv.org/abs/2210.04443},
}

EvEntS ReaLM: Event Reasoning of Entity States via Language Models

Conference on Empirical Methods in Natural Language Processing (EMNLP)
Evangelia Spiliopoulou, Artidoro Pagnoni,

Yonatan Bisk

, Eduard Hovy
Paper BibTeX


@inproceedings{Spiliopoulou2022,
    author    = {Evangelia Spiliopoulou and Artidoro Pagnoni and Yonatan Bisk and Eduard Hovy},
    title     = {{EvEntS ReaLM: Event Reasoning of Entity States via Language Models}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing},
    year      = {2022},
    url       = {https://arxiv.org/abs/2211.05392},
}

On Advances in Text Generation from Images Beyond Captioning: A Case Study in Self-Rationalization

Findings of the Conference on Empirical Methods in Natural Language Processing (Findings of EMNLP)
Shruti Palaskar, Akshita Bhagia,

Yonatan Bisk

, Florian Metze, Alan W Black, Ana Marasovic
Paper BibTeX


@inproceedings{Palaskar2022,
    author    = {Shruti Palaskar and Akshita Bhagia and Yonatan Bisk and Florian Metze and Alan W Black and Ana Marasovic},
    title     = {{On Advances in Text Generation from Images Beyond Captioning: A Case Study in Self-Rationalization}},
    booktitle = {Findings of the Conference on Empirical Methods in Natural Language Processing},
    year      = {2022},
    url       = {https://arxiv.org/abs/2205.11686},
}

Transformers are Adaptable Task Planners

Conference on Robot Learning (CoRL)

Vidhi Jain

, Yixin Lin, Eric Undersander,

Yonatan Bisk

, Akshara Rai
Paper BibTeX Video


@inproceedings{Jain2022,
    author    = {Vidhi Jain and Yixin Lin and Eric Undersander and Yonatan Bisk and Akshara Rai},
    title     = {{Transformers are Adaptable Task Planners}},
    booktitle = {Conference on Robot Learning},
    year      = {2022},
    url       = {https://arxiv.org/abs/2207.02442},
}

Training Vision-Language Transformers from Captions

Transactions on Machine Learning Research (TMLR)
Liangke Gui,

Yingshan Chang

, Qiuyuan Huang, Subhojit Som, Alex Hauptmann, Jianfeng Gao,

Yonatan Bisk

Paper BibTeX Code


@article{Gui2023,
    author    = {Liangke Gui and Yingshan Chang and Qiuyuan Huang and Subhojit Som and Alex Hauptmann and Jianfeng Gao and Yonatan Bisk},
    title     = {{Training Vision-Language Transformers from Captions}},
    journal = {Transactions on Machine Learning Research},
    year      = {2023},
    url       = {https://arxiv.org/abs/2205.09256},
}

Symmetric Machine Theory of Mind

39th International Conference on Machine Learning (ICML)
Melanie Sclar, Graham Neubig,

Yonatan Bisk

Paper BibTeX Talk


@inproceedings{Sclar2022,
    author    = {Melanie Sclar and Graham Neubig and Yonatan Bisk},
    title     = {{Symmetric Machine Theory of Mind}},
    booktitle = {39th International Conference on Machine Learning},
    year      = {2022},
    url       = {https://proceedings.mlr.press/v162/sclar22a.html},
}

A Framework for Learning to Request Rich and Contextually Useful Information from Humans

39th International Conference on Machine Learning, (ICML)
Khanh Nguyen,

Yonatan Bisk

, Hal Daume III
Paper BibTeX Talk


@inproceedings{Nguyen2022,
    author    = {Khanh Nguyen and Yonatan Bisk and Hal Daume III},
    title     = {{A Framework for Learning to Request Rich and Contextually Useful Information from Humans}},
    booktitle = {39th International Conference on Machine Learning,},
    year      = {2022},
    url       = {http://arxiv.org/abs/2110.08258},
}

Simulated Language Learning from Communicative Goals and Linguistic Input

Annual Meeting of the Cognitive Science Society (CogSci (Oral))

Hao Zhu

,

Yonatan Bisk

, Graham Neubig
Paper BibTeX


@inproceedings{Zhu2022,
    author    = {Hao Zhu and Yonatan Bisk and Graham Neubig},
    title     = {{Simulated Language Learning from Communicative Goals and Linguistic Input}},
    booktitle = {Annual Meeting of the Cognitive Science Society},
    year      = {2022},
    url       = {https://escholarship.org/uc/item/7p65n371},
}

HEAR 2021: Holistic Evaluation of Audio Representations

Proceedings of Machine Learning Research (PMLR): NeurIPS 2021 Competition Track (PMLR)
Joseph Turian, Jordie Shier, Humair Raj Khan, Bhiksha Raj, Björn W. Schuller, Christian J. Steinmetz, Colin Malloy, George Tzanetakis, Gissel Velarde, Kirk McNally, Max Henry, Nicolas Pinto, Camille Noufi, Christian Clough, Dorien Herremans, Eduardo Fonseca, Jesse Engel, Justin Salamon, Philippe Esling, Pranay Manocha, Shinji Watanabe, Zeyu Jin,

Yonatan Bisk

Paper BibTeX


@inproceedings{Turian2022,
    author    = {Joseph Turian and Jordie Shier and Humair Raj Khan and Bhiksha Raj and Björn W. Schuller and Christian J. Steinmetz and Colin Malloy and George Tzanetakis and Gissel Velarde and Kirk McNally and Max Henry and Nicolas Pinto and Camille Noufi and Christian Clough and Dorien Herremans and Eduardo Fonseca and Jesse Engel and Justin Salamon and Philippe Esling and Pranay Manocha and Shinji Watanabe and Zeyu Jin and Yonatan Bisk},
    title     = {{HEAR 2021: Holistic Evaluation of Audio Representations}},
    booktitle = {Proceedings of Machine Learning Research (PMLR): NeurIPS 2021 Competition Track},
    year      = {2022},
    url       = {http://arxiv.org/abs/2203.03022},
}

KAT: A Knowledge Augmented Transformer for Vision-and-Language

Annual Conference of the North American Chapter of the Association for Computational Linguistics (NAACL (Oral))
Liangke Gui, Borui Wang, Qiuyuan Huang, Alex Hauptmann,

Yonatan Bisk

, Jianfeng Gao
Paper BibTeX Code


@inproceedings{Gui2022,
    author    = {Liangke Gui and Borui Wang and Qiuyuan Huang and Alex Hauptmann and Yonatan Bisk and Jianfeng Gao},
    title     = {{KAT: A Knowledge Augmented Transformer for Vision-and-Language}},
    booktitle = {Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    year      = {2022},
    url       = {https://arxiv.org/abs/2112.08614},
}

FILM: Following Instructions in Language with Modular Methods

The Tenth International Conference on Learning Representations (ICLR)

So Yeon Min

, Devendra Singh Chaplot, Pradeep Ravikumar,

Yonatan Bisk

, Ruslan Salakhutdinov
Paper BibTeX Code Video


@inproceedings{Min2022,
    author    = {So Yeon Min and Devendra Singh Chaplot and Pradeep Ravikumar and Yonatan Bisk and Ruslan Salakhutdinov},
    title     = {{FILM: Following Instructions in Language with Modular Methods}},
    booktitle = {The Tenth International Conference on Learning Representations},
    year      = {2022},
    url       = {https://soyeonm.github.io/FILM_webpage/},
}

WebQA: Multihop and Multimodal QA

Conference on Computer Vision and Pattern Recognition (CVPR (Oral))

Yingshan Chang

, Mridu Narang, Hisami Suzuki, Guihong Cao, Jianfeng Gao,

Yonatan Bisk

Paper BibTeX Project Leaderboard Challenge Summary


@inproceedings{Chang2022,
    author    = {Yingshan Chang and Mridu Narang and Hisami Suzuki and Guihong Cao and Jianfeng Gao and Yonatan Bisk},
    title     = {{WebQA: Multihop and Multimodal QA}},
    booktitle = {Conference on Computer Vision and Pattern Recognition},
    year      = {2022},
    url       = {https://arxiv.org/abs/2109.00590},
}

Dependency Induction Through the Lens of Visual Perception

The SIGNLL Conference on Computational Natural Language Learning (CoNLL)
Ruisi Su, Shruti Rijhwani,

Hao Zhu

, Junxian He, Xinyu Wang,

Yonatan Bisk

, Graham Neubig
Paper BibTeX


@inproceedings{Su2021,
    author    = {Ruisi Su and Shruti Rijhwani and Hao Zhu and Junxian He and Xinyu Wang and Yonatan Bisk and Graham Neubig},
    title     = {{Dependency Induction Through the Lens of Visual Perception}},
    booktitle = {The SIGNLL Conference on Computational Natural Language Learning},
    year      = {2021},
    url       = {https://arxiv.org/abs/2109.09790},
}

Language Grounding with 3D Objects

Conference on Robot Learning (CoRL)
Jesse Thomason, Mohit Shridhar,

Yonatan Bisk

, Chris Paxton, Luke Zettlemoyer
Paper BibTeX Data


@inproceedings{Thomason2021,
    author    = {Jesse Thomason and Mohit Shridhar and Yonatan Bisk and Chris Paxton and Luke Zettlemoyer},
    title     = {{Language Grounding with 3D Objects}},
    booktitle = {Conference on Robot Learning},
    year      = {2021},
    url       = {https://arxiv.org/abs/2107.12514},
}

TACo: Token-aware Cascade Contrastive Learning for Video-Text Alignment

International Conference on Computer Vision (ICCV)
Jianwei Yang,

Yonatan Bisk

, Jianfeng Gao
Paper BibTeX


@inproceedings{Yang2021,
    author    = {Jianwei Yang and Yonatan Bisk and Jianfeng Gao},
    title     = {{TACo: Token-aware Cascade Contrastive Learning for Video-Text Alignment}},
    booktitle = {International Conference on Computer Vision},
    year      = {2021},
    url       = {https://arxiv.org/abs/2108.09980},
}

Worst of Both Worlds: Biases Compound in Pre-trained Vision-and-Language Models

4th Workshop on Gender Bias in Natural Language Processing (GeBNLP)
Tejas Srinivasan,

Yonatan Bisk

Paper BibTeX


@inproceedings{Srinivasan2022,
    author    = {Tejas Srinivasan and Yonatan Bisk},
    title     = {{Worst of Both Worlds: Biases Compound in Pre-trained Vision-and-Language Models}},
    booktitle = {4th Workshop on Gender Bias in Natural Language Processing},
    year      = {2022},
    url       = {https://arxiv.org/abs/2104.08666},
}

KB-VLP: Knowledge Based Vision and Language Pretraining

ICML21 Workshop on Self-Supervised Learning (ICML-SSL)
Kezhen Chen, Qiuyuan Huang,

Yonatan Bisk

, Daniel McDuff, Jianfeng Gao
Paper BibTeX


@inproceedings{Chen2021,
    author    = {Kezhen Chen and Qiuyuan Huang and Yonatan Bisk and Daniel McDuff and Jianfeng Gao},
    title     = {{KB-VLP: Knowledge Based Vision and Language Pretraining}},
    booktitle = {ICML21 Workshop on Self-Supervised Learning},
    year      = {2021},
    url       = {https://www.microsoft.com/en-us/research/publication/kb-vlp-knowledge-based-vision-and-language-pretraining-2/},
}

Few-shot Language Coordination by Modeling Theory of Mind

The Thirty-eighth International Conference on Machine Learning (ICML)

Hao Zhu

, Graham Neubig,

Yonatan Bisk

Paper BibTeX Code


@inproceedings{Zhu2021,
    author    = {Hao Zhu and Graham Neubig and Yonatan Bisk},
    title     = {{Few-shot Language Coordination by Modeling Theory of Mind}},
    booktitle = {The Thirty-eighth International Conference on Machine Learning},
    year      = {2021},
    url       = {http://proceedings.mlr.press/v139/zhu21d.html},
}

Grounding `Grounding' in NLP

Findings of The 2021 Conference of the Association for Computational Linguistics (Findings of ACL)
Khyathi Raghavi Chandu,

Yonatan Bisk

, Alan W Black
Paper BibTeX


@inproceedings{Chandu2021,
    author    = {Khyathi Raghavi Chandu and Yonatan Bisk and Alan W Black},
    title     = {{Grounding `Grounding' in NLP}},
    booktitle = {Findings of The 2021 Conference of the Association for Computational Linguistics},
    year      = {2021},
    url       = {https://arxiv.org/abs/2106.02192},
}

An Empirical Study on the Generalization Power of Neural Representations Learned via Visual Guessing Games

The 2021 Conference of the European Chapter of the Association for Computational Linguistics (EACL (Oral))
Alessandro Suglia,

Yonatan Bisk

, Ioannis Konstas, Antonio Vergari, Emanuele Bastianelli, Andrea Vanzo, Oliver Lemon
Paper BibTeX


@inproceedings{Suglia2021,
    author    = {Alessandro Suglia and Yonatan Bisk and Ioannis Konstas and Antonio Vergari and Emanuele Bastianelli and Andrea Vanzo and Oliver Lemon},
    title     = {{An Empirical Study on the Generalization Power of Neural Representations Learned via Visual Guessing Games}},
    booktitle = {The 2021 Conference of the European Chapter of the Association for Computational Linguistics},
    year      = {2021},
    url       = {https://arxiv.org/abs/2102.00424},
}

Knowledge-driven Data Construction for Zero-shot Evaluation in Commonsense Question Answering

35th AAAI Conference on Artificial Intelligence (AAAI)
Kaixin Ma, Filip Ilievski, Jonathan Francis,

Yonatan Bisk

, Eric Nyberg, Alessandro Oltramari
Paper BibTeX Code


@inproceedings{Ma2021,
    author    = {Kaixin Ma and Filip Ilievski and Jonathan Francis and Yonatan Bisk and Eric Nyberg and Alessandro Oltramari},
    title     = {{Knowledge-driven Data Construction for Zero-shot Evaluation in Commonsense Question Answering}},
    booktitle = {35th AAAI Conference on Artificial Intelligence},
    year      = {2021},
    url       = {https://arxiv.org/abs/2011.03863},
}

ALFWorld: Aligning Text and Embodied Environments for Interactive Learning

International Conference on Learning Representations (ICLR)
Mohit Shridhar, Xingdi Yuan, Marc-Alexandre Côté,

Yonatan Bisk

, Adam Trischler, Matthew Hausknecht
Paper BibTeX Code


@inproceedings{Shridhar2021,
    author    = {Mohit Shridhar and Xingdi Yuan and Marc-Alexandre C{\^o}t{\'e} and Yonatan Bisk and Adam Trischler and Matthew Hausknecht},
    title     = {{ALFWorld: Aligning Text and Embodied Environments for Interactive Learning}},
    booktitle = {International Conference on Learning Representations},
    year      = {2021},
    url       = {https://alfworld.github.io/},
}

Imagining Grounded Conceptual Representations from Perceptual Information in Situated Guessing Games

The 28th International Conference on Computational Linguistics (Coling (Oral))
Alessandro Suglia, Antonio Vergari, Ioannis Konstas,

Yonatan Bisk

, Emanuele Bastianelli, Andrea Vanzo, Oliver Lemon
Paper BibTeX


@inproceedings{Suglia2020,
    author    = {Alessandro Suglia and Antonio Vergari and Ioannis Konstas and Yonatan Bisk and Emanuele Bastianelli and Andrea Vanzo and Oliver Lemon},
    title     = {{Imagining Grounded Conceptual Representations from Perceptual Information in Situated Guessing Games}},
    booktitle = {The 28th International Conference on Computational Linguistics},
    year      = {2020},
    url       = {https://arxiv.org/abs/2011.02917},
}

The Return of Lexical Dependencies: Neural Lexicalized PCFGs

Transactions of the Association for Computational Linguistics (TACL)

Hao Zhu

,

Yonatan Bisk

, Graham Neubig
Paper BibTeX Code


@article{Zhu2020,
    author    = {Hao Zhu and Yonatan Bisk and Graham Neubig},
    title     = {{The Return of Lexical Dependencies: Neural Lexicalized PCFGs}},
    journal = {Transactions of the Association for Computational Linguistics},
    year      = {2020},
    url       = {http://arxiv.org/abs/2007.15135},
}

Experience Grounds Language

Conference on Empirical Methods in Natural Language Processing (EMNLP) (EMNLP (Oral))

Yonatan Bisk

, Ari Holtzman, Jesse Thomason, Jacob Andreas, Yoshua Bengio, Joyce Chai, Mirella Lapata, Angeliki Lazaridou, Jonathan May, Aleksandr Nisnevich, Nicolas Pinto, Joseph Turian
Paper BibTeX Video Slides


@inproceedings{Bisk2020,
    author    = {Yonatan Bisk and Ari Holtzman and Jesse Thomason and Jacob Andreas and Yoshua Bengio and Joyce Chai and Mirella Lapata and Angeliki Lazaridou and Jonathan May and Aleksandr Nisnevich and Nicolas Pinto and Joseph Turian},
    title     = {{Experience Grounds Language}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    year      = {2020},
    url       = {https://arxiv.org/abs/2004.10151},
}

RMM: A Recursive Mental Model for Dialog Navigation

Conference on Empirical Methods in Natural Language Processing: Findings (Findings of EMNLP)
Homero Roman Roman,

Yonatan Bisk

, Jesse Thomason, Asli Celikyilmaz, Jianfeng Gao
Paper BibTeX Code


@inproceedings{Roman2020,
    author    = {Homero Roman Roman and Yonatan Bisk and Jesse Thomason and Asli Celikyilmaz and Jianfeng Gao},
    title     = {{RMM: A Recursive Mental Model for Dialog Navigation}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing: Findings},
    year      = {2020},
    url       = {https://arxiv.org/abs/2005.00728},
}

A Benchmark for Structured Procedural Knowledge Extraction from Cooking Videos

Workshop on NLP Beyond Text (NLPBT)
Frank F. Xu, Lei Ji, Botian Shi, Junyi Du, Graham Neubig,

Yonatan Bisk

, Nan Duan
Paper BibTeX Code


@inproceedings{Xu2020,
    author    = {Frank F. Xu and Lei Ji and Botian Shi and Junyi Du and Graham Neubig and Yonatan Bisk and Nan Duan},
    title     = {{A Benchmark for Structured Procedural Knowledge Extraction from Cooking Videos}},
    booktitle = {Workshop on NLP Beyond Text},
    year      = {2020},
    url       = {https://arxiv.org/abs/2005.00706},
}

Multi-View Learning for Vision-and-Language Navigation

ArXiv (ArXiv)
Qiaolin Xia, Xiujun Li, Chunyuan Li,

Yonatan Bisk

, Zhifang Sui, Jianfeng Gao, Yejin Choi, Noah A. Smith
Paper BibTeX


@article{Xia2020,
    author    = {Qiaolin Xia and Xiujun Li and Chunyuan Li and Yonatan Bisk and Zhifang Sui and Jianfeng Gao and Yejin Choi and Noah A. Smith},
    title     = {{Multi-View Learning for Vision-and-Language Navigation}},
    journal = {ArXiv},
    year      = {2020},
    url       = {https://arxiv.org/abs/2003.00857},
}

ALFRED: A Benchmark for Interpreting Grounded Instructions for Everyday Tasks

The IEEE Conference on Computer Vision and Pattern Recognition (CVPR) (CVPR)
Mohit Shridhar, Jesse Thomason, Daniel Gordon,

Yonatan Bisk

, Winson Han, Roozbeh Mottaghi, Luke Zettlemoyer, Dieter Fox
Paper BibTeX


@inproceedings{Shridhar2020,
    author    = {Mohit Shridhar and Jesse Thomason and Daniel Gordon and Yonatan Bisk and Winson Han and Roozbeh Mottaghi and Luke Zettlemoyer and Dieter Fox},
    title     = {{ALFRED: A Benchmark for Interpreting Grounded Instructions for Everyday Tasks}},
    booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
    year      = {2020},
    url       = {https://askforalfred.com/},
}

PIQA: Reasoning about Physical Commonsense in Natural Language

Thirty-Fourth AAAI Conference on Artificial Intelligence (AAAI (Oral))

Yonatan Bisk

, Rowan Zellers, Ronan Le Bras, Jianfeng Gao, Yejin Choi
Paper BibTeX Slides


@inproceedings{Bisk2020,
    author    = {Yonatan Bisk and Rowan Zellers and Ronan Le Bras and Jianfeng Gao and Yejin Choi},
    title     = {{PIQA: Reasoning about Physical Commonsense in Natural Language}},
    booktitle = {Thirty-Fourth AAAI Conference on Artificial Intelligence},
    year      = {2020},
    url       = {https://yonatanbisk.com/piqa},
}

Robust Navigation with Language Pretraining and Stochastic Sampling

Conference on Empirical Methods in Natural Language Processing and 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)
Xiujun Li, Chunyuan Li, Qiaolin Xia,

Yonatan Bisk

, Asli Celikyilmaz, Jianfeng Gao, Noah A. Smith, Yejin Choi
Paper BibTeX


@inproceedings{Li2019,
    author    = {Xiujun Li and Chunyuan Li and Qiaolin Xia and Yonatan Bisk and Asli Celikyilmaz and Jianfeng Gao and Noah A. Smith and Yejin Choi},
    title     = {{Robust Navigation with Language Pretraining and Stochastic Sampling}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing and 9th International Joint Conference on Natural Language Processing},
    year      = {2019},
    url       = {https://arxiv.org/abs/1909.02244},
}

Defending Against Neural Fake News

Thirty-third Conference on Neural Information Processing Systems (NeurIPS)
Rowan Zellers, Ari Holtzman, Hannah Rashkin,

Yonatan Bisk

, Ali Farhadi, Franziska Roesner, Yejin Choi
Paper BibTeX Demo Models


@inproceedings{Zellers2019,
    author    = {Rowan Zellers and Ari Holtzman and Hannah Rashkin and Yonatan Bisk and Ali Farhadi and Franziska Roesner and Yejin Choi},
    title     = {{Defending Against Neural Fake News}},
    booktitle = {Thirty-third Conference on Neural Information Processing Systems},
    year      = {2019},
    url       = {https://arxiv.org/abs/1905.12616},
}

FIND: Identifying Functionally and Structurally Important Features in Protein Sequences with Deep Neural Networks

bioRxiv (bioRxiv)
Ranjani Murali, James Hemp, Victoria Orphan,

Yonatan Bisk

Paper BibTeX


@article{Murali2019,
    author    = {Ranjani Murali and James Hemp and Victoria Orphan and Yonatan Bisk},
    title     = {{FIND: Identifying Functionally and Structurally Important Features in Protein Sequences with Deep Neural Networks}},
    journal = {bioRxiv},
    year      = {2019},
    url       = {https://www.biorxiv.org/content/10.1101/592808v1},
}

Improving Robot Success Detection using Static Object Data

IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)
Rosario Scalise, Jesse Thomason,

Yonatan Bisk

, Siddhartha Srinivasa
Paper BibTeX Video


@inproceedings{Scalise2019,
    author    = {Rosario Scalise and Jesse Thomason and Yonatan Bisk and Siddhartha Srinivasa},
    title     = {{Improving Robot Success Detection using Static Object Data}},
    booktitle = {IEEE/RSJ International Conference on Intelligent Robots and Systems},
    year      = {2019},
    url       = {https://arxiv.org/abs/1904.01650},
}

Early Fusion for Goal Directed Robotic Vision

2019 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)
Aaron Walsman,

Yonatan Bisk

, Saadia Gabriel, Dipendra Misra, Yoav Artzi, Yejin Choi, Dieter Fox
Paper BibTeX


@inproceedings{Walsman2019,
    author    = {Aaron Walsman and Yonatan Bisk and Saadia Gabriel and Dipendra Misra and Yoav Artzi and Yejin Choi and Dieter Fox},
    title     = {{Early Fusion for Goal Directed Robotic Vision}},
    booktitle = {2019 IEEE/RSJ International Conference on Intelligent Robots and Systems},
    year      = {2019},
    url       = {https://arxiv.org/abs/1811.08824},
}

HellaSwag: Can a Machine Really Finish Your Sentence?

Association for Computational Linguistics (ACL)
Rowan Zellers, Ari Holtzman,

Yonatan Bisk

, Ali Farhadi, Yejin Choi
Paper BibTeX


@inproceedings{Zellers2019,
    author    = {Rowan Zellers and Ari Holtzman and Yonatan Bisk and Ali Farhadi and Yejin Choi},
    title     = {{HellaSwag: Can a Machine Really Finish Your Sentence? }},
    booktitle = {Association for Computational Linguistics},
    year      = {2019},
    url       = {https://arxiv.org/abs/1905.07830},
}

From Recognition to Cognition: Visual Commonsense Reasoning

The IEEE Conference on Computer Vision and Pattern Recognition (CVPR) (CVPR (Oral))
Rowan Zellers,

Yonatan Bisk

, Ali Farhadi, Yejin Choi
Paper BibTeX


@inproceedings{Zellers2019,
    author    = {Rowan Zellers and Yonatan Bisk and Ali Farhadi and Yejin Choi},
    title     = {{From Recognition to Cognition: Visual Commonsense Reasoning}},
    booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
    year      = {2019},
    url       = {http://visualcommonsense.com/},
}

Tactical Rewind: Self-Correction via Backtracking in Vision-and-Language Navigation

The IEEE Conference on Computer Vision and Pattern Recognition (CVPR) (CVPR (Oral))
Liyiming Ke, Xiujun Li,

Yonatan Bisk

, Ari Holtzman, Zhe Gan, Jingjing Liu, Jianfeng Gao, Yejin Choi, Siddhartha Srinivasa
Paper BibTeX Video


@inproceedings{Ke2019,
    author    = {Liyiming Ke and Xiujun Li and Yonatan Bisk and Ari Holtzman and Zhe Gan and Jingjing Liu and Jianfeng Gao and Yejin Choi and Siddhartha Srinivasa},
    title     = {{Tactical Rewind: Self-Correction via Backtracking in Vision-and-Language Navigation}},
    booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
    year      = {2019},
    url       = {https://arxiv.org/abs/1903.02547},
}

Prospection: Interpretable Plans From Language By Predicting the Future

International Conference on Robotics and Automation (ICRA) (ICRA)
Chris Paxton,

Yonatan Bisk

, Jesse Thomason, Arunkumar Byravan, Dieter Fox
Paper BibTeX


@inproceedings{Paxton2019,
    author    = {Chris Paxton and Yonatan Bisk and Jesse Thomason and Arunkumar Byravan and Dieter Fox},
    title     = {{Prospection: Interpretable Plans From Language By Predicting the Future}},
    booktitle = {International Conference on Robotics and Automation (ICRA)},
    year      = {2019},
    url       = {https://arxiv.org/abs/1903.08309},
}

Shifting the Baseline: Single Modality Performance on Visual Navigation & QA

Annual Conference of the North American Chapter of the Association for Computational Linguistics (NAACL Short)
Jesse Thomason, Daniel Gordon,

Yonatan Bisk

Paper BibTeX


@inproceedings{Thomason2019,
    author    = {Jesse Thomason and Daniel Gordon and Yonatan Bisk},
    title     = {{Shifting the Baseline: Single Modality Performance on Visual Navigation & QA}},
    booktitle = {Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    year      = {2019},
    url       = {https://arxiv.org/abs/1811.00613},
}

Benchmarking Hierarchical Script Knowledge

Annual Conference of the North American Chapter of the Association for Computational Linguistics (NAACL Short (Oral))

Yonatan Bisk

, Jan Buys, Karl Pichotta, Yejin Choi
Paper BibTeX Code


@inproceedings{Bisk2019,
    author    = {Yonatan Bisk and Jan Buys and Karl Pichotta and Yejin Choi},
    title     = {{Benchmarking Hierarchical Script Knowledge}},
    booktitle = {Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    year      = {2019},
    url       = {https://www.aclweb.org/anthology/N19-1412},
}

Character-based Surprisal as a Model of Reading Difficulty in the Presence of Errors

The 41st Annual Meeting of the Cognitive Science Society (CogSci (Oral))
Michael Hahn, Frank Keller,

Yonatan Bisk

, Yonatan Belinkov
Paper BibTeX


@inproceedings{Hahn2019,
    author    = {Michael Hahn and Frank Keller and Yonatan Bisk and Yonatan Belinkov},
    title     = {{Character-based Surprisal as a Model of Reading Difficulty in the Presence of Errors}},
    booktitle = {The 41st Annual Meeting of the Cognitive Science Society},
    year      = {2019},
    url       = {https://psyarxiv.com/zcwgn/},
}

SWAG: A Large-Scale Adversarial Dataset for Grounded Commonsense Inference

Conference on Empirical Methods in Natural Language Processing (EMNLP) (EMNLP (Oral))
Rowan Zellers,

Yonatan Bisk

, Roy Schwartz, Yejin Choi
Paper BibTeX Talk


@inproceedings{Zellers2018,
    author    = {Rowan Zellers and Yonatan Bisk and Roy Schwartz and Yejin Choi},
    title     = {{SWAG: A Large-Scale Adversarial Dataset for Grounded Commonsense Inference}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    year      = {2018},
    url       = {http://rowanzellers.com/swag/},
}

Inducing Grammars with and for Neural Machine Translation

2nd Workshop on Neural Machine Translation (Wksp on NMT (best paper))
Ke Tran,

Yonatan Bisk

Paper BibTeX


@inproceedings{Tran2018,
    author    = {Ke Tran and Yonatan Bisk},
    title     = {{Inducing Grammars with and for Neural Machine Translation}},
    booktitle = {2nd Workshop on Neural Machine Translation},
    year      = {2018},
    url       = {https://arxiv.org/abs/1805.10850},
}

Bridging HMMs and RNNs through Architectural Transformations

32nd Conference on Neural Information Processing Systems (NIPS 2018), IRASL workshop (IRASL wksp @ NeurIPS)
Jan Buys,

Yonatan Bisk

, Yejin Choi
Paper BibTeX


@inproceedings{Buys2018,
    author    = {Jan Buys and Yonatan Bisk and Yejin Choi},
    title     = {{Bridging HMMs and RNNs through Architectural Transformations}},
    booktitle = {32nd Conference on Neural Information Processing Systems (NIPS 2018), IRASL workshop},
    year      = {2018},
    url       = {https://openreview.net/pdf?id=rJxEso0osm},
}

Synthetic and Natural Noise Both Break Neural Machine Translation

6th International Conference on Learning Representations (ICLR (Oral))
Yonatan Belinkov,

Yonatan Bisk

Paper BibTeX


@inproceedings{Belinkov2018,
    author    = {Yonatan Belinkov and Yonatan Bisk},
    title     = {{Synthetic and Natural Noise Both Break Neural Machine Translation }},
    booktitle = {6th International Conference on Learning Representations},
    year      = {2018},
    url       = {https://arxiv.org/abs/1711.02173},
}

CHALET: Cornell House Agent Learning Environment

ArXiv Preprint 1801.07357 (Tech Report)
Claudia Yan, Dipendra Misra, Andrew Bennett, Aaron Walsman,

Yonatan Bisk

, Yoav Artzi
Paper BibTeX Simulator


@article{Yan2018,
    author    = {Claudia Yan and Dipendra Misra and Andrew Bennett and Aaron Walsman and Yonatan Bisk and Yoav Artzi},
    title     = {{CHALET: Cornell House Agent Learning Environment }},
    journal = {ArXiv Preprint 1801.07357},
    year      = {2018},
    url       = {https://arxiv.org/abs/1801.07357},
}

Learning Interpretable Spatial Operations in a Rich 3D Blocks World

Thirty-Second Conference on Artificial Intelligence (AAAI-18) (AAAI)

Yonatan Bisk

, Kevin Shih, Yejin Choi, Daniel Marcu
Paper BibTeX Data Poster


@inproceedings{Bisk2018,
    author    = {Yonatan Bisk and Kevin Shih and Yejin Choi and Daniel Marcu},
    title     = {{Learning Interpretable Spatial Operations in a Rich 3D Blocks World }},
    booktitle = {Thirty-Second Conference on Artificial Intelligence (AAAI-18)},
    year      = {2018},
    url       = {https://arxiv.org/abs/1712.03463},
}

Natural Language Inference from Multiple Premises

Eighth International Joint Conference on Natural Language Processing (IJCNLP (Oral))
Alice Lai,

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX


@inproceedings{Lai2017,
    author    = {Alice Lai and Yonatan Bisk and Julia Hockenmaier},
    title     = {{Natural Language Inference from Multiple Premises}},
    booktitle = {Eighth International Joint Conference on Natural Language Processing},
    year      = {2017},
    url       = {https://arxiv.org/abs/1710.02925},
}

Natural Language Communication with Robots

15th Annual Conference of the North American Chapter of the Association for Computational Linguistics (NAACL (Oral))

Yonatan Bisk

, Deniz Yuret, Daniel Marcu
Paper BibTeX Data Slides


@inproceedings{Bisk2016,
    author    = {Yonatan Bisk and Deniz Yuret and Daniel Marcu},
    title     = {{Natural Language Communication with Robots }},
    booktitle = {15th Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    year      = {2016},
    url       = {https://www.aclweb.org/anthology/N16-1089/},
}

Supertagging with LSTMs

15th Annual Conference of the North American Chapter of the Association for Computational Linguistics (Short Papers) (NAACL Short (Oral))
Ashish Vaswani,

Yonatan Bisk

, Kenji Sagae, Ryan Musa
Paper BibTeX


@inproceedings{Vaswani2016,
    author    = {Ashish Vaswani and Yonatan Bisk and Kenji Sagae and Ryan Musa},
    title     = {{Supertagging with LSTMs}},
    booktitle = {15th Annual Conference of the North American Chapter of the Association for Computational Linguistics (Short Papers)},
    year      = {2016},
    url       = {https://www.aclweb.org/anthology/N16-1027/},
}

Towards a Dataset for Human Computer Communication via Grounded Language Acquisition

AAAI 2016 Workshop on Symbiotic Cognitive Systems (Symbolic Cognitive Systems Wksp)

Yonatan Bisk

, Daniel Marcu, William Wong
Paper BibTeX


@inproceedings{Bisk2016,
    author    = {Yonatan Bisk and Daniel Marcu and William Wong},
    title     = {{Towards a Dataset for Human Computer Communication via Grounded Language Acquisition}},
    booktitle = {AAAI 2016 Workshop on Symbiotic Cognitive Systems},
    year      = {2016},
    url       = {http://yonatanbisk.com/papers/2016-AAAI-Wksp.pdf},
}

Unsupervised Neural Hidden Markov Models

Workshop on Structured Prediction for NLP (Structure Pred Wksp (Oral))
Ke Tran,

Yonatan Bisk

, Ashish Vaswani, Daniel Marcu, Kevin Knight
Paper BibTeX Slides


@inproceedings{Tran2016,
    author    = {Ke Tran and Yonatan Bisk and Ashish Vaswani and Daniel Marcu and Kevin Knight},
    title     = {{Unsupervised Neural Hidden Markov Models  }},
    booktitle = {Workshop on Structured Prediction for NLP},
    year      = {2016},
    url       = {https://www.aclweb.org/anthology/W16-5907/},
}

Evaluating Induced CCG Parsers on Grounded Semantic Parsing

2016 Conference on Empirical Methods in Natural Language Processing (EMNLP)

Yonatan Bisk

, Siva Reddy, John Blitzer, Julia Hockenmaier, Mark Steedman
Paper BibTeX


@inproceedings{Bisk2016,
    author    = {Yonatan Bisk and Siva Reddy and John Blitzer and Julia Hockenmaier and Mark Steedman},
    title     = {{Evaluating Induced CCG Parsers on Grounded Semantic Parsing }},
    booktitle = {2016 Conference on Empirical Methods in Natural Language Processing},
    year      = {2016},
    url       = {http://sivareddy.in/papers/bisk2016evaluating.pdf},
}

Unsupervised Grammar Induction with Combinatory Categorial Grammars

University of Illinois at Urbana-Champaign (PhD Thesis)

Yonatan Bisk

Paper BibTeX


@phdthesis{Bisk2015,
    author    = {Yonatan Bisk},
    title     = {{Unsupervised Grammar Induction with Combinatory Categorial Grammars}},
    school = {University of Illinois at Urbana-Champaign},
    year      = {2015},
    url       = {http://yonatanbisk.com/papers/Thesis.pdf},
}

Probing the Linguistic Strengths and Limitations of Unsupervised Grammar Induction

53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 1: Long Papers) (ACL (Oral))

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX


@inproceedings{Bisk2015,
    author    = {Yonatan Bisk and Julia Hockenmaier},
    title     = {{Probing the Linguistic Strengths and Limitations of Unsupervised Grammar Induction}},
    booktitle = {53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
    year      = {2015},
    url       = {https://www.aclweb.org/anthology/P15-1135/},
}

Labeled Grammar Induction with Minimal Supervision

53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 2: Short Papers) (ACL Short)

Yonatan Bisk

, Christos Christodoulopoulos, Julia Hockenmaier
Paper BibTeX Poster


@inproceedings{Bisk2015,
    author    = {Yonatan Bisk and Christos Christodoulopoulos and Julia Hockenmaier},
    title     = {{Labeled Grammar Induction with Minimal Supervision  }},
    booktitle = {53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 2: Short Papers)},
    year      = {2015},
    url       = {http://yonatanbisk.com/papers/2015-ACLShort-Fixed.pdf},
}

An HDP Model for Inducing Combinatory Categorial Grammars

Transactions of the Association for Computational Linguistics (TACL Vol 1.)

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX Slides


@article{Bisk2013,
    author    = {Yonatan Bisk and Julia Hockenmaier},
    title     = {{An HDP Model for Inducing Combinatory Categorial Grammars}},
    journal = {Transactions of the Association for Computational Linguistics},
    year      = {2013},
    url       = {https://www.aclweb.org/anthology/Q13-1007/},
}

Simple Robust Grammar Induction with Combinatory Categorial Grammar

Twenty-Sixth Conference on Artificial Intelligence (AAAI-12) (AAAI)

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX


@inproceedings{Bisk2012,
    author    = {Yonatan Bisk and Julia Hockenmaier},
    title     = {{Simple Robust Grammar Induction with Combinatory Categorial Grammar }},
    booktitle = {Twenty-Sixth Conference on Artificial Intelligence (AAAI-12)},
    year      = {2012},
    url       = {https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5130},
}

Document-Topic Hierarchies from Document Graphs

21st ACM international conference on Information and knowledge management (CIKM 2012) (CIKM)
Tim Weninger,

Yonatan Bisk

, Jiawei Han
Paper BibTeX


@inproceedings{Weninger2012,
    author    = {Tim Weninger and Yonatan Bisk and Jiawei Han},
    title     = {{Document-Topic Hierarchies from Document Graphs }},
    booktitle = {21st ACM international conference on Information and knowledge management (CIKM 2012)},
    year      = {2012},
    url       = {http://yonatanbisk.com/papers/2012-CIKM.pdf},
}

Induction of Linguistic Structure with Combinatory Categorial Grammars

NAACL HLT Workshop on Induction of Linguistic Structure (Inducing Ling Structure Wksp)

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX


@inproceedings{Bisk2012,
    author    = {Yonatan Bisk and Julia Hockenmaier},
    title     = {{Induction of Linguistic Structure with Combinatory Categorial Grammars }},
    booktitle = {NAACL HLT Workshop on Induction of Linguistic Structure},
    year      = {2012},
    url       = {http://yonatanbisk.com/papers/2012-NAACL-WILS.pdf},
}

Normal-form parsing for CCGs with generalized composition and type-raising

23rd International Conference on Computational Linguistics (Coling 2010) (Coling)
Julia Hockenmaier,

Yonatan Bisk

Paper BibTeX


@inproceedings{Hockenmaier2010,
    author    = {Julia Hockenmaier and Yonatan Bisk},
    title     = {{Normal-form parsing for CCGs with generalized composition and type-raising}},
    booktitle = {23rd International Conference on Computational Linguistics (Coling 2010)},
    year      = {2010},
    url       = {https://www.aclweb.org/anthology/C10-1053/},
}

Connecting Language to Actions & the World @ CMU

EGL: You can't learn language ...

... from the radio
Text → Perception

... by watching TV
Perception → Action

... by yourself
Action → Social

Publications (preprints and workshops in grey)

Connecting Language to Actions & the World @ CMU

EGL: You can't learn language ...

... from the radio Text → Perception

... by watching TV Perception → Action

... by yourself Action → Social

Publications (preprints and workshops in grey)

... from the radio
Text → Perception

... by watching TV
Perception → Action

... by yourself
Action → Social