CLAW Publications

Computational Language Acquisition with Theory of Mind

The Eleventh International Conference on Learning Representations
Andy Liu,

Yonatan Bisk


@inproceedings{Liu2023,
    author    = {Andy Liu and Hao Zhu and Emmy Liu and Yonatan Bisk and Graham Neubig},
    title     = {{Computational Language Acquisition with Theory of Mind}},
    booktitle = {The Eleventh International Conference on Learning Representations},
    year      = {2023},
    url       = {https://openreview.net/forum?id=C2ulri4duIs},
}

Don't Copy the Teacher: Data and Model Challenges in Embodied Dialogue

Conference on Empirical Methods in Natural Language Processing

So Yeon Min

,

Hao Zhu

, Ruslan Salakhutdinov,

Yonatan Bisk

Paper BibTeX


@inproceedings{Min2022,
    author    = {So Yeon Min and Hao Zhu and Ruslan Salakhutdinov and Yonatan Bisk},
    title     = {{Don't Copy the Teacher: Data and Model Challenges in Embodied Dialogue}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing},
    year      = {2022},
    url       = {https://arxiv.org/abs/2210.04443},
}

EvEntS ReaLM: Event Reasoning of Entity States via Language Models

Conference on Empirical Methods in Natural Language Processing
Evangelia Spiliopoulou, Artidoro Pagnoni,

Yonatan Bisk

, Eduard Hovy
Paper BibTeX


@inproceedings{Spiliopoulou2022,
    author    = {Evangelia Spiliopoulou and Artidoro Pagnoni and Yonatan Bisk and Eduard Hovy},
    title     = {{EvEntS ReaLM: Event Reasoning of Entity States via Language Models}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing},
    year      = {2022},
    url       = {https://arxiv.org/abs/2211.05392},
}

On Advances in Text Generation from Images Beyond Captioning: A Case Study in Self-Rationalization

Findings of the Conference on Empirical Methods in Natural Language Processing
Shruti Palaskar, Akshita Bhagia,

Yonatan Bisk

, Florian Metze, Alan W Black, Ana Marasovic
Paper BibTeX


@inproceedings{Palaskar2022,
    author    = {Shruti Palaskar and Akshita Bhagia and Yonatan Bisk and Florian Metze and Alan W Black and Ana Marasovic},
    title     = {{On Advances in Text Generation from Images Beyond Captioning: A Case Study in Self-Rationalization}},
    booktitle = {Findings of the Conference on Empirical Methods in Natural Language Processing},
    year      = {2022},
    url       = {https://arxiv.org/abs/2205.11686},
}

Transformers are Adaptable Task Planners

Conference on Robot Learning

Vidhi Jain

, Yixin Lin, Eric Undersander,

Yonatan Bisk

, Akshara Rai
Paper BibTeX Video


@inproceedings{Jain2022,
    author    = {Vidhi Jain and Yixin Lin and Eric Undersander and Yonatan Bisk and Akshara Rai},
    title     = {{Transformers are Adaptable Task Planners}},
    booktitle = {Conference on Robot Learning},
    year      = {2022},
    url       = {https://arxiv.org/abs/2207.02442},
}

Symmetric Machine Theory of Mind

39th International Conference on Machine Learning
Melanie Sclar, Graham Neubig,

Yonatan Bisk

Paper BibTeX Talk


@inproceedings{Sclar2022,
    author    = {Melanie Sclar and Graham Neubig and Yonatan Bisk},
    title     = {{Symmetric Machine Theory of Mind}},
    booktitle = {39th International Conference on Machine Learning},
    year      = {2022},
    url       = {https://proceedings.mlr.press/v162/sclar22a.html},
}

A Framework for Learning to Request Rich and Contextually Useful Information from Humans

39th International Conference on Machine Learning,
Khanh Nguyen,

Yonatan Bisk

, Hal Daume III
Paper BibTeX Talk


@inproceedings{Nguyen2022,
    author    = {Khanh Nguyen and Yonatan Bisk and Hal Daume III},
    title     = {{A Framework for Learning to Request Rich and Contextually Useful Information from Humans}},
    booktitle = {39th International Conference on Machine Learning,},
    year      = {2022},
    url       = {http://arxiv.org/abs/2110.08258},
}

Simulated Language Learning from Communicative Goals and Linguistic Input

Annual Meeting of the Cognitive Science Society

Hao Zhu

,

Yonatan Bisk

, Graham Neubig
Paper BibTeX


@inproceedings{Zhu2022,
    author    = {Hao Zhu and Yonatan Bisk and Graham Neubig},
    title     = {{Simulated Language Learning from Communicative Goals and Linguistic Input}},
    booktitle = {Annual Meeting of the Cognitive Science Society},
    year      = {2022},
    url       = {https://escholarship.org/uc/item/7p65n371},
}

HEAR 2021: Holistic Evaluation of Audio Representations

Proceedings of Machine Learning Research (PMLR): NeurIPS 2021 Competition Track
Joseph Turian, Jordie Shier, Humair Raj Khan, Bhiksha Raj, Björn W. Schuller, Christian J. Steinmetz, Colin Malloy, George Tzanetakis, Gissel Velarde, Kirk McNally, Max Henry, Nicolas Pinto, Camille Noufi, Christian Clough, Dorien Herremans, Eduardo Fonseca, Jesse Engel, Justin Salamon, Philippe Esling, Pranay Manocha, Shinji Watanabe, Zeyu Jin,

Yonatan Bisk

Paper BibTeX


@inproceedings{Turian2022,
    author    = {Joseph Turian and Jordie Shier and Humair Raj Khan and Bhiksha Raj and Björn W. Schuller and Christian J. Steinmetz and Colin Malloy and George Tzanetakis and Gissel Velarde and Kirk McNally and Max Henry and Nicolas Pinto and Camille Noufi and Christian Clough and Dorien Herremans and Eduardo Fonseca and Jesse Engel and Justin Salamon and Philippe Esling and Pranay Manocha and Shinji Watanabe and Zeyu Jin and Yonatan Bisk},
    title     = {{HEAR 2021: Holistic Evaluation of Audio Representations}},
    booktitle = {Proceedings of Machine Learning Research (PMLR): NeurIPS 2021 Competition Track},
    year      = {2022},
    url       = {http://arxiv.org/abs/2203.03022},
}

KAT: A Knowledge Augmented Transformer for Vision-and-Language

Annual Conference of the North American Chapter of the Association for Computational Linguistics
Liangke Gui, Borui Wang, Qiuyuan Huang, Alex Hauptmann,

Yonatan Bisk

, Jianfeng Gao
Paper BibTeX Code


@inproceedings{Gui2022,
    author    = {Liangke Gui and Borui Wang and Qiuyuan Huang and Alex Hauptmann and Yonatan Bisk and Jianfeng Gao},
    title     = {{KAT: A Knowledge Augmented Transformer for Vision-and-Language}},
    booktitle = {Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    year      = {2022},
    url       = {https://arxiv.org/abs/2112.08614},
}

FILM: Following Instructions in Language with Modular Methods

The Tenth International Conference on Learning Representations

So Yeon Min

, Devendra Singh Chaplot, Pradeep Ravikumar,

Yonatan Bisk

, Ruslan Salakhutdinov
Paper BibTeX Code


@inproceedings{Min2022,
    author    = {So Yeon Min and Devendra Singh Chaplot and Pradeep Ravikumar and Yonatan Bisk and Ruslan Salakhutdinov},
    title     = {{FILM: Following Instructions in Language with Modular Methods}},
    booktitle = {The Tenth International Conference on Learning Representations},
    year      = {2022},
    url       = {https://soyeonm.github.io/FILM_webpage/},
}

WebQA: Multihop and Multimodal QA

Conference on Computer Vision and Pattern Recognition

Yingshan Chang

, Mridu Narang, Hisami Suzuki, Guihong Cao, Jianfeng Gao,

Yonatan Bisk

Paper BibTeX Project Leaderboard Challenge Summary


@inproceedings{Chang2022,
    author    = {Yingshan Chang and Mridu Narang and Hisami Suzuki and Guihong Cao and Jianfeng Gao and Yonatan Bisk},
    title     = {{WebQA: Multihop and Multimodal QA}},
    booktitle = {Conference on Computer Vision and Pattern Recognition},
    year      = {2022},
    url       = {https://arxiv.org/abs/2109.00590},
}

Dependency Induction Through the Lens of Visual Perception

The SIGNLL Conference on Computational Natural Language Learning
Ruisi Su, Shruti Rijhwani,

Hao Zhu

, Junxian He, Xinyu Wang,

Yonatan Bisk

, Graham Neubig
Paper BibTeX


@inproceedings{Su2021,
    author    = {Ruisi Su and Shruti Rijhwani and Hao Zhu and Junxian He and Xinyu Wang and Yonatan Bisk and Graham Neubig},
    title     = {{Dependency Induction Through the Lens of Visual Perception}},
    booktitle = {The SIGNLL Conference on Computational Natural Language Learning},
    year      = {2021},
    url       = {https://arxiv.org/abs/2109.09790},
}

Language Grounding with 3D Objects

Conference on Robot Learning
Jesse Thomason, Mohit Shridhar,

Yonatan Bisk

, Chris Paxton, Luke Zettlemoyer
Paper BibTeX Data


@inproceedings{Thomason2021,
    author    = {Jesse Thomason and Mohit Shridhar and Yonatan Bisk and Chris Paxton and Luke Zettlemoyer},
    title     = {{Language Grounding with 3D Objects}},
    booktitle = {Conference on Robot Learning},
    year      = {2021},
    url       = {https://arxiv.org/abs/2107.12514},
}

TACo: Token-aware Cascade Contrastive Learning for Video-Text Alignment

International Conference on Computer Vision
Jianwei Yang,

Yonatan Bisk

, Jianfeng Gao
Paper BibTeX


@inproceedings{Yang2021,
    author    = {Jianwei Yang and Yonatan Bisk and Jianfeng Gao},
    title     = {{TACo: Token-aware Cascade Contrastive Learning for Video-Text Alignment}},
    booktitle = {International Conference on Computer Vision},
    year      = {2021},
    url       = {https://arxiv.org/abs/2108.09980},
}

Few-shot Language Coordination by Modeling Theory of Mind

The Thirty-eighth International Conference on Machine Learning

Hao Zhu

, Graham Neubig,

Yonatan Bisk

Paper BibTeX Code


@inproceedings{Zhu2021,
    author    = {Hao Zhu and Graham Neubig and Yonatan Bisk},
    title     = {{Few-shot Language Coordination by Modeling Theory of Mind}},
    booktitle = {The Thirty-eighth International Conference on Machine Learning},
    year      = {2021},
    url       = {http://proceedings.mlr.press/v139/zhu21d.html},
}

Grounding `Grounding' in NLP

Findings of The 2021 Conference of the Association for Computational Linguistics
Khyathi Raghavi Chandu,

Yonatan Bisk

, Alan W Black
Paper BibTeX


@inproceedings{Chandu2021,
    author    = {Khyathi Raghavi Chandu and Yonatan Bisk and Alan W Black},
    title     = {{Grounding `Grounding' in NLP}},
    booktitle = {Findings of The 2021 Conference of the Association for Computational Linguistics},
    year      = {2021},
    url       = {https://arxiv.org/abs/2106.02192},
}

An Empirical Study on the Generalization Power of Neural Representations Learned via Visual Guessing Games

The 2021 Conference of the European Chapter of the Association for Computational Linguistics
Alessandro Suglia,

Yonatan Bisk

, Ioannis Konstas, Antonio Vergari, Emanuele Bastianelli, Andrea Vanzo, Oliver Lemon
Paper BibTeX


@inproceedings{Suglia2021,
    author    = {Alessandro Suglia and Yonatan Bisk and Ioannis Konstas and Antonio Vergari and Emanuele Bastianelli and Andrea Vanzo and Oliver Lemon},
    title     = {{An Empirical Study on the Generalization Power of Neural Representations Learned via Visual Guessing Games}},
    booktitle = {The 2021 Conference of the European Chapter of the Association for Computational Linguistics},
    year      = {2021},
    url       = {https://arxiv.org/abs/2102.00424},
}

Knowledge-driven Data Construction for Zero-shot Evaluation in Commonsense Question Answering

35th AAAI Conference on Artificial Intelligence
Kaixin Ma, Filip Ilievski, Jonathan Francis,

Yonatan Bisk

, Eric Nyberg, Alessandro Oltramari
Paper BibTeX Code


@inproceedings{Ma2021,
    author    = {Kaixin Ma and Filip Ilievski and Jonathan Francis and Yonatan Bisk and Eric Nyberg and Alessandro Oltramari},
    title     = {{Knowledge-driven Data Construction for Zero-shot Evaluation in Commonsense Question Answering}},
    booktitle = {35th AAAI Conference on Artificial Intelligence},
    year      = {2021},
    url       = {https://arxiv.org/abs/2011.03863},
}

ALFWorld: Aligning Text and Embodied Environments for Interactive Learning

International Conference on Learning Representations
Mohit Shridhar, Xingdi Yuan, Marc-Alexandre Côté,

Yonatan Bisk

, Adam Trischler, Matthew Hausknecht
Paper BibTeX Code


@inproceedings{Shridhar2021,
    author    = {Mohit Shridhar and Xingdi Yuan and Marc-Alexandre C{\^o}t{\'e} and Yonatan Bisk and Adam Trischler and Matthew Hausknecht},
    title     = {{ALFWorld: Aligning Text and Embodied Environments for Interactive Learning}},
    booktitle = {International Conference on Learning Representations},
    year      = {2021},
    url       = {https://alfworld.github.io/},
}

Imagining Grounded Conceptual Representations from Perceptual Information in Situated Guessing Games

The 28th International Conference on Computational Linguistics
Alessandro Suglia, Antonio Vergari, Ioannis Konstas,

Yonatan Bisk

, Emanuele Bastianelli, Andrea Vanzo, Oliver Lemon
Paper BibTeX


@inproceedings{Suglia2020,
    author    = {Alessandro Suglia and Antonio Vergari and Ioannis Konstas and Yonatan Bisk and Emanuele Bastianelli and Andrea Vanzo and Oliver Lemon},
    title     = {{Imagining Grounded Conceptual Representations from Perceptual Information in Situated Guessing Games}},
    booktitle = {The 28th International Conference on Computational Linguistics},
    year      = {2020},
    url       = {https://arxiv.org/abs/2011.02917},
}

The Return of Lexical Dependencies: Neural Lexicalized PCFGs

Transactions of the Association for Computational Linguistics

Hao Zhu

,

Yonatan Bisk

, Graham Neubig
Paper BibTeX Code


@article{Zhu2020,
    author    = {Hao Zhu and Yonatan Bisk and Graham Neubig},
    title     = {{The Return of Lexical Dependencies: Neural Lexicalized PCFGs}},
    journal = {Transactions of the Association for Computational Linguistics},
    year      = {2020},
    url       = {http://arxiv.org/abs/2007.15135},
}

Experience Grounds Language

Conference on Empirical Methods in Natural Language Processing (EMNLP)

Yonatan Bisk

, Ari Holtzman, Jesse Thomason, Jacob Andreas, Yoshua Bengio, Joyce Chai, Mirella Lapata, Angeliki Lazaridou, Jonathan May, Aleksandr Nisnevich, Nicolas Pinto, Joseph Turian
Paper BibTeX Video Slides


@inproceedings{Bisk2020,
    author    = {Yonatan Bisk and Ari Holtzman and Jesse Thomason and Jacob Andreas and Yoshua Bengio and Joyce Chai and Mirella Lapata and Angeliki Lazaridou and Jonathan May and Aleksandr Nisnevich and Nicolas Pinto and Joseph Turian},
    title     = {{Experience Grounds Language}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    year      = {2020},
    url       = {https://arxiv.org/abs/2004.10151},
}

RMM: A Recursive Mental Model for Dialog Navigation

Conference on Empirical Methods in Natural Language Processing: Findings
Homero Roman Roman,

Yonatan Bisk

, Jesse Thomason, Asli Celikyilmaz, Jianfeng Gao
Paper BibTeX Code


@inproceedings{Roman2020,
    author    = {Homero Roman Roman and Yonatan Bisk and Jesse Thomason and Asli Celikyilmaz and Jianfeng Gao},
    title     = {{RMM: A Recursive Mental Model for Dialog Navigation}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing: Findings},
    year      = {2020},
    url       = {https://arxiv.org/abs/2005.00728},
}

ALFRED: A Benchmark for Interpreting Grounded Instructions for Everyday Tasks

The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)
Mohit Shridhar, Jesse Thomason, Daniel Gordon,

Yonatan Bisk

, Winson Han, Roozbeh Mottaghi, Luke Zettlemoyer, Dieter Fox
Paper BibTeX


@inproceedings{Shridhar2020,
    author    = {Mohit Shridhar and Jesse Thomason and Daniel Gordon and Yonatan Bisk and Winson Han and Roozbeh Mottaghi and Luke Zettlemoyer and Dieter Fox},
    title     = {{ALFRED: A Benchmark for Interpreting Grounded Instructions for Everyday Tasks}},
    booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
    year      = {2020},
    url       = {https://askforalfred.com/},
}

PIQA: Reasoning about Physical Commonsense in Natural Language

Thirty-Fourth AAAI Conference on Artificial Intelligence

Yonatan Bisk

, Rowan Zellers, Ronan Le Bras, Jianfeng Gao, Yejin Choi
Paper BibTeX Slides


@inproceedings{Bisk2020,
    author    = {Yonatan Bisk and Rowan Zellers and Ronan Le Bras and Jianfeng Gao and Yejin Choi},
    title     = {{PIQA: Reasoning about Physical Commonsense in Natural Language}},
    booktitle = {Thirty-Fourth AAAI Conference on Artificial Intelligence},
    year      = {2020},
    url       = {https://yonatanbisk.com/piqa},
}

Robust Navigation with Language Pretraining and Stochastic Sampling

Conference on Empirical Methods in Natural Language Processing and 9th International Joint Conference on Natural Language Processing
Xiujun Li, Chunyuan Li, Qiaolin Xia,

Yonatan Bisk

, Asli Celikyilmaz, Jianfeng Gao, Noah A. Smith, Yejin Choi
Paper BibTeX


@inproceedings{Li2019,
    author    = {Xiujun Li and Chunyuan Li and Qiaolin Xia and Yonatan Bisk and Asli Celikyilmaz and Jianfeng Gao and Noah A. Smith and Yejin Choi},
    title     = {{Robust Navigation with Language Pretraining and Stochastic Sampling}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing and 9th International Joint Conference on Natural Language Processing},
    year      = {2019},
    url       = {https://arxiv.org/abs/1909.02244},
}

Defending Against Neural Fake News

Thirty-third Conference on Neural Information Processing Systems
Rowan Zellers, Ari Holtzman, Hannah Rashkin,

Yonatan Bisk

, Ali Farhadi, Franziska Roesner, Yejin Choi
Paper BibTeX Demo Models


@inproceedings{Zellers2019,
    author    = {Rowan Zellers and Ari Holtzman and Hannah Rashkin and Yonatan Bisk and Ali Farhadi and Franziska Roesner and Yejin Choi},
    title     = {{Defending Against Neural Fake News}},
    booktitle = {Thirty-third Conference on Neural Information Processing Systems},
    year      = {2019},
    url       = {https://arxiv.org/abs/1905.12616},
}

Improving Robot Success Detection using Static Object Data

IEEE/RSJ International Conference on Intelligent Robots and Systems
Rosario Scalise, Jesse Thomason,

Yonatan Bisk

, Siddhartha Srinivasa
Paper BibTeX Video


@inproceedings{Scalise2019,
    author    = {Rosario Scalise and Jesse Thomason and Yonatan Bisk and Siddhartha Srinivasa},
    title     = {{Improving Robot Success Detection using Static Object Data}},
    booktitle = {IEEE/RSJ International Conference on Intelligent Robots and Systems},
    year      = {2019},
    url       = {https://arxiv.org/abs/1904.01650},
}

Early Fusion for Goal Directed Robotic Vision

2019 IEEE/RSJ International Conference on Intelligent Robots and Systems
Aaron Walsman,

Yonatan Bisk

, Saadia Gabriel, Dipendra Misra, Yoav Artzi, Yejin Choi, Dieter Fox
Paper BibTeX


@inproceedings{Walsman2019,
    author    = {Aaron Walsman and Yonatan Bisk and Saadia Gabriel and Dipendra Misra and Yoav Artzi and Yejin Choi and Dieter Fox},
    title     = {{Early Fusion for Goal Directed Robotic Vision}},
    booktitle = {2019 IEEE/RSJ International Conference on Intelligent Robots and Systems},
    year      = {2019},
    url       = {https://arxiv.org/abs/1811.08824},
}

HellaSwag: Can a Machine Really Finish Your Sentence?

Association for Computational Linguistics
Rowan Zellers, Ari Holtzman,

Yonatan Bisk

, Ali Farhadi, Yejin Choi
Paper BibTeX


@inproceedings{Zellers2019,
    author    = {Rowan Zellers and Ari Holtzman and Yonatan Bisk and Ali Farhadi and Yejin Choi},
    title     = {{HellaSwag: Can a Machine Really Finish Your Sentence? }},
    booktitle = {Association for Computational Linguistics},
    year      = {2019},
    url       = {https://arxiv.org/abs/1905.07830},
}

From Recognition to Cognition: Visual Commonsense Reasoning

The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)
Rowan Zellers,

Yonatan Bisk

, Ali Farhadi, Yejin Choi
Paper BibTeX


@inproceedings{Zellers2019,
    author    = {Rowan Zellers and Yonatan Bisk and Ali Farhadi and Yejin Choi},
    title     = {{From Recognition to Cognition: Visual Commonsense Reasoning}},
    booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
    year      = {2019},
    url       = {http://visualcommonsense.com/},
}

Tactical Rewind: Self-Correction via Backtracking in Vision-and-Language Navigation

The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)
Liyiming Ke, Xiujun Li,

Yonatan Bisk

, Ari Holtzman, Zhe Gan, Jingjing Liu, Jianfeng Gao, Yejin Choi, Siddhartha Srinivasa
Paper BibTeX Video


@inproceedings{Ke2019,
    author    = {Liyiming Ke and Xiujun Li and Yonatan Bisk and Ari Holtzman and Zhe Gan and Jingjing Liu and Jianfeng Gao and Yejin Choi and Siddhartha Srinivasa},
    title     = {{Tactical Rewind: Self-Correction via Backtracking in Vision-and-Language Navigation}},
    booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
    year      = {2019},
    url       = {https://arxiv.org/abs/1903.02547},
}

Prospection: Interpretable Plans From Language By Predicting the Future

International Conference on Robotics and Automation (ICRA)
Chris Paxton,

Yonatan Bisk

, Jesse Thomason, Arunkumar Byravan, Dieter Fox
Paper BibTeX


@inproceedings{Paxton2019,
    author    = {Chris Paxton and Yonatan Bisk and Jesse Thomason and Arunkumar Byravan and Dieter Fox},
    title     = {{Prospection: Interpretable Plans From Language By Predicting the Future}},
    booktitle = {International Conference on Robotics and Automation (ICRA)},
    year      = {2019},
    url       = {https://arxiv.org/abs/1903.08309},
}

Shifting the Baseline: Single Modality Performance on Visual Navigation & QA

Annual Conference of the North American Chapter of the Association for Computational Linguistics
Jesse Thomason, Daniel Gordon,

Yonatan Bisk

Paper BibTeX


@inproceedings{Thomason2019,
    author    = {Jesse Thomason and Daniel Gordon and Yonatan Bisk},
    title     = {{Shifting the Baseline: Single Modality Performance on Visual Navigation & QA}},
    booktitle = {Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    year      = {2019},
    url       = {https://arxiv.org/abs/1811.00613},
}

Benchmarking Hierarchical Script Knowledge

Annual Conference of the North American Chapter of the Association for Computational Linguistics

Yonatan Bisk

, Jan Buys, Karl Pichotta, Yejin Choi
Paper BibTeX Code


@inproceedings{Bisk2019,
    author    = {Yonatan Bisk and Jan Buys and Karl Pichotta and Yejin Choi},
    title     = {{Benchmarking Hierarchical Script Knowledge}},
    booktitle = {Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    year      = {2019},
    url       = {https://www.aclweb.org/anthology/N19-1412},
}

Character-based Surprisal as a Model of Reading Difficulty in the Presence of Errors

The 41st Annual Meeting of the Cognitive Science Society
Michael Hahn, Frank Keller,

Yonatan Bisk

, Yonatan Belinkov
Paper BibTeX


@inproceedings{Hahn2019,
    author    = {Michael Hahn and Frank Keller and Yonatan Bisk and Yonatan Belinkov},
    title     = {{Character-based Surprisal as a Model of Reading Difficulty in the Presence of Errors}},
    booktitle = {The 41st Annual Meeting of the Cognitive Science Society},
    year      = {2019},
    url       = {https://psyarxiv.com/zcwgn/},
}

SWAG: A Large-Scale Adversarial Dataset for Grounded Commonsense Inference

Conference on Empirical Methods in Natural Language Processing (EMNLP)
Rowan Zellers,

Yonatan Bisk

, Roy Schwartz, Yejin Choi
Paper BibTeX Talk


@inproceedings{Zellers2018,
    author    = {Rowan Zellers and Yonatan Bisk and Roy Schwartz and Yejin Choi},
    title     = {{SWAG: A Large-Scale Adversarial Dataset for Grounded Commonsense Inference}},
    booktitle = {Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    year      = {2018},
    url       = {http://rowanzellers.com/swag/},
}

Synthetic and Natural Noise Both Break Neural Machine Translation

6th International Conference on Learning Representations
Yonatan Belinkov,

Yonatan Bisk

Paper BibTeX


@inproceedings{Belinkov2018,
    author    = {Yonatan Belinkov and Yonatan Bisk},
    title     = {{Synthetic and Natural Noise Both Break Neural Machine Translation }},
    booktitle = {6th International Conference on Learning Representations},
    year      = {2018},
    url       = {https://arxiv.org/abs/1711.02173},
}

Learning Interpretable Spatial Operations in a Rich 3D Blocks World

Thirty-Second Conference on Artificial Intelligence (AAAI-18)

Yonatan Bisk

, Kevin Shih, Yejin Choi, Daniel Marcu
Paper BibTeX Data Poster


@inproceedings{Bisk2018,
    author    = {Yonatan Bisk and Kevin Shih and Yejin Choi and Daniel Marcu},
    title     = {{Learning Interpretable Spatial Operations in a Rich 3D Blocks World }},
    booktitle = {Thirty-Second Conference on Artificial Intelligence (AAAI-18)},
    year      = {2018},
    url       = {https://arxiv.org/abs/1712.03463},
}

Natural Language Inference from Multiple Premises

Eighth International Joint Conference on Natural Language Processing
Alice Lai,

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX


@inproceedings{Lai2017,
    author    = {Alice Lai and Yonatan Bisk and Julia Hockenmaier},
    title     = {{Natural Language Inference from Multiple Premises}},
    booktitle = {Eighth International Joint Conference on Natural Language Processing},
    year      = {2017},
    url       = {https://arxiv.org/abs/1710.02925},
}

Natural Language Communication with Robots

15th Annual Conference of the North American Chapter of the Association for Computational Linguistics

Yonatan Bisk

, Deniz Yuret, Daniel Marcu
Paper BibTeX Data Slides


@inproceedings{Bisk2016,
    author    = {Yonatan Bisk and Deniz Yuret and Daniel Marcu},
    title     = {{Natural Language Communication with Robots }},
    booktitle = {15th Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    year      = {2016},
    url       = {https://www.aclweb.org/anthology/N16-1089/},
}

Supertagging with LSTMs

15th Annual Conference of the North American Chapter of the Association for Computational Linguistics (Short Papers)
Ashish Vaswani,

Yonatan Bisk

, Kenji Sagae, Ryan Musa
Paper BibTeX


@inproceedings{Vaswani2016,
    author    = {Ashish Vaswani and Yonatan Bisk and Kenji Sagae and Ryan Musa},
    title     = {{Supertagging with LSTMs}},
    booktitle = {15th Annual Conference of the North American Chapter of the Association for Computational Linguistics (Short Papers)},
    year      = {2016},
    url       = {https://www.aclweb.org/anthology/N16-1027/},
}

Evaluating Induced CCG Parsers on Grounded Semantic Parsing

2016 Conference on Empirical Methods in Natural Language Processing

Yonatan Bisk

, Siva Reddy, John Blitzer, Julia Hockenmaier, Mark Steedman
Paper BibTeX


@inproceedings{Bisk2016,
    author    = {Yonatan Bisk and Siva Reddy and John Blitzer and Julia Hockenmaier and Mark Steedman},
    title     = {{Evaluating Induced CCG Parsers on Grounded Semantic Parsing }},
    booktitle = {2016 Conference on Empirical Methods in Natural Language Processing},
    year      = {2016},
    url       = {http://sivareddy.in/papers/bisk2016evaluating.pdf},
}

Probing the Linguistic Strengths and Limitations of Unsupervised Grammar Induction

53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX


@inproceedings{Bisk2015,
    author    = {Yonatan Bisk and Julia Hockenmaier},
    title     = {{Probing the Linguistic Strengths and Limitations of Unsupervised Grammar Induction}},
    booktitle = {53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
    year      = {2015},
    url       = {https://www.aclweb.org/anthology/P15-1135/},
}

Labeled Grammar Induction with Minimal Supervision

53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 2: Short Papers)

Yonatan Bisk

, Christos Christodoulopoulos, Julia Hockenmaier
Paper BibTeX Poster


@inproceedings{Bisk2015,
    author    = {Yonatan Bisk and Christos Christodoulopoulos and Julia Hockenmaier},
    title     = {{Labeled Grammar Induction with Minimal Supervision  }},
    booktitle = {53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 2: Short Papers)},
    year      = {2015},
    url       = {http://yonatanbisk.com/papers/2015-ACLShort-Fixed.pdf},
}

An HDP Model for Inducing Combinatory Categorial Grammars

Transactions of the Association for Computational Linguistics

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX Slides


@article{Bisk2013,
    author    = {Yonatan Bisk and Julia Hockenmaier},
    title     = {{An HDP Model for Inducing Combinatory Categorial Grammars}},
    journal = {Transactions of the Association for Computational Linguistics},
    year      = {2013},
    url       = {https://www.aclweb.org/anthology/Q13-1007/},
}

Simple Robust Grammar Induction with Combinatory Categorial Grammar

Twenty-Sixth Conference on Artificial Intelligence (AAAI-12)

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX


@inproceedings{Bisk2012,
    author    = {Yonatan Bisk and Julia Hockenmaier},
    title     = {{Simple Robust Grammar Induction with Combinatory Categorial Grammar }},
    booktitle = {Twenty-Sixth Conference on Artificial Intelligence (AAAI-12)},
    year      = {2012},
    url       = {https://www.aaai.org/ocs/index.php/AAAI/AAAI12/paper/view/5130},
}

Document-Topic Hierarchies from Document Graphs

21st ACM international conference on Information and knowledge management (CIKM 2012)
Tim Weninger,

Yonatan Bisk

, Jiawei Han
Paper BibTeX


@inproceedings{Weninger2012,
    author    = {Tim Weninger and Yonatan Bisk and Jiawei Han},
    title     = {{Document-Topic Hierarchies from Document Graphs }},
    booktitle = {21st ACM international conference on Information and knowledge management (CIKM 2012)},
    year      = {2012},
    url       = {http://yonatanbisk.com/papers/2012-CIKM.pdf},
}

Normal-form parsing for CCGs with generalized composition and type-raising

23rd International Conference on Computational Linguistics (Coling 2010)
Julia Hockenmaier,

Yonatan Bisk

Paper BibTeX


@inproceedings{Hockenmaier2010,
    author    = {Julia Hockenmaier and Yonatan Bisk},
    title     = {{Normal-form parsing for CCGs with generalized composition and type-raising}},
    booktitle = {23rd International Conference on Computational Linguistics (Coling 2010)},
    year      = {2010},
    url       = {https://www.aclweb.org/anthology/C10-1053/},
}

The Framework Tax: Disparities Between Inference Efficiency in Research and Deployment

ArXiv

Jared Fernandez

, Jacob Kahn, Clara Na,

Yonatan Bisk

, Emma Strubell
Paper BibTeX


@article{Fernandez2023,
    author    = {Jared Fernandez and Jacob Kahn and Clara Na and Yonatan Bisk and Emma Strubell},
    title     = {{The Framework Tax: Disparities Between Inference Efficiency in Research and Deployment}},
    journal = {ArXiv},
    year      = {2023},
    url       = {https://arxiv.org/abs/2302.06117},
}

Object Goal Navigation with End-to-End Self-Supervision

ArXiv

So Yeon Min

, Yao-Hung Hubert Tsai, Wei Ding, Ali Farhadi, Ruslan Salakhutdinov,

Yonatan Bisk

, Jian Zhang
Paper BibTeX


@article{Min2022,
    author    = {So Yeon Min and Yao-Hung Hubert Tsai and Wei Ding and Ali Farhadi and Ruslan Salakhutdinov and Yonatan Bisk and Jian Zhang},
    title     = {{Object Goal Navigation with End-to-End Self-Supervision}},
    journal = {ArXiv},
    year      = {2022},
    url       = {https://soyeonm.github.io/OGN_SelfSup_webpage/},
}

MAEA: Multimodal Attribution for Embodied AI

Progress and Challenges in Building Trustworthy Embodied AI at NeurIPS 2022

Vidhi Jain

, Jayant Sravan Tamarapalli, Sahiti Yerramilli,

Yonatan Bisk

Paper BibTeX


@inproceedings{Jain2022,
    author    = {Vidhi Jain and Jayant Sravan Tamarapalli and Sahiti Yerramilli and Yonatan Bisk},
    title     = {{MAEA: Multimodal Attribution for Embodied AI}},
    booktitle = {Progress and Challenges in Building Trustworthy Embodied AI at NeurIPS 2022},
    year      = {2022},
    url       = {https://openreview.net/pdf?id=OUs2us5Xhx},
}

Tackling AlfWorld with Action Attention and Common Sense from Language Models

Language and Reinforcement Learning Workshop at NeurIPS 2022
Yue Wu,

So Yeon Min

,

Yonatan Bisk

, Ruslan Salakhutdinov, Shrimai Prabhumoye
Paper BibTeX


@inproceedings{Wu2022,
    author    = {Yue Wu and So Yeon Min and Yonatan Bisk and Ruslan Salakhutdinov and Shrimai Prabhumoye},
    title     = {{Tackling AlfWorld with Action Attention and Common Sense from Language Models}},
    booktitle = {Language and Reinforcement Learning Workshop at NeurIPS 2022},
    year      = {2022},
    url       = {https://openreview.net/forum?id=AqkPRUZ-YkO},
}

Retrospectives on the Embodied AI Workshop

ArXiv
Matt Deitke, Dhruv Batra,

Yonatan Bisk

, Tommaso Campari, Angel X. Chang, Devendra Singh Chaplot, Changan Chen, Claudia Pérez D'Arpino, Kiana Ehsani, Ali Farhadi, Li Fei-Fei, Anthony Francis, Chuang Gan, Kristen Grauman, David Hall, Winson Han, Unnat Jain, Aniruddha Kembhavi, Jacob Krantz, Stefan Lee, Chengshu Li, Sagnik Majumder, Oleksandr Maksymets, Roberto Martín-Martín, Roozbeh Mottaghi, Sonia Raychaudhuri, Mike Roberts, Silvio Savarese, Manolis Savva, Mohit Shridhar, Niko Sünderhauf, Andrew Szot, Ben Talbot, Joshua B. Tenenbaum, Jesse Thomason, Alexander Toshev, Joanne Truong, Luca Weihs, Jiajun Wu
Paper BibTeX


@article{Deitke2022,
    author    = {Matt Deitke and Dhruv Batra and Yonatan Bisk and Tommaso Campari and Angel X. Chang and Devendra Singh Chaplot and Changan Chen and Claudia Pérez D'Arpino and Kiana Ehsani and Ali Farhadi and Li Fei-Fei and Anthony Francis and Chuang Gan and Kristen Grauman and David Hall and Winson Han and Unnat Jain and Aniruddha Kembhavi and Jacob Krantz and Stefan Lee and Chengshu Li and Sagnik Majumder and Oleksandr Maksymets and Roberto Martín-Martín and Roozbeh Mottaghi and Sonia Raychaudhuri and Mike Roberts and Silvio Savarese and Manolis Savva and Mohit Shridhar and Niko Sünderhauf and Andrew Szot and Ben Talbot and Joshua B. Tenenbaum and Jesse Thomason and Alexander Toshev and Joanne Truong and Luca Weihs and Jiajun Wu},
    title     = {{Retrospectives on the Embodied AI Workshop}},
    journal = {ArXiv},
    year      = {2022},
    url       = {https://arxiv.org/abs/2210.06849},
}

Training Vision-Language Transformers from Captions Alone

ArXiv
Liangke Gui, Qiuyuan Huang, Alex Hauptmann,

Yonatan Bisk

, Jianfeng Gao
Paper BibTeX Code


@article{Gui2022,
    author    = {Liangke Gui and Qiuyuan Huang and Alex Hauptmann and Yonatan Bisk and Jianfeng Gao},
    title     = {{Training Vision-Language Transformers from Captions Alone}},
    journal = {ArXiv},
    year      = {2022},
    url       = {https://arxiv.org/abs/2205.09256},
}

Worst of Both Worlds: Biases Compound in Pre-trained Vision-and-Language Models

4th Workshop on Gender Bias in Natural Language Processing
Tejas Srinivasan,

Yonatan Bisk

Paper BibTeX


@inproceedings{Srinivasan2022,
    author    = {Tejas Srinivasan and Yonatan Bisk},
    title     = {{Worst of Both Worlds: Biases Compound in Pre-trained Vision-and-Language Models}},
    booktitle = {4th Workshop on Gender Bias in Natural Language Processing},
    year      = {2022},
    url       = {https://arxiv.org/abs/2104.08666},
}

KB-VLP: Knowledge Based Vision and Language Pretraining

ICML21 Workshop on Self-Supervised Learning
Kezhen Chen, Qiuyuan Huang,

Yonatan Bisk

, Daniel McDuff, Jianfeng Gao
Paper BibTeX


@inproceedings{Chen2021,
    author    = {Kezhen Chen and Qiuyuan Huang and Yonatan Bisk and Daniel McDuff and Jianfeng Gao},
    title     = {{KB-VLP: Knowledge Based Vision and Language Pretraining}},
    booktitle = {ICML21 Workshop on Self-Supervised Learning},
    year      = {2021},
    url       = {https://www.microsoft.com/en-us/research/publication/kb-vlp-knowledge-based-vision-and-language-pretraining-2/},
}

A Benchmark for Structured Procedural Knowledge Extraction from Cooking Videos

Workshop on NLP Beyond Text
Frank F. Xu, Lei Ji, Botian Shi, Junyi Du, Graham Neubig,

Yonatan Bisk

, Nan Duan
Paper BibTeX Code


@inproceedings{Xu2020,
    author    = {Frank F. Xu and Lei Ji and Botian Shi and Junyi Du and Graham Neubig and Yonatan Bisk and Nan Duan},
    title     = {{A Benchmark for Structured Procedural Knowledge Extraction from Cooking Videos}},
    booktitle = {Workshop on NLP Beyond Text},
    year      = {2020},
    url       = {https://arxiv.org/abs/2005.00706},
}

Multi-View Learning for Vision-and-Language Navigation

ArXiv
Qiaolin Xia, Xiujun Li, Chunyuan Li,

Yonatan Bisk

, Zhifang Sui, Jianfeng Gao, Yejin Choi, Noah A. Smith
Paper BibTeX


@article{Xia2020,
    author    = {Qiaolin Xia and Xiujun Li and Chunyuan Li and Yonatan Bisk and Zhifang Sui and Jianfeng Gao and Yejin Choi and Noah A. Smith},
    title     = {{Multi-View Learning for Vision-and-Language Navigation}},
    journal = {ArXiv},
    year      = {2020},
    url       = {https://arxiv.org/abs/2003.00857},
}

FIND: Identifying Functionally and Structurally Important Features in Protein Sequences with Deep Neural Networks

bioRxiv
Ranjani Murali, James Hemp, Victoria Orphan,

Yonatan Bisk

Paper BibTeX


@article{Murali2019,
    author    = {Ranjani Murali and James Hemp and Victoria Orphan and Yonatan Bisk},
    title     = {{FIND: Identifying Functionally and Structurally Important Features in Protein Sequences with Deep Neural Networks}},
    journal = {bioRxiv},
    year      = {2019},
    url       = {https://www.biorxiv.org/content/10.1101/592808v1},
}

Inducing Grammars with and for Neural Machine Translation

2nd Workshop on Neural Machine Translation
Ke Tran,

Yonatan Bisk

Paper BibTeX


@inproceedings{Tran2018,
    author    = {Ke Tran and Yonatan Bisk},
    title     = {{Inducing Grammars with and for Neural Machine Translation}},
    booktitle = {2nd Workshop on Neural Machine Translation},
    year      = {2018},
    url       = {https://arxiv.org/abs/1805.10850},
}

Bridging HMMs and RNNs through Architectural Transformations

32nd Conference on Neural Information Processing Systems (NIPS 2018), IRASL workshop
Jan Buys,

Yonatan Bisk

, Yejin Choi
Paper BibTeX


@inproceedings{Buys2018,
    author    = {Jan Buys and Yonatan Bisk and Yejin Choi},
    title     = {{Bridging HMMs and RNNs through Architectural Transformations}},
    booktitle = {32nd Conference on Neural Information Processing Systems (NIPS 2018), IRASL workshop},
    year      = {2018},
    url       = {https://openreview.net/pdf?id=rJxEso0osm},
}

CHALET: Cornell House Agent Learning Environment

ArXiv Preprint 1801.07357
Claudia Yan, Dipendra Misra, Andrew Bennett, Aaron Walsman,

Yonatan Bisk

, Yoav Artzi
Paper BibTeX Simulator


@article{Yan2018,
    author    = {Claudia Yan and Dipendra Misra and Andrew Bennett and Aaron Walsman and Yonatan Bisk and Yoav Artzi},
    title     = {{CHALET: Cornell House Agent Learning Environment }},
    journal = {ArXiv Preprint 1801.07357},
    year      = {2018},
    url       = {https://arxiv.org/abs/1801.07357},
}

Towards a Dataset for Human Computer Communication via Grounded Language Acquisition

AAAI 2016 Workshop on Symbiotic Cognitive Systems

Yonatan Bisk

, Daniel Marcu, William Wong
Paper BibTeX


@inproceedings{Bisk2016,
    author    = {Yonatan Bisk and Daniel Marcu and William Wong},
    title     = {{Towards a Dataset for Human Computer Communication via Grounded Language Acquisition}},
    booktitle = {AAAI 2016 Workshop on Symbiotic Cognitive Systems},
    year      = {2016},
    url       = {http://yonatanbisk.com/papers/2016-AAAI-Wksp.pdf},
}

Unsupervised Neural Hidden Markov Models

Workshop on Structured Prediction for NLP
Ke Tran,

Yonatan Bisk

, Ashish Vaswani, Daniel Marcu, Kevin Knight
Paper BibTeX Slides


@inproceedings{Tran2016,
    author    = {Ke Tran and Yonatan Bisk and Ashish Vaswani and Daniel Marcu and Kevin Knight},
    title     = {{Unsupervised Neural Hidden Markov Models  }},
    booktitle = {Workshop on Structured Prediction for NLP},
    year      = {2016},
    url       = {https://www.aclweb.org/anthology/W16-5907/},
}

Unsupervised Grammar Induction with Combinatory Categorial Grammars

University of Illinois at Urbana-Champaign

Yonatan Bisk

Paper BibTeX


@phdthesis{Bisk2015,
    author    = {Yonatan Bisk},
    title     = {{Unsupervised Grammar Induction with Combinatory Categorial Grammars}},
    school = {University of Illinois at Urbana-Champaign},
    year      = {2015},
    url       = {http://yonatanbisk.com/papers/Thesis.pdf},
}

Induction of Linguistic Structure with Combinatory Categorial Grammars

NAACL HLT Workshop on Induction of Linguistic Structure

Yonatan Bisk

, Julia Hockenmaier
Paper BibTeX


@inproceedings{Bisk2012,
    author    = {Yonatan Bisk and Julia Hockenmaier},
    title     = {{Induction of Linguistic Structure with Combinatory Categorial Grammars }},
    booktitle = {NAACL HLT Workshop on Induction of Linguistic Structure},
    year      = {2012},
    url       = {http://yonatanbisk.com/papers/2012-NAACL-WILS.pdf},
}

Connecting Language to Actions & the World @ CMU

EGL: You can't learn language ...

... from the radio
Text → Perception

... by watching TV
Perception → Action

... by yourself
Action → Social

Journals and Conferences

Preprints, Tech reports, and Workshops

Connecting Language to Actions & the World @ CMU

EGL: You can't learn language ...

... from the radio Text → Perception

... by watching TV Perception → Action

... by yourself Action → Social

Journals and Conferences

Preprints, Tech reports, and Workshops

... from the radio
Text → Perception

... by watching TV
Perception → Action

... by yourself
Action → Social