source/authored/reference.bib

@article{jelassi2024repeat,
  title={Repeat after me: Transformers are better than state space models at copying},
  author={Jelassi, Samy and Brandfonbrener, David and Kakade, Sham M and Malach, Eran},
  journal={arXiv preprint arXiv:2402.01032},
  year={2024}
}

@article{berger2003rate,
  title={Rate-distortion theory},
  author={Berger, Toby},
  journal={Wiley Encyclopedia of Telecommunications},
  year={2003},
  publisher={Wiley Online Library}
}

@inproceedings{peng2024limitations,
  title={On limitations of the transformer architecture},
  author={Peng, Binghui and Narayanan, Srini and Papadimitriou, Christos},
  booktitle={First Conference on Language Modeling},
  year={2024}
}

@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}

@inproceedings{gu2021efficiently,
  author       = {Albert Gu and
                  Karan Goel and
                  Christopher R{\'{e}}},
  title        = {Efficiently Modeling Long Sequences with Structured State Spaces},
  booktitle    = {The Tenth International Conference on Learning Representations, {ICLR}
                  2022, Virtual Event, April 25-29, 2022},
  publisher    = {OpenReview.net},
  year         = {2022},
  url          = {https://openreview.net/forum?id=uYLFoz1vlAC},
  timestamp    = {Sat, 20 Aug 2022 01:15:42 +0200},
  biburl       = {https://dblp.org/rec/conf/iclr/GuGR22.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{
gu2024mamba,
title={Mamba: Linear-Time Sequence Modeling with Selective State Spaces},
author={Albert Gu and Tri Dao},
booktitle={First Conference on Language Modeling},
year={2024},
url={https://openreview.net/forum?id=tEYskw1VY2}
}

@article{hochreiter1997long,
  title={Long short-term memory},
  author={Hochreiter, Sepp and Schmidhuber, J{\"u}rgen},
  journal={Neural computation},
  volume={9},
  number={8},
  pages={1735--1780},
  year={1997},
  publisher={MIT press}
}

@article{blakeman2025nemotron,
  title={Nemotron-h: A family of accurate and efficient hybrid mamba-transformer models},
  author={Blakeman, Aaron and Basant, Aarti and Khattar, Abhinav and Renduchintala, Adithya and Bercovich, Akhiad and Ficek, Aleksander and Bjorlin, Alexis and Taghibakhshi, Ali and Deshmukh, Amala Sanjay and Mahabaleshwarkar, Ameya Sunil and others},
  journal={arXiv preprint arXiv:2504.03624},
  year={2025}
}

@article{huang2024formal,
  title={A formal framework for understanding length generalization in transformers},
  author={Huang, Xinting and Yang, Andy and Bhattamishra, Satwik and Sarrof, Yash and Krebs, Andreas and Zhou, Hattie and Nakkiran, Preetum and Hahn, Michael},
  journal={arXiv preprint arXiv:2410.02140},
  year={2024}
}

@article{malach2025infinity,
  title={To Infinity and Beyond: Tool-Use Unlocks Length Generalization in State Space Models},
  author={Malach, Eran and Saremi, Omid and Williamson, Sinead and Bradley, Arwen and Lotfi, Aryo and Abbe, Emmanuel and Susskind, Josh and Littwin, Etai},
  journal={arXiv preprint arXiv:2510.14826},
  year={2025}
}

@inproceedings{li2025power,
  title={On the power of convolution-augmented transformer},
  author={Li, Mingchen and Zhang, Xuechen and Huang, Yixiao and Oymak, Samet},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={39},
  number={17},
  pages={18393--18402},
  year={2025}
}

@article{liu2023exposing,
  title={Exposing attention glitches with flip-flop language modeling},
  author={Liu, Bingbin and Ash, Jordan and Goel, Surbhi and Krishnamurthy, Akshay and Zhang, Cyril},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  pages={25549--25583},
  year={2023}
}



@article{arora2023zoology,
  title={Zoology: Measuring and improving recall in efficient language models},
  author={Arora, Simran and Eyuboglu, Sabri and Timalsina, Aman and Johnson, Isys and Poli, Michael and Zou, James and Rudra, Atri and R{\'e}, Christopher},
  journal={arXiv preprint arXiv:2312.04927},
  year={2023}
}


@article{park2024can,
  title={Can mamba learn how to learn? a comparative study on in-context learning tasks},
  author={Park, Jongho and Park, Jaeseung and Xiong, Zheyang and Lee, Nayoung and Cho, Jaewoong and Oymak, Samet and Lee, Kangwook and Papailiopoulos, Dimitris},
  journal={arXiv preprint arXiv:2402.04248},
  year={2024}
}

@article{lieber2024jamba,
  title={Jamba: A hybrid transformer-mamba language model},
  author={Lieber, Opher and Lenz, Barak and Bata, Hofit and Cohen, Gal and Osin, Jhonathan and Dalmedigos, Itay and Safahi, Erez and Meirom, Shaked and Belinkov, Yonatan and Shalev-Shwartz, Shai and others},
  journal={arXiv preprint arXiv:2403.19887},
  year={2024}
}

@article{zhan2025overcoming,
  title={Overcoming Long-Context Limitations of State-Space Models via Context-Dependent Sparse Attention},
  author={Zhan, Zhihao and Zhao, Jianan and Zhu, Zhaocheng and Tang, Jian},
  journal={arXiv preprint arXiv:2507.00449},
  year={2025}
}


@article{waleffe2024empirical,
  title={An empirical study of mamba-based language models},
  author={Waleffe, Roger and Byeon, Wonmin and Riach, Duncan and Norick, Brandon and Korthikanti, Vijay and Dao, Tri and Gu, Albert and Hatamizadeh, Ali and Singh, Sudhakar and Narayanan, Deepak and others},
  journal={arXiv preprint arXiv:2406.07887},
  year={2024}
}

@article{ren2024samba,
  title={Samba: Simple hybrid state space models for efficient unlimited context language modeling},
  author={Ren, Liliang and Liu, Yang and Lu, Yadong and Shen, Yelong and Liang, Chen and Chen, Weizhu},
  journal={arXiv preprint arXiv:2406.07522},
  year={2024}
}

@inproceedings{merrill2024illusion,
  title={The Illusion of State in State-Space Models},
  author={Merrill, William and Petty, Jackson and Sabharwal, Ashish},
  booktitle={International Conference on Machine Learning},
  pages={35492--35506},
  year={2024},
  organization={PMLR}
}

@article{merrill2023parallelism,
  title={The parallelism tradeoff: Limitations of log-precision transformers},
  author={Merrill, William and Sabharwal, Ashish},
  journal={Transactions of the Association for Computational Linguistics},
  volume={11},
  pages={531--545},
  year={2023},
  publisher={MIT Press One Broadway, 12th Floor, Cambridge, Massachusetts 02142, USA~…}
}


@article{chen2024theoretical,
  title={Theoretical limitations of multi-layer transformer},
  author={Chen, Lijie and Peng, Binghui and Wu, Hongxun},
  journal={arXiv preprint arXiv:2412.02975},
  year={2024}
}


@article{yehudai2025depth,
  title={Depth-width tradeoffs in algorithmic reasoning of graph tasks with transformers},
  author={Yehudai, Gilad and Sanford, Clayton and Bechler-Speicher, Maya and Fischer, Orr and Gilad-Bachrach, Ran and Globerson, Amir},
  journal={arXiv preprint arXiv:2503.01805},
  year={2025}
}

@inproceedings{lee2025understanding,
  title={Understanding and Enhancing Mamba-Transformer Hybrids for Memory Recall and Language Modeling},
  author={Lee, Hyunji and Yu, Wenhao and Zhang, Hongming and Ma, Kaixin and Kim, Jiyeon and Yu, Dong and Seo, Minjoon},
  booktitle={Proceedings of the First BabyLM Workshop},
  pages={380--398},
  year={2025}
}

@article{fu2022hungry,
  title={Hungry hungry hippos: Towards language modeling with state space models},
  author={Fu, Daniel Y and Dao, Tri and Saab, Khaled K and Thomas, Armin W and Rudra, Atri and R{\'e}, Christopher},
  journal={arXiv preprint arXiv:2212.14052},
  year={2022}
}

@article{dziri2023faith,
  title={Faith and fate: Limits of transformers on compositionality},
  author={Dziri, Nouha and Lu, Ximing and Sclar, Melanie and Li, Xiang Lorraine and Jiang, Liwei and Lin, Bill Yuchen and Welleck, Sean and West, Peter and Bhagavatula, Chandra and Le Bras, Ronan and others},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  pages={70293--70332},
  year={2023}
}


@article{elman1990finding,
  title={Finding structure in time},
  author={Elman, Jeffrey L},
  journal={Cognitive science},
  volume={14},
  number={2},
  pages={179--211},
  year={1990},
  publisher={Wiley Online Library}
}

@article{gu2020hippo,
  title={Hippo: Recurrent memory with optimal polynomial projections},
  author={Gu, Albert and Dao, Tri and Ermon, Stefano and Rudra, Atri and R{\'e}, Christopher},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={1474--1487},
  year={2020}
}



@article{gu2022parameterization,
  title={On the parameterization and initialization of diagonal state space models},
  author={Gu, Albert and Goel, Karan and Gupta, Ankit and R{\'e}, Christopher},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={35971--35983},
  year={2022}
}


@inproceedings{dai2019transformer,
  title={Transformer-xl: Attentive language models beyond a fixed-length context},
  author={Dai, Zihang and Yang, Zhilin and Yang, Yiming and Carbonell, Jaime G and Le, Quoc and Salakhutdinov, Ruslan},
  booktitle={Proceedings of the 57th annual meeting of the association for computational linguistics},
  pages={2978--2988},
  year={2019}
}

@inproceedings{black-etal-2022-gpt,
    title = "{GPT}-{N}eo{X}-20{B}: An Open-Source Autoregressive Language Model",
    author = "Black, Sidney  and
      Biderman, Stella  and
      Hallahan, Eric  and
      Anthony, Quentin  and
      Gao, Leo  and
      Golding, Laurence  and
      He, Horace  and
      Leahy, Connor  and
      McDonell, Kyle  and
      Phang, Jason  and
      Pieler, Michael  and
      Prashanth, Usvsn Sai  and
      Purohit, Shivanshu  and
      Reynolds, Laria  and
      Tow, Jonathan  and
      Wang, Ben  and
      Weinbach, Samuel",
    editor = "Fan, Angela  and
      Ilic, Suzana  and
      Wolf, Thomas  and
      Gall{\'e}, Matthias",
    booktitle = "Proceedings of BigScience Episode {\#}5 -- Workshop on Challenges {\&} Perspectives in Creating Large Language Models",
    month = may,
    year = "2022",
    address = "virtual+Dublin",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.bigscience-1.9/",
    doi = "10.18653/v1/2022.bigscience-1.9",
    pages = "95--136",
    abstract = "We introduce GPT-NeoX-20B, a 20 billion parameter autoregressive language model trained on the Pile, whose weights will be made freely and openly available to the public through a permissive license. It is, to the best of our knowledge, the largest dense autoregressive model that has publicly available weights at the time of submission. In this work, we describe GPT-NeoX-20B{'}s architecture and training, and evaluate its performance. We open-source the training and evaluation code, as well as the model weights, at \url{https://github.com/EleutherAI/gpt-neox}."
}

@article{Olsson2022IncontextLA,
  title={In-context Learning and Induction Heads},
  author={Catherine Olsson and Nelson Elhage and Neel Nanda and Nicholas Joseph and Nova Dassarma and T. J. Henighan and Benjamin Mann and Amanda Askell and Yuntao Bai and Anna Chen and Tom Conerly and Dawn Drain and Deep Ganguli and Zac Hatfield-Dodds and Danny Hernandez and Scott Johnston and Andy Jones and John Kernion and Liane Lovitt and Kamal Ndousse and Dario Amodei and Tom B. Brown and Jack Clark and Jared Kaplan and Sam McCandlish and Chris Olah},
  journal={ArXiv},
  year={2022},
  volume={abs/2209.11895},
  url={https://api.semanticscholar.org/CorpusID:252532078}
}