add more citations

app/src/content/bibliography.bib

@@ -57,6 +57,16 @@
   note          = {Blog post}
 }
 
+@misc{s1k,
+  title         = {s1: Simple Test-Time Scaling},
+  author        = {Niklas Muennighoff and Zitong Yang and Weijia Shi and Xiang Lisa Li and Li Fei-Fei and Hannaneh Hajishirzi and Luke Zettlemoyer and Percy Liang and Emmanuel Candès and Tatsunori Hashimoto},
+  year          = {2025},
+  eprint        = {2501.19393},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/2501.19393}
+}
+
 % Synthetic data methods
 @inproceedings{wrap,
   title         = {Rephrasing the Web: A Recipe for Compute and Data-Efficient Language Modeling},
@@ -110,6 +120,14 @@
   url           = {https://arxiv.org/abs/2502.02737}
 }
 
+@misc{smollm3,
+  title         = {SmolLM3: Smol, Multilingual, Long-Context Reasoner},
+  author        = {{Hugging Face}},
+  year          = {2025},
+  url           = {https://huggingface.co/blog/smollm3},
+  note          = {Blog post}
+}
+
 @misc{glm45,
   title         = {GLM-4.5: Agentic, Reasoning, and Coding (ARC) Foundation Models},
   author        = { 5 Team and Aohan Zeng and Xin Lv and Qinkai Zheng and Zhenyu Hou and Bin Chen and Chengxing Xie and Cunxiang Wang and Da Yin and Hao Zeng and Jiajie Zhang and Kedong Wang and Lucen Zhong and Mingdao Liu and Rui Lu and Shulin Cao and Xiaohan Zhang and Xuancheng Huang and Yao Wei and Yean Cheng and Yifan An and Yilin Niu and Yuanhao Wen and Yushi Bai and Zhengxiao Du and Zihan Wang and Zilin Zhu and Bohan Zhang and Bosi Wen and Bowen Wu and Bowen Xu and Can Huang and Casey Zhao and Changpeng Cai and Chao Yu and Chen Li and Chendi Ge and Chenghua Huang and Chenhui Zhang and Chenxi Xu and Chenzheng Zhu and Chuang Li and Congfeng Yin and Daoyan Lin and Dayong Yang and Dazhi Jiang and Ding Ai and Erle Zhu and Fei Wang and Gengzheng Pan and Guo Wang and Hailong Sun and Haitao Li and Haiyang Li and Haiyi Hu and Hanyu Zhang and Hao Peng and Hao Tai and Haoke Zhang and Haoran Wang and Haoyu Yang and He Liu and He Zhao and Hongwei Liu and Hongxi Yan and Huan Liu and Huilong Chen and Ji Li and Jiajing Zhao and Jiamin Ren and Jian Jiao and Jiani Zhao and Jianyang Yan and Jiaqi Wang and Jiayi Gui and Jiayue Zhao and Jie Liu and Jijie Li and Jing Li and Jing Lu and Jingsen Wang and Jingwei Yuan and Jingxuan Li and Jingzhao Du and Jinhua Du and Jinxin Liu and Junkai Zhi and Junli Gao and Ke Wang and Lekang Yang and Liang Xu and Lin Fan and Lindong Wu and Lintao Ding and Lu Wang and Man Zhang and Minghao Li and Minghuan Xu and Mingming Zhao and Mingshu Zhai and Pengfan Du and Qian Dong and Shangde Lei and Shangqing Tu and Shangtong Yang and Shaoyou Lu and Shijie Li and Shuang Li and Shuang-Li and Shuxun Yang and Sibo Yi and Tianshu Yu and Wei Tian and Weihan Wang and Wenbo Yu and Weng Lam Tam and Wenjie Liang and Wentao Liu and Xiao Wang and Xiaohan Jia and Xiaotao Gu and Xiaoying Ling and Xin Wang and Xing Fan and Xingru Pan and Xinyuan Zhang and Xinze Zhang and Xiuqing Fu and Xunkai Zhang and Yabo Xu and Yandong Wu and Yida Lu and Yidong Wang and Yilin Zhou and Yiming Pan and Ying Zhang and Yingli Wang and Yingru Li and Yinpei Su and Yipeng Geng and Yitong Zhu and Yongkun Yang and Yuhang Li and Yuhao Wu and Yujiang Li and Yunan Liu and Yunqing Wang and Yuntao Li and Yuxuan Zhang and Zezhen Liu and Zhen Yang and Zhengda Zhou and Zhongpei Qiao and Zhuoer Feng and Zhuorui Liu and Zichen Zhang and Zihan Wang and Zijun Yao and Zikang Wang and Ziqiang Liu and Ziwei Chai and Zixuan Li and Zuodong Zhao and Wenguang Chen and Jidong Zhai and Bin Xu and Minlie Huang and Hongning Wang and Juanzi Li and Yuxiao Dong and Jie Tang},
@@ -130,14 +148,14 @@
   url           = {https://arxiv.org/abs/2512.20856}
 }
 
-@misc{qwen3,
-  title         = {Qwen3 Technical Report},
-  author        = {An Yang and Anfeng Li and Baosong Yang and Beichen Zhang and Binyuan Hui and Bo Zheng and Bowen Yu and Chang Gao and Chengen Huang and Chenxu Lv and Chujie Zheng and Dayiheng Liu and Fan Zhou and Fei Huang and Junyang Lin and Jingren Zhou},
-  year          = {2025},
-  eprint        = {2505.09388},
+@misc{qwen,
+  title         = {Qwen Technical Report},
+  author        = {Jinze Bai and Shuai Bai and Yunfei Chu and Zeyu Cui and Kai Dang and Xiaodong Deng and Yang Fan and Wenbin Ge and Yu Han and Fei Huang and Binyuan Hui and Luo Ji and Mei Li and Junyang Lin and Runji Lin and Dayiheng Liu and Gao Liu and Chengqiang Lu and Keming Lu and Jianxin Ma and Rui Men and Xingzhang Ren and Xuancheng Ren and Chuanqi Tan and Sinan Tan and Jianhong Tu and Peng Wang and Shijie Wang and Wei Wang and Shengguang Wu and Benfeng Xu and Jin Xu and An Yang and Hao Yang and Jian Yang and Shusheng Yang and Yang Yao and Bowen Yu and Hongyi Yuan and Zheng Yuan and Jianwei Zhang and Xingxuan Zhang and Yichang Zhang and Zhenru Zhang and Chang Zhou and Jingren Zhou and Xiaohuan Zhou and Tianhang Zhu},
+  year          = {2023},
+  eprint        = {2309.16609},
   archiveprefix = {arXiv},
   primaryclass  = {cs.CL},
-  url           = {https://arxiv.org/abs/2505.09388}
+  url           = {https://arxiv.org/abs/2309.16609}
 }
 
 @misc{qwen2,
@@ -150,6 +168,26 @@
   url           = {https://arxiv.org/abs/2407.10671}
 }
 
+@misc{qwen25,
+  title         = {Qwen2.5 Technical Report},
+  author        = {An Yang and Baosong Yang and Beichen Zhang and Binyuan Hui and Bo Zheng and Bowen Yu and Chengyuan Li and Dayiheng Liu and Fei Huang and Haoran Wei and Huan Lin and Jian Yang and Jianhong Tu and Jianwei Zhang and Jianxin Yang and Junyang Lin and Jingren Zhou},
+  year          = {2024},
+  eprint        = {2412.15115},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/2412.15115}
+}
+
+@misc{qwen3,
+  title         = {Qwen3 Technical Report},
+  author        = {An Yang and Anfeng Li and Baosong Yang and Beichen Zhang and Binyuan Hui and Bo Zheng and Bowen Yu and Chang Gao and Chengen Huang and Chenxu Lv and Chujie Zheng and Dayiheng Liu and Fan Zhou and Fei Huang and Junyang Lin and Jingren Zhou},
+  year          = {2025},
+  eprint        = {2505.09388},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/2505.09388}
+}
+
 @misc{phi4,
   title         = {Phi-4 Technical Report},
   author        = {Marah Abdin and Sahaj Agarwal and Ahmed Awadallah and Vidhisha Balachandran and Harkirat Behl and Lingjiao Chen and Gustavo de Rosa and Suriya Gunasekar and Mojan Javaheripi and Neel Jain and Piero Kauffmann and Yin Tat Lee and Yuanzhi Li and Anh Nguyen and Olatunji Ruwase and Olli Saarikivi and Adil Salim and Shital Shah and Michael Santacroce and Harsha Nori and Xin Wang and Rachel Ward and Philipp Witte and Cyril Zhang and Yi Zhang},
@@ -246,6 +284,28 @@
   url           = {https://arxiv.org/abs/2508.10925}
 }
 
+% Architecture
+@inproceedings{gqa,
+  title         = {GQA: Training Generalized Multi-Query Transformer Models from Multi-Head Checkpoints},
+  author        = {Joshua Ainslie and James Lee-Thorp and Michiel de Jong and Yury Zemlyanskiy and Federico Lebrón and Sumit Sanghai},
+  booktitle     = {Conference on Empirical Methods in Natural Language Processing},
+  year          = {2023},
+  eprint        = {2305.13245},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/2305.13245}
+}
+
+@article{rope,
+  title         = {RoFormer: Enhanced Transformer with Rotary Position Embedding},
+  author        = {Jianlin Su and Murtadha Ahmed and Yu Lu and Shengfeng Pan and Wen Bo and Yunfeng Liu},
+  journal       = {Neurocomputing},
+  volume        = {568},
+  pages         = {127063},
+  year          = {2024},
+  doi           = {10.1016/j.neucom.2023.127063}
+}
+
 % Inference
 @inproceedings{vllm,
   title         = {Efficient Memory Management for Large Language Model Serving with PagedAttention},
@@ -279,6 +339,27 @@
   url           = {https://arxiv.org/abs/2307.08691}
 }
 
+@misc{flashinfer,
+  title         = {FlashInfer: Efficient and Customizable Attention Engine for LLM Inference Serving},
+  author        = {Zihao Ye and Lequn Chen and Ruihang Lai and Yilong Zhao and Size Zheng and Junru Shao and Bohan Hou and Hongyi Jin and Yifei Zuo and Liangsheng Yin and Tianqi Chen and Luis Ceze},
+  year          = {2025},
+  eprint        = {2501.01005},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.DC},
+  url           = {https://arxiv.org/abs/2501.01005}
+}
+
+@inproceedings{speculativedecoding,
+  title         = {Fast Inference from Transformers via Speculative Decoding},
+  author        = {Yaniv Leviathan and Matan Kalman and Yossi Matias},
+  booktitle     = {International Conference on Machine Learning},
+  year          = {2023},
+  eprint        = {2211.17192},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.LG},
+  url           = {https://arxiv.org/abs/2211.17192}
+}
+
 @misc{dflash,
   title         = {DFlash: Block Diffusion for Flash Speculative Decoding},
   author        = {Jian Chen and Yesheng Liang and Zhijian Liu},
@@ -289,6 +370,28 @@
   url           = {https://arxiv.org/abs/2602.06036}
 }
 
+% Training
+@inproceedings{adamw,
+  title         = {Decoupled Weight Decay Regularization},
+  author        = {Ilya Loshchilov and Frank Hutter},
+  booktitle     = {International Conference on Learning Representations},
+  year          = {2019},
+  eprint        = {1711.05101},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.LG},
+  url           = {https://arxiv.org/abs/1711.05101}
+}
+
+@misc{minicpm,
+  title         = {MiniCPM: Unveiling the Potential of Small Language Models with Scalable Training Strategies},
+  author        = {Shengding Hu and Yuge Tu and Xu Han and Chaoqun He and Ganqu Cui and Xiang Long and Zhi Zheng and Yewei Fang and Yuxiang Huang and Weilin Zhao and Xinrong Zhang and Zheng Leng Thai and Kaihuo Zhang and Chongyi Wang and Yuan Yao and Chenyang Zhao and Jie Zhou and Jie Cai and Zhongwu Zhai and Ning Ding and Chao Jia and Guoyang Zeng and Dahai Li and Zhiyuan Liu and Maosong Sun},
+  year          = {2024},
+  eprint        = {2404.06395},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/2404.06395}
+}
+
 % Tools
 @inproceedings{dspy,
   title         = {DSPy: Compiling Declarative Language Model Calls into State-of-the-Art Pipelines},
@@ -299,4 +402,132 @@
   archiveprefix = {arXiv},
   primaryclass  = {cs.CL},
   url           = {https://arxiv.org/abs/2310.03714}
-}
+}
+
+@software{datatrove,
+  title         = {DataTrove: Large Scale Data Processing},
+  author        = {Guilherme Penedo and Hynek Kydlíček and Thomas Wolf and Leandro von Werra},
+  year          = {2024},
+  url           = {https://github.com/huggingface/datatrove},
+  note          = {GitHub repository}
+}
+
+% Benchmarks
+@misc{arc,
+  title         = {Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge},
+  author        = {Peter Clark and Isaac Cowhey and Oren Etzioni and Tushar Khot and Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
+  year          = {2018},
+  eprint        = {1803.05457},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.AI},
+  url           = {https://arxiv.org/abs/1803.05457}
+}
+
+@misc{hellaswag,
+  title         = {HellaSwag: Can a Machine Really Finish Your Sentence?},
+  author        = {Rowan Zellers and Ari Holtzman and Yonatan Bisk and Ali Farhadi and Yejin Choi},
+  year          = {2019},
+  eprint        = {1905.07830},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/1905.07830}
+}
+
+@misc{mmluredux,
+  title         = {Are We Done with MMLU?},
+  author        = {Aryo Pradipta Gema and Joshua Ong Jun Leang and Giwon Hong and Alessio Devoto and Alberto Carlo Maria Mancino and Rohit Saxena and Xuanli He and Yu Zhao and Xiaotang Du and Mohammad Reza Ghasemi Madani and Claire Barale and Robert McHardy and Joshua Harris and Jean Kaddour and Emile van Krieken and Pasquale Minervini},
+  year          = {2024},
+  eprint        = {2406.04127},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/2406.04127}
+}
+
+@inproceedings{xcsqa,
+  title         = {Common Sense Beyond English: Evaluating and Improving Multilingual Language Models for Commonsense Reasoning},
+  author        = {Bill Yuchen Lin and Seyeon Lee and Xiaoyang Qiao and Xiang Ren},
+  booktitle     = {Annual Meeting of the Association for Computational Linguistics},
+  year          = {2021},
+  url           = {https://aclanthology.org/2021.acl-long.102/}
+}
+
+@misc{openbookqa,
+  title         = {Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering},
+  author        = {Todor Mihaylov and Peter Clark and Tushar Khot and Ashish Sabharwal},
+  year          = {2018},
+  eprint        = {1809.02789},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/1809.02789}
+}
+
+@misc{winogrande,
+  title         = {WinoGrande: An Adversarial Winograd Schema Challenge at Scale},
+  author        = {Keisuke Sakaguchi and Ronan Le Bras and Chandra Bhagavatula and Yejin Choi},
+  year          = {2019},
+  eprint        = {1907.10641},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.AI},
+  url           = {https://arxiv.org/abs/1907.10641}
+}
+
+@misc{piqa,
+  title         = {PIQA: Reasoning about Physical Intuition by Question Answering},
+  author        = {Yonatan Bisk and Rowan Zellers and Ronan Le Bras and Jianfeng Gao and Yejin Choi},
+  year          = {2019},
+  eprint        = {1911.11641},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/1911.11641}
+}
+
+@misc{squad2,
+  title         = {Know What You Don't Know: Unanswerable Questions for SQuAD},
+  author        = {Pranav Rajpurkar and Robin Jia and Percy Liang},
+  year          = {2018},
+  eprint        = {1806.03822},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/1806.03822}
+}
+
+@misc{drop,
+  title         = {DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs},
+  author        = {Dheeru Dua and Yizhong Wang and Pradeep Dasigi and Gabriel Stanovsky and Sameer Singh and Matt Gardner},
+  year          = {2019},
+  eprint        = {1903.00161},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/1903.00161}
+}
+
+@inproceedings{wikitablequestions,
+  title         = {Compositional Semantic Parsing on Semi-Structured Tables},
+  author        = {Panupong Pasupat and Percy Liang},
+  booktitle     = {Annual Meeting of the Association for Computational Linguistics},
+  year          = {2015},
+  eprint        = {1508.00305},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/1508.00305}
+}
+
+@misc{triviaqa,
+  title         = {TriviaQA: A Large Scale Distantly Supervised Challenge Dataset for Reading Comprehension},
+  author        = {Mandar Joshi and Eunsol Choi and Daniel Weld and Luke Zettlemoyer},
+  year          = {2017},
+  eprint        = {1705.03551},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.CL},
+  url           = {https://arxiv.org/abs/1705.03551}
+}
+
+@misc{gsm8k,
+  title         = {Training Verifiers to Solve Math Word Problems},
+  author        = {Karl Cobbe and Vineet Kosaraju and Mohammad Bavarian and Mark Chen and Heewoo Jun and Lukasz Kaiser and Matthias Plappert and Jerry Tworek and Jacob Hilton and Reiichiro Nakano and Christopher Hesse and John Schulman},
+  year          = {2021},
+  eprint        = {2110.14168},
+  archiveprefix = {arXiv},
+  primaryclass  = {cs.LG},
+  url           = {https://arxiv.org/abs/2110.14168}
+}

app/src/content/chapters/appendix.mdx

@@ -2,7 +2,7 @@
 
 ### Details on the experiments
 
-For our ablations we train a 1.2B parameter language model using a Qwen2-style [@qwen2] architecture with 28 layers, a hidden dimension of 2048, 16 attention heads with 8 key-value heads (grouped-query attention), and an intermediate size of 6144. The model utilized the Llama 3.2 [@llama3] tokenizer ( `hynky/Llama-3.2-1B-no-bos` ) with a vocabulary size of 128,256 tokens. Training was conducted on 64 NVIDIA H100 80GB GPUs across 8 nodes using pure data parallelism (DP=64) with a global batch size of 512 and a sequence length of 4,096 tokens, accumulating to approximately 21 billion tokens total over 10,000 steps. We employed the AdamW optimizer with a learning rate of 5×10⁻⁴, β₁=0.9, β₂=0.95, weight decay of 0.1, and gradient clipping at 1.0. All training utilized bfloat16 precision with Flash Attention 2 [@flashattention2], fused operations (RMS normalization and rotary embeddings), and document masking to prevent cross-document attention. We aim to rephrase at least 10B tokens per experiment but due to wildly varying number of completion tokens by prompt we sometimes get less than that. In these cases we train on some of the data twice.
+For our ablations we train a 1.2B parameter language model using a Qwen2-style [@qwen2] architecture with 28 layers, a hidden dimension of 2048, 16 attention heads with 8 key-value heads (grouped-query attention [@gqa]), and an intermediate size of 6144. The model utilized the Llama 3.2 [@llama3] tokenizer ( `hynky/Llama-3.2-1B-no-bos` ) with a vocabulary size of 128,256 tokens. Training was conducted on 64 NVIDIA H100 80GB GPUs across 8 nodes using pure data parallelism (DP=64) with a global batch size of 512 and a sequence length of 4,096 tokens, accumulating to approximately 21 billion tokens total over 10,000 steps. We employed the AdamW [@adamw] optimizer with a learning rate of 5×10⁻⁴, β₁=0.9, β₂=0.95, weight decay of 0.1, and gradient clipping at 1.0. All training utilized bfloat16 precision with Flash Attention 2 [@flashattention2], fused operations (RMS normalization and rotary embeddings [@rope]), and document masking to prevent cross-document attention. We aim to rephrase at least 10B tokens per experiment but due to wildly varying number of completion tokens by prompt we sometimes get less than that. In these cases we train on some of the data twice.
 
 ### Prompts
 
@@ -212,7 +212,7 @@ Original Draft: [TEXT]
 
 ### Decay vs Scratch
 
-We explored two distinct training paradigms. In the  **from-scratch**  setup ( `decay_exp=false` ), models were trained for the full 10,000 steps (~21B tokens) on a single dataset or mixture of datasets. In contrast, the  **decay**  experiments ( `decay_exp=true` ) aimed to obtain quicker signal with fewer rephrased tokens by leveraging a two-stage training approach. These decay experiments resumed training from a checkpoint at step 9,000 of a model previously trained on lower-quality data ( `fw_edu_lq` ), then continued training with a new dataset (or mixture) for the final 1,000 steps (~2B tokens) during the learning rate decay phase. We selected the low quality fineweb dataset for the first training phase so we can see effects of the ablated data mixtures more clearly. This design allowed us to evaluate the impact of high-quality rephrased or synthetic data more efficiently, requiring around 2B rephrased tokens rather than the full 21B needed for from-scratch training, thus reducing computational costs by 90% per experimental condition while still providing meaningful signal about data quality effects. To enable the decay experiments, we used a warmup-stable-decay (WSD) learning rate schedule with 1% warmup (100 steps), 89% stable training, and 10% linear decay (1,000 steps) to a minimum of 5×10⁻⁵.
+We explored two distinct training paradigms. In the  **from-scratch**  setup ( `decay_exp=false` ), models were trained for the full 10,000 steps (~21B tokens) on a single dataset or mixture of datasets. In contrast, the  **decay**  experiments ( `decay_exp=true` ) aimed to obtain quicker signal with fewer rephrased tokens by leveraging a two-stage training approach. These decay experiments resumed training from a checkpoint at step 9,000 of a model previously trained on lower-quality data ( `fw_edu_lq` ), then continued training with a new dataset (or mixture) for the final 1,000 steps (~2B tokens) during the learning rate decay phase. We selected the low quality fineweb dataset for the first training phase so we can see effects of the ablated data mixtures more clearly. This design allowed us to evaluate the impact of high-quality rephrased or synthetic data more efficiently, requiring around 2B rephrased tokens rather than the full 21B needed for from-scratch training, thus reducing computational costs by 90% per experimental condition while still providing meaningful signal about data quality effects. To enable the decay experiments, we used a warmup-stable-decay (WSD) [@minicpm] learning rate schedule with 1% warmup (100 steps), 89% stable training, and 10% linear decay (1,000 steps) to a minimum of 5×10⁻⁵.
 
 #### Variance across seeds and data seeds
 

app/src/content/chapters/experiments.mdx

@@ -314,7 +314,7 @@ We hypothesize that the consistently strong performance of SmolLM2 originates fr
 So the model family clearly seems to matter. However, SmolLM2 is already a year old. Are newer models better than older ones?
 #### Does the model generation matter?
 
-We compare rephrasing with Qwen models from versions 1.5, 2, 2.5 and 3 using the tutorial prompt, one of the prompts that outperformed the DCLM baseline. While the differences are small we find a trend that newer versions lead to higher evaluation performance.
+We compare rephrasing with Qwen models from versions 1.5 [@qwen], 2, 2.5 [@qwen25] and 3 using the tutorial prompt, one of the prompts that outperformed the DCLM baseline. While the differences are small we find a trend that newer versions lead to higher evaluation performance.
 
 <HtmlEmbed
   id="model-generation"

app/src/content/chapters/infrastructure.mdx

@@ -10,11 +10,11 @@ Synthetic data has emerged as a key ingredient in training modern LLMs, providin
 
 <Image src={SyDLepVveg_2f81384e_bcac_806f_acb7_fd65c71dd9df} alt="Image" />
 
-Synthetic data also plays a central role in post-training via  *distillation* , where a capable model is used to generate high-quality responses for targeted domains such as reasoning, instruction-following, and tool-use. This data can then be used for supervised fine-tuning or preference optimization, allowing developers to shape a model's behaviour with labels that would be expensive or impractical to obtain from humans. For example, [SmolLM3](https://huggingface.co/spaces/HuggingFaceTB/smol-training-playbook) was post-trained almost entirely on a few billion tokens of data generated from models like DeepSeek-R1 [@deepseekr1] and Qwen3.
+Synthetic data also plays a central role in post-training via  *distillation* , where a capable model is used to generate high-quality responses for targeted domains such as reasoning, instruction-following, and tool-use. This data can then be used for supervised fine-tuning or preference optimization, allowing developers to shape a model's behaviour with labels that would be expensive or impractical to obtain from humans. For example, [SmolLM3](https://huggingface.co/spaces/HuggingFaceTB/smol-training-playbook) [@smollm3] was post-trained almost entirely on a few billion tokens of data generated from models like DeepSeek-R1 [@deepseekr1] and Qwen3.
 
 So what does it actually take to generate a trillion tokens of synthetic data? Thanks to fast inference engines like [vLLM](https://github.com/vllm-project/vllm) [@vllm] and [SGLang](https://github.com/sgl-project/sglang) [@sglang], it turns out that the bottleneck isn't the generation itself but the  *infrastructure*  around it: orchestrating thousands of prompts, keeping GPUs saturated, checkpointing outputs, and pushing everything to storage without losing progress when a worker crashes.
 
-Today we're excited to announce major extensions to [DataTrove](https://github.com/huggingface/datatrove) to manage this entire process. These extensions package the scaffolding we built for our own synthetic data pipelines and make it accessible to anyone who wants to generate high-quality datasets at scale. DataTrove supports both local generation and large-scale distributed runs on Slurm clusters, handling chunking, checkpointing, distributed queueing, and Hugging Face dataset management so you can focus on synthetic data design rather than operational glue.
+Today we're excited to announce major extensions to [DataTrove](https://github.com/huggingface/datatrove) [@datatrove] to manage this entire process. These extensions package the scaffolding we built for our own synthetic data pipelines and make it accessible to anyone who wants to generate high-quality datasets at scale. DataTrove supports both local generation and large-scale distributed runs on Slurm clusters, handling chunking, checkpointing, distributed queueing, and Hugging Face dataset management so you can focus on synthetic data design rather than operational glue.
 
 In this blog post we show how DataTrove can be used to generate a billion tokens per hour across several model scales, ranging from 1 billion to 1 trillion parameters. Let's dive in!
 
@@ -33,7 +33,7 @@ DataTrove provides two modes to generate synthetic data:
 -  **Local execution** : Run on a single machine with multiple workers for development and small-scale generation
 -  **Slurm cluster** : Distribute processing across multiple nodes for large-scale production workloads
 
-Here's a simple example of local execution on a node of 8 GPUs to generate solutions to math problems from the [s1K dataset](https://huggingface.co/datasets/simplescaling/s1K-1.1) using  `Qwen3-4B-Thinking-2507` :
+Here's a simple example of local execution on a node of 8 GPUs to generate solutions to math problems from the [s1K dataset](https://huggingface.co/datasets/simplescaling/s1K-1.1) [@s1k] using  `Qwen3-4B-Thinking-2507` :
 
 ```shell
 python examples/inference/benchmark/generate_data.py \
@@ -214,7 +214,7 @@ datacard_pipeline = [InferenceDatasetCardGenerator(params=params)]
 
 For synthetic data generation, we may run language model inference for millions of GPU hours. Finding a configuration that maximizes throughput is critical, as it could accelerate generation by days and save thousands of dollars. In this section, we describe our experiments to identify optimal parameters for a selection of popular models. We run the experiments once for a pre-training dataset and once for a post-training example. 
 
-The Flash-Attn VLLM backend is more than 50% faster than FlashInfer across setups.
+The Flash-Attn VLLM backend is more than 50% faster than FlashInfer [@flashinfer] across setups.
 
 #### Pre-training
 
@@ -257,6 +257,6 @@ We also explored what would be required to generate 1T tokens in a day. We belie
 
 You can find the benchmarking code [here](https://github.com/huggingface/datatrove/tree/main/examples/inference/benchmark) together with the [yaml config](https://github.com/huggingface/datatrove/blob/main/examples/inference/benchmark/sample_benchmark_config.yaml). 
 
-We experimented with speculative decoding using the [ngram method](https://docs.vllm.ai/en/stable/features/spec_decode.html?h=specula#speculating-by-matching-n-grams-in-the-prompt) but found no consistent speedups. We hypothesize this approach is unhelpful because the input in our benchmarking dataset is relatively short compared to the thinking tokens and output. We expect greater gains for tasks involving more copying from the input.
+We experimented with speculative decoding [@speculativedecoding] using the [ngram method](https://docs.vllm.ai/en/stable/features/spec_decode.html?h=specula#speculating-by-matching-n-grams-in-the-prompt) but found no consistent speedups. We hypothesize this approach is unhelpful because the input in our benchmarking dataset is relatively short compared to the thinking tokens and output. We expect greater gains for tasks involving more copying from the input.
 
 TODO: Optimize this section for pretraining: use that prompt and seq length configuration but mention in the end that for post training we can easily rerun this experiment with different prompts and datasets

app/src/content/chapters/setup.mdx

@@ -53,7 +53,7 @@ We compare against several baseline datasets for pretraining and data rephrasing
 
 ### Ablation Setup
 
-For our ablations we train a 1.2B parameter language model using a Qwen2-style architecture (see more details in the Appendix). We evaluate our model on a diverse set of 12 benchmark tasks spanning multiple reasoning and knowledge domains. For reasoning capabilities, we assess performance on ARC, HellaSwag, MMLU Redux, Cross-lingual CommonsenseQA (XCSQA), OpenBookQA, Winogrande, and PIQA. Question answering capabilities are evaluated using SQuAD v2, DROP, WikiTableQuestions, and TriviaQA. Mathematical reasoning is assessed via GSM8K. Given that our model is relatively small and trained on only 20 billion tokens, we employ the continuation format (CF) for most tasks rather than the standard multiple-choice format. The CF setup, which frames evaluation as a next-token prediction task, has been shown to provide more reliable assessments for smaller or less extensively trained models that may struggle with complex instruction following or multiple-choice formatting conventions. All evaluations are conducted using 3-shot prompting with a single seed to ensure reproducibility.
+For our ablations we train a 1.2B parameter language model using a Qwen2-style architecture (see more details in the Appendix). We evaluate our model on a diverse set of 12 benchmark tasks spanning multiple reasoning and knowledge domains. For reasoning capabilities, we assess performance on ARC [@arc], HellaSwag [@hellaswag], MMLU Redux [@mmluredux], Cross-lingual CommonsenseQA (XCSQA) [@xcsqa], OpenBookQA [@openbookqa], Winogrande [@winogrande], and PIQA [@piqa]. Question answering capabilities are evaluated using SQuAD v2 [@squad2], DROP [@drop], WikiTableQuestions [@wikitablequestions], and TriviaQA [@triviaqa]. Mathematical reasoning is assessed via GSM8K [@gsm8k]. Given that our model is relatively small and trained on only 20 billion tokens, we employ the continuation format (CF) for most tasks rather than the standard multiple-choice format. The CF setup, which frames evaluation as a next-token prediction task, has been shown to provide more reliable assessments for smaller or less extensively trained models that may struggle with complex instruction following or multiple-choice formatting conventions. All evaluations are conducted using 3-shot prompting with a single seed to ensure reproducibility.
 
 #### Naming