| diloco: |
| type: article |
| title: "DiLoCo: Distributed Low-Communication Training of Language Models" |
| author: ["Douillard, Arthur"] |
| date: 2023 |
| url: https://arxiv.org/abs/2311.08105 |
| mdlm: |
| type: article |
| title: Simple and Effective Masked Diffusion Language Models |
| author: ["Sahoo, Subham Sekhar"] |
| date: 2024 |
| url: https://arxiv.org/abs/2406.07524 |
| smdm: |
| type: article |
| title: Scaling up Masked Diffusion Models on Text |
| author: ["Nie, Shen"] |
| date: 2025 |
| url: https://arxiv.org/abs/2410.18514 |
| llada: |
| type: article |
| title: Large Language Diffusion Models |
| author: ["Nie, Shen"] |
| date: 2025 |
| url: https://arxiv.org/abs/2502.09992 |
| ultrafineweb: |
| type: article |
| title: "Ultra-FineWeb: Efficient Data Filtering and Verification for High-Quality LLM Training Data" |
| author: ["Wang, Yudong"] |
| date: 2025 |
| url: https://arxiv.org/abs/2505.05427 |
| olmo3: |
| type: web |
| title: "OLMo 3 stage-2 mid-training mix (dolma3_dolmino_mix-100B-1125)" |
| author: ["Team OLMo"] |
| date: 2025 |
| url: https://huggingface.co/datasets/allenai/dolma3_dolmino_mix-100B-1125 |
| d2z: |
| type: article |
| title: "Straight to Zero: Why Linearly Decaying the Learning Rate to Zero Works Best for LLMs" |
| author: ["Bergsma, Shane"] |
| date: 2025 |
| url: https://arxiv.org/abs/2502.15938 |
| muon: |
| type: web |
| title: "Muon: An optimizer for hidden layers in neural networks" |
| author: ["Jordan, Keller"] |
| date: 2024 |
| url: https://kellerjordan.github.io/posts/muon/ |
| moonlight: |
| type: article |
| title: Muon is Scalable for LLM Training |
| author: ["Liu, Jingyuan"] |
| date: 2025 |
| url: https://arxiv.org/abs/2502.16982 |
| cautious: |
| type: article |
| title: "Cautious Optimizers: Improving Training with One Line of Code" |
| author: ["Liang, Kaizhao"] |
| date: 2024 |
| url: https://arxiv.org/abs/2411.16085 |
| lmeval: |
| type: web |
| title: A framework for few-shot language model evaluation (lm-evaluation-harness) |
| author: ["Gao, Leo"] |
| date: 2024 |
| url: https://github.com/EleutherAI/lm-evaluation-harness |
| pythia: |
| type: article |
| title: "Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling" |
| author: ["Biderman, Stella"] |
| date: 2023 |
| url: https://arxiv.org/abs/2304.01373 |
| smollm2: |
| type: article |
| title: "SmolLM2: When Smol Goes Big - Data-Centric Training of a Small Language Model" |
| author: ["Ben Allal, Loubna"] |
| date: 2025 |
| url: https://arxiv.org/abs/2502.02737 |
| qwen25: |
| type: article |
| title: Qwen2.5 Technical Report |
| author: ["Qwen Team"] |
| date: 2024 |
| url: https://arxiv.org/abs/2412.15115 |
|
|
