{ "model": { "block_size": 512, "vocab_size": 50257, "n_layer": 12, "n_head": 12, "n_embd": 768, "dropout": 0.0, "bias": false, "pos_encoding": "rope", "rope_theta": 10000.0, "norm": "layernorm", "mlp": "swiglu", "qk_norm": true, "n_kv_head": 0, "block_repeat": 1, "n_experts": 1, "moe_top_k": 1, "moe_aux_weight": 0.01, "gelu_approx": "none", "causal": true, "mtp_heads": 1 }, "name": "mini-llm-base", "chat": false, "stage": 2, "params": 123552768, "datasets": [ "BaoMinhTran/mini-fineweb-edu", "HuggingFaceTB/cosmopedia" ], "base_model": null, "training": { "stage 1": { "data": "Mini FineWeb-Edu (small) + five Cosmopedia subsets (stories, wikihow, openstax, stanford, khanacademy), mixed by size", "steps": 30500, "tokens": 499712000, "batch_size": 32, "context": 512, "learning_rate": 0.0012, "warmup_steps": 1000, "hardware": "Colab L4, bfloat16", "time": "4 h 20 min" }, "stage 2": { "data": "Mini FineWeb-Edu (full) + the same Cosmopedia subsets; stage 1's validation documents removed from the training data", "continued_from_step": 30500, "steps": 152600, "tokens": 2500198400, "hardware": "Colab L4 to step 79,000, then A100; four sessions", "best_val_loss": 2.9633, "best_val_note": "on the stage 2 data mix" } }, "tokenizer": "gpt2 (tiktoken)", "source_id": "mini-fineweb-edu-ful-202611151103-bb160c", "code": "https://github.com/a-ramakrishnan/mini_llm" }