mini-llm-base / model_config.json
aramkris's picture
mini-llm-base: weights, config and model card
0c72526 verified
Raw History Blame Contribute Delete
1.6 kB
{
"model": {
"block_size": 512,
"vocab_size": 50257,
"n_layer": 12,
"n_head": 12,
"n_embd": 768,
"dropout": 0.0,
"bias": false,
"pos_encoding": "rope",
"rope_theta": 10000.0,
"norm": "layernorm",
"mlp": "swiglu",
"qk_norm": true,
"n_kv_head": 0,
"block_repeat": 1,
"n_experts": 1,
"moe_top_k": 1,
"moe_aux_weight": 0.01,
"gelu_approx": "none",
"causal": true,
"mtp_heads": 1
},
"name": "mini-llm-base",
"chat": false,
"stage": 2,
"params": 123552768,
"datasets": [
"BaoMinhTran/mini-fineweb-edu",
"HuggingFaceTB/cosmopedia"
],
"base_model": null,
"training": {
"stage 1": {
"data": "Mini FineWeb-Edu (small) + five Cosmopedia subsets (stories, wikihow, openstax, stanford, khanacademy), mixed by size",
"steps": 30500,
"tokens": 499712000,
"batch_size": 32,
"context": 512,
"learning_rate": 0.0012,
"warmup_steps": 1000,
"hardware": "Colab L4, bfloat16",
"time": "4 h 20 min"
},
"stage 2": {
"data": "Mini FineWeb-Edu (full) + the same Cosmopedia subsets; stage 1's validation documents removed from the training data",
"continued_from_step": 30500,
"steps": 152600,
"tokens": 2500198400,
"hardware": "Colab L4 to step 79,000, then A100; four sessions",
"best_val_loss": 2.9633,
"best_val_note": "on the stage 2 data mix"
}
},
"tokenizer": "gpt2 (tiktoken)",
"source_id": "mini-fineweb-edu-ful-202611151103-bb160c",
"code": "https://github.com/a-ramakrishnan/mini_llm"
}