Download model_config.json from aramkris/mini-llm-base: direct link, hf CLI and curl.
- Browser
- Download file 1.6 kB
-
https://huggingface.co/aramkris/mini-llm-base/resolve/main/model_config.json
- Command line
-
hf download hf://aramkris/mini-llm-base/model_config.json
-
curl -L -o model_config.json https://huggingface.co/aramkris/mini-llm-base/resolve/main/model_config.json
1.6 kB
| { | |
| "model": { | |
| "block_size": 512, | |
| "vocab_size": 50257, | |
| "n_layer": 12, | |
| "n_head": 12, | |
| "n_embd": 768, | |
| "dropout": 0.0, | |
| "bias": false, | |
| "pos_encoding": "rope", | |
| "rope_theta": 10000.0, | |
| "norm": "layernorm", | |
| "mlp": "swiglu", | |
| "qk_norm": true, | |
| "n_kv_head": 0, | |
| "block_repeat": 1, | |
| "n_experts": 1, | |
| "moe_top_k": 1, | |
| "moe_aux_weight": 0.01, | |
| "gelu_approx": "none", | |
| "causal": true, | |
| "mtp_heads": 1 | |
| }, | |
| "name": "mini-llm-base", | |
| "chat": false, | |
| "stage": 2, | |
| "params": 123552768, | |
| "datasets": [ | |
| "BaoMinhTran/mini-fineweb-edu", | |
| "HuggingFaceTB/cosmopedia" | |
| ], | |
| "base_model": null, | |
| "training": { | |
| "stage 1": { | |
| "data": "Mini FineWeb-Edu (small) + five Cosmopedia subsets (stories, wikihow, openstax, stanford, khanacademy), mixed by size", | |
| "steps": 30500, | |
| "tokens": 499712000, | |
| "batch_size": 32, | |
| "context": 512, | |
| "learning_rate": 0.0012, | |
| "warmup_steps": 1000, | |
| "hardware": "Colab L4, bfloat16", | |
| "time": "4 h 20 min" | |
| }, | |
| "stage 2": { | |
| "data": "Mini FineWeb-Edu (full) + the same Cosmopedia subsets; stage 1's validation documents removed from the training data", | |
| "continued_from_step": 30500, | |
| "steps": 152600, | |
| "tokens": 2500198400, | |
| "hardware": "Colab L4 to step 79,000, then A100; four sessions", | |
| "best_val_loss": 2.9633, | |
| "best_val_note": "on the stage 2 data mix" | |
| } | |
| }, | |
| "tokenizer": "gpt2 (tiktoken)", | |
| "source_id": "mini-fineweb-edu-ful-202611151103-bb160c", | |
| "code": "https://github.com/a-ramakrishnan/mini_llm" | |
| } |