| { | |
| "version": "v11", | |
| "core": "tiny-model-v11", | |
| "vocab_size": 71260, | |
| "dim": 512, | |
| "n_layers": 20, | |
| "ffn_dim": 2048, | |
| "n_heads": 8, | |
| "n_kv_heads": 4, | |
| "max_seq": 256, | |
| "rope_theta": 10000.0, | |
| "tie_embeddings": true, | |
| "tokenizer": "v11", | |
| "training": { | |
| "corpus": "TinyStories", | |
| "phase2_tokens": 16000000, | |
| "phase3_tokens": 8000000, | |
| "phase2_lr": 0.0003, | |
| "phase3_lr": 0.00015, | |
| "phase3_strategy": "frozen_ffn_retrain_attention" | |
| }, | |
| "_note": "Not a transformers architecture. Build TinyModel from the tiny_model_v11/ package shipped in this repo -- see the model card.", | |
| "pretrain_run": { | |
| "step": 15625, | |
| "tokens": 16000000, | |
| "param_count": 115149312, | |
| "seed": 42, | |
| "batch_size": 4, | |
| "lr": 0.0003, | |
| "phase": "phase 1 only -- no frozen-FFN phase, no maths mid-train", | |
| "corpus": "roneneldan/TinyStories @ f54c09fd23315a6f9c86f9dc80f725de7d8f9c64", | |
| "run_config": "configs/colab.json" | |
| } | |
| } | |