| { | |
| "model_type": "guido_small", | |
| "model_name": "GUIDO-small 200M (bugfixed)", | |
| "architecture": "Vathos PiCOFormer (custom)", | |
| "version": "v1-bugfix", | |
| "n_layers": 24, | |
| "d_model": 768, | |
| "n_heads": 12, | |
| "head_dim": 64, | |
| "d_ff": 3072, | |
| "vocab_size": 32768, | |
| "max_seq_len": 8192, | |
| "rope_base": 1000000.0, | |
| "logit_softcap": 30.0, | |
| "tied_embeddings": true, | |
| "qk_norm": true, | |
| "use_gated_attn": true, | |
| "use_smear_gate": true, | |
| "gate_input_dim": 12, | |
| "mlp_kind": "udlp", | |
| "orthogonalizer": "ns5", | |
| "muon_backend_steps": 5, | |
| "training_dtype": "fp32-master/bf16-autocast", | |
| "checkpoint_dtype": "float32", | |
| "training_steps": 16000, | |
| "training_tokens": 4194304000, | |
| "training_data": "corpus_v2 mixture 70/22/8 (math/fineweb/cosmopedia)", | |
| "training_mixture": { | |
| "openmath_full": 0.34, | |
| "tinygsm": 0.14, | |
| "openmathreasoning": 0.10, | |
| "numina_15": 0.07, | |
| "numina_cot": 0.05, | |
| "fineweb_edu": 0.22, | |
| "cosmopedia": 0.08 | |
| }, | |
| "tokenizer": "mistralai/Mathstral-7B-v0.1", | |
| "params_total": 195092992, | |
| "params_total_human": "195M", | |
| "eval_gsm8k_greedy_pct": 43.0, | |
| "eval_gsm8k_maj16_pct": 57.5, | |
| "eval_math500_greedy_pct": 21.0, | |
| "eval_math500_maj16_pct": 30.0 | |
| } | |