File size: 1,197 Bytes
d8c733f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | {
"model_type": "guido_small",
"model_name": "GUIDO-small 200M (bugfixed)",
"architecture": "Vathos PiCOFormer (custom)",
"version": "v1-bugfix",
"n_layers": 24,
"d_model": 768,
"n_heads": 12,
"head_dim": 64,
"d_ff": 3072,
"vocab_size": 32768,
"max_seq_len": 8192,
"rope_base": 1000000.0,
"logit_softcap": 30.0,
"tied_embeddings": true,
"qk_norm": true,
"use_gated_attn": true,
"use_smear_gate": true,
"gate_input_dim": 12,
"mlp_kind": "udlp",
"orthogonalizer": "ns5",
"muon_backend_steps": 5,
"training_dtype": "fp32-master/bf16-autocast",
"checkpoint_dtype": "float32",
"training_steps": 16000,
"training_tokens": 4194304000,
"training_data": "corpus_v2 mixture 70/22/8 (math/fineweb/cosmopedia)",
"training_mixture": {
"openmath_full": 0.34,
"tinygsm": 0.14,
"openmathreasoning": 0.10,
"numina_15": 0.07,
"numina_cot": 0.05,
"fineweb_edu": 0.22,
"cosmopedia": 0.08
},
"tokenizer": "mistralai/Mathstral-7B-v0.1",
"params_total": 195092992,
"params_total_human": "195M",
"eval_gsm8k_greedy_pct": 43.0,
"eval_gsm8k_maj16_pct": 57.5,
"eval_math500_greedy_pct": 21.0,
"eval_math500_maj16_pct": 30.0
}
|