| { | |
| "attn_bias": 0, | |
| "block": 4096, | |
| "ckpt_sha256": "7d092d4d0bf838da1115a5369bf0a6729fe4c170e0d5ad303873f7320d6c1866", | |
| "data_seed": 20261023, | |
| "ffn": "swiglu", | |
| "ffn_mult": 2.667, | |
| "logit_cap": 0.0, | |
| "n_embd": 2048, | |
| "n_head": 16, | |
| "n_layer": 20, | |
| "n_params": 1073912851, | |
| "norm": "rmsnorm", | |
| "norm_eps": 1e-06, | |
| "pos": "rope", | |
| "qk_norm": true, | |
| "rope_theta": 500000.0, | |
| "step": 1430, | |
| "tied_weights": { | |
| "tok.weight": "head.weight" | |
| }, | |
| "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", | |
| "trainer": "train_gpt_ref_r23.py", | |
| "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", | |
| "value_residual": true, | |
| "vocab": 32768, | |
| "z_loss": 0.0 | |
| } |