| { | |
| "architecture": "Foundry4", | |
| "config": { | |
| "vocab_size": 65536, | |
| "n_specials": 16, | |
| "d_model": 1280, | |
| "n_layers": 24, | |
| "n_heads": 10, | |
| "head_dim": 128, | |
| "seq_len": 4096, | |
| "dropout": 0.0, | |
| "rope_theta": 10000.0, | |
| "qk_norm": true, | |
| "attn_every": 4, | |
| "gdn_heads": 10, | |
| "gdn_head_k": 128, | |
| "gdn_head_v": 128, | |
| "gdn_chunk": 32, | |
| "n_experts": 384, | |
| "expert_hidden": 128, | |
| "moe_topk": 8, | |
| "shared_hidden": 640, | |
| "lb_weight": 0.01, | |
| "kg_buckets": 262144, | |
| "kg_dim": 80, | |
| "kg_max_n": 4, | |
| "kg_probes": 2, | |
| "ponder_iters": 4, | |
| "ponder_hidden": 2560, | |
| "ponder_iter_weight_decay": 1.0, | |
| "judge_hidden": 512, | |
| "use_mtp": true, | |
| "mtp_weight": 0.05, | |
| "moe_ckpt": false | |
| }, | |
| "trained_steps": 2000, | |
| "dtype": "bfloat16" | |
| } |