| { | |
| "vocab_size": 32832, | |
| "n_layer": 24, | |
| "d_model": 1024, | |
| "arch": "hybrid", | |
| "attn_every": 4, | |
| "n_head": 16, | |
| "n_kv_head": 4, | |
| "head_dim": 128, | |
| "gdn_heads": 16, | |
| "gdn_head_dim": 128, | |
| "ffn_dim": 3584, | |
| "attnres_blocks": 8, | |
| "rope_theta": 10000.0, | |
| "max_seq_len": 4096, | |
| "softcap": 15.0, | |
| "grad_ckpt": false | |
| } |