| { | |
| "architecture": "SparseLM", | |
| "vocab_size": 49152, | |
| "d_model": 1536, | |
| "n_layers": 16, | |
| "n_heads": 24, | |
| "n_kv_heads": 24, | |
| "ffn_hidden": 4096, | |
| "max_seq_len": 4096, | |
| "rope_theta": 10000.0, | |
| "gated_ffn": true, | |
| "l1_coeff": 0.0, | |
| "tie_embeddings": true, | |
| "qk_norm": true, | |
| "z_loss_coeff": 0.0001, | |
| "attn_mode": "dsa", | |
| "dsa_sparsify": false, | |
| "dsa_top_k": 2048, | |
| "dsa_local": 128, | |
| "dsa_indexer_heads": 4, | |
| "dsa_indexer_dim": 64, | |
| "dsa_index_loss_weight": 1.0, | |
| "dsa_block_size": 128, | |
| "dsa_index_rows": 256 | |
| } | |