| { | |
| "vocab_size": 16384, | |
| "max_seq_len": 2048, | |
| "dim": 512, | |
| "n_layers": 8, | |
| "n_dense_layers": 2, | |
| "dense_hidden_dim": 1408, | |
| "norm_eps": 1e-06, | |
| "init_std": 0.02, | |
| "tie_embeddings": true, | |
| "n_heads": 8, | |
| "q_lora_rank": 256, | |
| "kv_lora_rank": 128, | |
| "qk_nope_head_dim": 64, | |
| "qk_rope_head_dim": 32, | |
| "v_head_dim": 64, | |
| "rope_theta": 10000.0, | |
| "rope_original_seq_len": 0, | |
| "rope_factor": 1.0, | |
| "rope_beta_fast": 32.0, | |
| "rope_beta_slow": 1.0, | |
| "rope_mscale": 1.0, | |
| "n_routed_experts": 240, | |
| "n_shared_experts": 2, | |
| "n_activated_experts": 8, | |
| "moe_hidden_dim": 128, | |
| "n_expert_groups": 8, | |
| "n_limited_groups": 4, | |
| "route_scale": 1.0, | |
| "bias_update_speed": 0.001, | |
| "seq_aux_loss_alpha": 0.0001, | |
| "moe_backend": "capacity", | |
| "capacity_factor": 2.0 | |
| } |