| { |
| "vocab_size": 206464, |
| "d_model": 1024, |
| "n_layers": 18, |
| "max_seq_len": 4096, |
| "mha_layer_indices": [ |
| 5, |
| 11, |
| 17 |
| ], |
| "dense_ffn_layer_indices": [ |
| 1, |
| 3, |
| 5, |
| 7, |
| 9, |
| 11, |
| 13, |
| 15, |
| 17 |
| ], |
| "hse_ffn_layer_indices": [ |
| 0, |
| 2, |
| 4, |
| 6, |
| 8, |
| 10, |
| 12, |
| 14, |
| 16 |
| ], |
| "arg_d_inner": 1024, |
| "arg_d_state": 48, |
| "arg_d_rank": 48, |
| "arg_conv_kernel": 4, |
| "arg_local_window": 64, |
| "arg_local_n_heads": 8, |
| "arg_local_n_kv_heads": 2, |
| "arg_head_dim": 64, |
| "arg_scalar_gate": true, |
| "mha_n_heads": 16, |
| "mha_n_kv_heads": 4, |
| "mha_head_dim": 64, |
| "rope_theta": 500000.0, |
| "rope_scaling_type": null, |
| "rope_scaling_factor": 1.0, |
| "dense_d_ff": 2560, |
| "hse_n_domains": 4, |
| "hse_n_experts_per_domain": 6, |
| "hse_top_k": 2, |
| "hse_d_ff_expert": 210, |
| "hse_capacity_factor": 1.25, |
| "hse_expert_dropout": 0.1, |
| "hse_aux_loss_alpha": 0.01, |
| "hse_gumbel_tau_init": 1.0, |
| "hse_gumbel_tau_final": 0.1, |
| "hse_gumbel_tau_warmup_tokens": 10000000000, |
| "norm_type": "dgn", |
| "dgn_n_groups": 16, |
| "norm_eps": 1e-06, |
| "init_std": 0.02, |
| "init_std_residual_scale": true, |
| "dropout": 0.0, |
| "tie_word_embeddings": true, |
| "use_cache": true, |
| "pad_token_id": 0, |
| "bos_token_id": 1, |
| "eos_token_id": 2, |
| "use_flash_attention": true, |
| "use_triton_kernels": true, |
| "use_torch_compile": true, |
| "compile_mode": "max-autotune", |
| "use_gradient_checkpointing": true, |
| "gradient_checkpointing_ratio": 1.0 |
| } |
