{
  "vocab_size": 206464,
  "d_model": 1024,
  "n_layers": 18,
  "max_seq_len": 4096,
  "mha_layer_indices": [
    5,
    11,
    17
  ],
  "dense_ffn_layer_indices": [
    1,
    3,
    5,
    7,
    9,
    11,
    13,
    15,
    17
  ],
  "hse_ffn_layer_indices": [
    0,
    2,
    4,
    6,
    8,
    10,
    12,
    14,
    16
  ],
  "arg_d_inner": 1024,
  "arg_d_state": 48,
  "arg_d_rank": 48,
  "arg_conv_kernel": 4,
  "arg_local_window": 64,
  "arg_local_n_heads": 8,
  "arg_local_n_kv_heads": 2,
  "arg_head_dim": 64,
  "arg_scalar_gate": true,
  "mha_n_heads": 16,
  "mha_n_kv_heads": 4,
  "mha_head_dim": 64,
  "rope_theta": 500000.0,
  "rope_scaling_type": null,
  "rope_scaling_factor": 1.0,
  "dense_d_ff": 2560,
  "hse_n_domains": 4,
  "hse_n_experts_per_domain": 6,
  "hse_top_k": 2,
  "hse_d_ff_expert": 210,
  "hse_capacity_factor": 1.25,
  "hse_expert_dropout": 0.1,
  "hse_aux_loss_alpha": 0.01,
  "hse_gumbel_tau_init": 1.0,
  "hse_gumbel_tau_final": 0.1,
  "hse_gumbel_tau_warmup_tokens": 10000000000,
  "norm_type": "dgn",
  "dgn_n_groups": 16,
  "norm_eps": 1e-06,
  "init_std": 0.02,
  "init_std_residual_scale": true,
  "dropout": 0.0,
  "tie_word_embeddings": true,
  "use_cache": true,
  "pad_token_id": 0,
  "bos_token_id": 1,
  "eos_token_id": 2,
  "use_flash_attention": true,
  "use_triton_kernels": true,
  "use_torch_compile": true,
  "compile_mode": "max-autotune",
  "use_gradient_checkpointing": true,
  "gradient_checkpointing_ratio": 1.0
}