config.json

{
  "att_dropout": 0.0,
  "att_experts": null,
  "att_groups": 8,
  "att_heads": 16,
  "att_query_experts": null,
  "att_query_groups": 8,
  "att_type": "sqa",
  "debug_interval": 10,
  "debug_mode": false,
  "embed_dim": 512,
  "interlayer_att_dropout": 0.0,
  "interlayer_att_experts": null,
  "interlayer_att_groups": 8,
  "interlayer_att_query_experts": null,
  "interlayer_att_query_groups": 8,
  "interlayer_att_type": "sqa",
  "norm_decay": 0.9,
  "norm_init_gate": -2.0,
  "norm_per_dim_scale": false,
  "norm_type": "classic-rms",
  "num_groups": 3,
  "num_layers": 21,
  "residual_gate_init": 3.0,
  "residual_gate_slot_status_type": "mean",
  "residual_gate_type": "elementwise",
  "residual_per_slot_gate": true,
  "rope_base": 100000,
  "seq_len": 8192,
  "stm_size": 4096,
  "use_flash_attention": true,
  "use_gated_residual": true,
  "use_tanh_residual_gate": false
}