{
  "attention_bias": false,
  "attention_dropout": 0.0,
  "bos_token_id": 200000,
  "draft_position_mode": "rope",
  "eos_token_id": [
    200001,
    200008
  ],
  "ffn_dim_multiplier": 2.0,
  "head_dim": 64,
  "hidden_act": "silu",
  "hidden_size": 8192,
  "initializer_range": 0.02,
  "intermediate_size": 16384,
  "intermediate_size_mlp": 16384,
  "layer_types": [
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention"
  ],
  "max_position_embeddings": 1048576,
  "metap": {
    "base_width": 256.0,
    "m_emb": 1.0,
    "metap_mode": "sp",
    "router_has_one_inf_dim": true,
    "use_metap": true,
    "use_metap_output_multiplier_without_metap": false
  },
  "model_type": "muse_spark_dspark",
  "non_linearity": "swiglu",
  "norm_affine": true,
  "norm_zero_centered_gamma": true,
  "num_attention_heads": 128,
  "num_hidden_layers": 5,
  "num_key_value_heads": 16,
  "output_norm_gain_center_type": "zero",
  "output_soft_cap_temp": 20.0,
  "pad_token_id": 200018,
  "post_norm_affine": false,
  "post_norm_gain_center_type": "one",
  "residual_dtype": "fp32",
  "rms_norm_eps": 1e-05,
  "rms_post_norm_eps": 1e-08,
  "rope_theta": 500000.0,
  "rope_use_fp32_in_outer_product": true,
  "sliding_window": 2048,
  "tie_word_embeddings": false,
  "use_attn_o_gate": true,
  "use_attn_o_norm": true,
  "use_ffn_learnable_scales": true,
  "use_qk_norm": true,
  "use_residual_gate": true,
  "use_rope": true,
  "use_sliding_window": true,
  "use_sqrt_attn_scaling": false,
  "vocab_size": 202048,
  "architectures": [
    "MuseSparkDSparkModel"
  ],
  "moe": {
    "is_enabled": false
  },
  "global_attn_cfg": [
    2048
  ],
  "every_n_layers_nope": null,
  "num_target_layers": 5,
  "target_hidden_size": 8192,
  "target_num_hidden_layers": 62,
  "target_layer_ids": [
    13,
    29,
    45,
    57,
    61
  ],
  "draft_vocab_size": 202048,
  "mask_token_id": 200018,
  "markov_rank": 512,
  "markov_head_type": "vanilla",
  "enable_confidence_head": true,
  "confidence_head_with_markov": true,
  "sample_from_anchor": true,
  "block_size": 7,
  "sliding_window_non_causal": false,
  "full_attention_causal": false,
  "use_aux_hidden_state": true,
  "dflash_config": {
    "causal": true,
    "mask_token_id": 200018,
    "target_layer_ids": [
      13,
      29,
      45,
      57,
      61
    ],
    "use_aux_hidden_state": true
  },
  "dtype": "bfloat16"
}