| { |
| "attention_bias": false, |
| "attention_dropout": 0.0, |
| "bos_token_id": 200000, |
| "draft_position_mode": "rope", |
| "eos_token_id": [ |
| 200001, |
| 200008 |
| ], |
| "ffn_dim_multiplier": 2.0, |
| "head_dim": 64, |
| "hidden_act": "silu", |
| "hidden_size": 8192, |
| "initializer_range": 0.02, |
| "intermediate_size": 16384, |
| "intermediate_size_mlp": 16384, |
| "layer_types": [ |
| "sliding_attention", |
| "sliding_attention", |
| "sliding_attention", |
| "sliding_attention", |
| "sliding_attention" |
| ], |
| "max_position_embeddings": 1048576, |
| "metap": { |
| "base_width": 256.0, |
| "m_emb": 1.0, |
| "metap_mode": "sp", |
| "router_has_one_inf_dim": true, |
| "use_metap": true, |
| "use_metap_output_multiplier_without_metap": false |
| }, |
| "model_type": "muse_spark_dspark", |
| "non_linearity": "swiglu", |
| "norm_affine": true, |
| "norm_zero_centered_gamma": true, |
| "num_attention_heads": 128, |
| "num_hidden_layers": 5, |
| "num_key_value_heads": 16, |
| "output_norm_gain_center_type": "zero", |
| "output_soft_cap_temp": 20.0, |
| "pad_token_id": 200018, |
| "post_norm_affine": false, |
| "post_norm_gain_center_type": "one", |
| "residual_dtype": "fp32", |
| "rms_norm_eps": 1e-05, |
| "rms_post_norm_eps": 1e-08, |
| "rope_theta": 500000.0, |
| "rope_use_fp32_in_outer_product": true, |
| "sliding_window": 2048, |
| "tie_word_embeddings": false, |
| "use_attn_o_gate": true, |
| "use_attn_o_norm": true, |
| "use_ffn_learnable_scales": true, |
| "use_qk_norm": true, |
| "use_residual_gate": true, |
| "use_rope": true, |
| "use_sliding_window": true, |
| "use_sqrt_attn_scaling": false, |
| "vocab_size": 202048, |
| "architectures": [ |
| "MuseSparkDSparkModel" |
| ], |
| "moe": { |
| "is_enabled": false |
| }, |
| "global_attn_cfg": [ |
| 2048 |
| ], |
| "every_n_layers_nope": null, |
| "num_target_layers": 5, |
| "target_hidden_size": 8192, |
| "target_num_hidden_layers": 62, |
| "target_layer_ids": [ |
| 13, |
| 29, |
| 45, |
| 57, |
| 61 |
| ], |
| "draft_vocab_size": 202048, |
| "mask_token_id": 200018, |
| "markov_rank": 512, |
| "markov_head_type": "vanilla", |
| "enable_confidence_head": true, |
| "confidence_head_with_markov": true, |
| "sample_from_anchor": true, |
| "block_size": 7, |
| "sliding_window_non_causal": false, |
| "full_attention_causal": false, |
| "use_aux_hidden_state": true, |
| "dflash_config": { |
| "causal": true, |
| "mask_token_id": 200018, |
| "target_layer_ids": [ |
| 13, |
| 29, |
| 45, |
| 57, |
| 61 |
| ], |
| "use_aux_hidden_state": true |
| }, |
| "dtype": "bfloat16" |
| } |
|
|
