Update config.json

config.json

@@ -3,17 +3,22 @@
         "Step3p5ForCausalLM"
     ],
     "model_type": "step3p5",
+    "auto_map": {
+        "AutoConfig": "configuration_step3p5.Step3p5Config",
+        "AutoModelForCausalLM": "modeling_step3p5.Step3p5ForCausalLM"
+    },
     "hidden_size": 4096,
     "intermediate_size": 11264,
     "num_hidden_layers": 45,
-    "max_seq_len": 262144,
+    "max_seq_len": 32768,
+    "max_position_embedding": 32768,
     "vocab_size": 128815,
     "torch_dtype": "bfloat16",
-    "use_qk_norm": false,
+    "use_qk_norm": true,
     "moe_layers_enum": "3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44",
-    "use_mfa": false,
     "num_attention_heads": 64,
     "num_attention_groups": 8,
+    "num_nextn_predict_layers": 1,
     "head_dim": 128,
     "use_moe": true,
     "moe_num_experts": 288,
@@ -72,8 +77,6 @@
         10000.0,
         10000.0,
         1000000.0,
-        10000.0,
-        10000.0,
         10000.0
     ],
     "use_head_wise_attn_gate": true,
@@ -127,12 +130,9 @@
         "sliding_attention",
         "sliding_attention",
         "full_attention",
-        "sliding_attention",
-        "sliding_attention",
         "sliding_attention"
     ],
     "use_rope_layers": [],
-    "num_nextn_predict_layers": 3,
     "partial_rotary_factors": [
         0.5,
         1.0,
@@ -179,14 +179,11 @@
         1.0,
         1.0,
         0.5,
-        1.0,
-        1.0,
         1.0
     ],
     "eos_token_id": [
         1,
-        2,
-        128007
+        2
     ],
     "bos_token_id": 0,
     "attention_other_setting": {
@@ -242,8 +239,6 @@
         0.0,
         7,
         7,
-        0.0,
-        0.0,
         0.0
     ],
     "swiglu_limits_shared": [
@@ -292,14 +287,7 @@
         0.0,
         0.0,
         16,
-        0.0,
-        0.0,
         0.0
     ],
-    "zero_centered": true,
-    "max_position_embeddings": 262144,
-    "yarn_only_types": [
-        "full_attention"
-    ],
-    "model_max_lengths": 262144
+    "zero_centered": true
 }