Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,65 @@
+{{ bos_token }}{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' }}
+        {% generation %}
+        {{- content}}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>' }}
+        {% endgeneration%}
+        {{- '\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,515 @@
+{
+  "architectures": [
+    "AfmoeForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_afmoe.AfmoeConfig",
+    "AutoModel": "modeling_afmoe.AfmoeModel",
+    "AutoModelForCausalLM": "modeling_afmoe.AfmoeForCausalLM"
+  },
+  "dtype": "bfloat16",
+  "global_attn_every_n_layers": 4,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 12288,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "load_balance_coeff": 5e-05,
+  "max_position_embeddings": 262144,
+  "model_type": "afmoe",
+  "moe_intermediate_size": 3072,
+  "mup_enabled": true,
+  "n_group": 1,
+  "num_attention_heads": 48,
+  "num_dense_layers": 6,
+  "num_expert_groups": 1,
+  "num_experts": 256,
+  "num_experts_per_tok": 4,
+  "num_hidden_layers": 60,
+  "num_key_value_heads": 8,
+  "num_limited_groups": 1,
+  "num_shared_experts": 1,
+  "quantization_config": {
+    "config_groups": {
+      "group_0": {
+        "format": "float-quantized",
+        "input_activations": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": true,
+          "group_size": 128,
+          "num_bits": 8,
+          "observer": null,
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "group",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        },
+        "output_activations": null,
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": [
+            128,
+            128
+          ],
+          "dynamic": false,
+          "group_size": null,
+          "num_bits": 8,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "block",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        }
+      }
+    },
+    "format": "float-quantized",
+    "global_compression_ratio": null,
+    "ignore": [
+      "model.layers.0.self_attn.q_proj",
+      "model.layers.0.self_attn.k_proj",
+      "model.layers.0.self_attn.v_proj",
+      "model.layers.0.self_attn.o_proj",
+      "model.layers.0.self_attn.gate_proj",
+      "model.layers.1.self_attn.q_proj",
+      "model.layers.1.self_attn.k_proj",
+      "model.layers.1.self_attn.v_proj",
+      "model.layers.1.self_attn.o_proj",
+      "model.layers.1.self_attn.gate_proj",
+      "model.layers.2.self_attn.q_proj",
+      "model.layers.2.self_attn.k_proj",
+      "model.layers.2.self_attn.v_proj",
+      "model.layers.2.self_attn.o_proj",
+      "model.layers.2.self_attn.gate_proj",
+      "model.layers.3.self_attn.q_proj",
+      "model.layers.3.self_attn.k_proj",
+      "model.layers.3.self_attn.v_proj",
+      "model.layers.3.self_attn.o_proj",
+      "model.layers.3.self_attn.gate_proj",
+      "model.layers.4.self_attn.q_proj",
+      "model.layers.4.self_attn.k_proj",
+      "model.layers.4.self_attn.v_proj",
+      "model.layers.4.self_attn.o_proj",
+      "model.layers.4.self_attn.gate_proj",
+      "model.layers.5.self_attn.q_proj",
+      "model.layers.5.self_attn.k_proj",
+      "model.layers.5.self_attn.v_proj",
+      "model.layers.5.self_attn.o_proj",
+      "model.layers.5.self_attn.gate_proj",
+      "model.layers.6.self_attn.q_proj",
+      "model.layers.6.self_attn.k_proj",
+      "model.layers.6.self_attn.v_proj",
+      "model.layers.6.self_attn.o_proj",
+      "model.layers.6.self_attn.gate_proj",
+      "model.layers.6.mlp.router.gate",
+      "model.layers.7.self_attn.q_proj",
+      "model.layers.7.self_attn.k_proj",
+      "model.layers.7.self_attn.v_proj",
+      "model.layers.7.self_attn.o_proj",
+      "model.layers.7.self_attn.gate_proj",
+      "model.layers.7.mlp.router.gate",
+      "model.layers.8.self_attn.q_proj",
+      "model.layers.8.self_attn.k_proj",
+      "model.layers.8.self_attn.v_proj",
+      "model.layers.8.self_attn.o_proj",
+      "model.layers.8.self_attn.gate_proj",
+      "model.layers.8.mlp.router.gate",
+      "model.layers.9.self_attn.q_proj",
+      "model.layers.9.self_attn.k_proj",
+      "model.layers.9.self_attn.v_proj",
+      "model.layers.9.self_attn.o_proj",
+      "model.layers.9.self_attn.gate_proj",
+      "model.layers.9.mlp.router.gate",
+      "model.layers.10.self_attn.q_proj",
+      "model.layers.10.self_attn.k_proj",
+      "model.layers.10.self_attn.v_proj",
+      "model.layers.10.self_attn.o_proj",
+      "model.layers.10.self_attn.gate_proj",
+      "model.layers.10.mlp.router.gate",
+      "model.layers.11.self_attn.q_proj",
+      "model.layers.11.self_attn.k_proj",
+      "model.layers.11.self_attn.v_proj",
+      "model.layers.11.self_attn.o_proj",
+      "model.layers.11.self_attn.gate_proj",
+      "model.layers.11.mlp.router.gate",
+      "model.layers.12.self_attn.q_proj",
+      "model.layers.12.self_attn.k_proj",
+      "model.layers.12.self_attn.v_proj",
+      "model.layers.12.self_attn.o_proj",
+      "model.layers.12.self_attn.gate_proj",
+      "model.layers.12.mlp.router.gate",
+      "model.layers.13.self_attn.q_proj",
+      "model.layers.13.self_attn.k_proj",
+      "model.layers.13.self_attn.v_proj",
+      "model.layers.13.self_attn.o_proj",
+      "model.layers.13.self_attn.gate_proj",
+      "model.layers.13.mlp.router.gate",
+      "model.layers.14.self_attn.q_proj",
+      "model.layers.14.self_attn.k_proj",
+      "model.layers.14.self_attn.v_proj",
+      "model.layers.14.self_attn.o_proj",
+      "model.layers.14.self_attn.gate_proj",
+      "model.layers.14.mlp.router.gate",
+      "model.layers.15.self_attn.q_proj",
+      "model.layers.15.self_attn.k_proj",
+      "model.layers.15.self_attn.v_proj",
+      "model.layers.15.self_attn.o_proj",
+      "model.layers.15.self_attn.gate_proj",
+      "model.layers.15.mlp.router.gate",
+      "model.layers.16.self_attn.q_proj",
+      "model.layers.16.self_attn.k_proj",
+      "model.layers.16.self_attn.v_proj",
+      "model.layers.16.self_attn.o_proj",
+      "model.layers.16.self_attn.gate_proj",
+      "model.layers.16.mlp.router.gate",
+      "model.layers.17.self_attn.q_proj",
+      "model.layers.17.self_attn.k_proj",
+      "model.layers.17.self_attn.v_proj",
+      "model.layers.17.self_attn.o_proj",
+      "model.layers.17.self_attn.gate_proj",
+      "model.layers.17.mlp.router.gate",
+      "model.layers.18.self_attn.q_proj",
+      "model.layers.18.self_attn.k_proj",
+      "model.layers.18.self_attn.v_proj",
+      "model.layers.18.self_attn.o_proj",
+      "model.layers.18.self_attn.gate_proj",
+      "model.layers.18.mlp.router.gate",
+      "model.layers.19.self_attn.q_proj",
+      "model.layers.19.self_attn.k_proj",
+      "model.layers.19.self_attn.v_proj",
+      "model.layers.19.self_attn.o_proj",
+      "model.layers.19.self_attn.gate_proj",
+      "model.layers.19.mlp.router.gate",
+      "model.layers.20.self_attn.q_proj",
+      "model.layers.20.self_attn.k_proj",
+      "model.layers.20.self_attn.v_proj",
+      "model.layers.20.self_attn.o_proj",
+      "model.layers.20.self_attn.gate_proj",
+      "model.layers.20.mlp.router.gate",
+      "model.layers.21.self_attn.q_proj",
+      "model.layers.21.self_attn.k_proj",
+      "model.layers.21.self_attn.v_proj",
+      "model.layers.21.self_attn.o_proj",
+      "model.layers.21.self_attn.gate_proj",
+      "model.layers.21.mlp.router.gate",
+      "model.layers.22.self_attn.q_proj",
+      "model.layers.22.self_attn.k_proj",
+      "model.layers.22.self_attn.v_proj",
+      "model.layers.22.self_attn.o_proj",
+      "model.layers.22.self_attn.gate_proj",
+      "model.layers.22.mlp.router.gate",
+      "model.layers.23.self_attn.q_proj",
+      "model.layers.23.self_attn.k_proj",
+      "model.layers.23.self_attn.v_proj",
+      "model.layers.23.self_attn.o_proj",
+      "model.layers.23.self_attn.gate_proj",
+      "model.layers.23.mlp.router.gate",
+      "model.layers.24.self_attn.q_proj",
+      "model.layers.24.self_attn.k_proj",
+      "model.layers.24.self_attn.v_proj",
+      "model.layers.24.self_attn.o_proj",
+      "model.layers.24.self_attn.gate_proj",
+      "model.layers.24.mlp.router.gate",
+      "model.layers.25.self_attn.q_proj",
+      "model.layers.25.self_attn.k_proj",
+      "model.layers.25.self_attn.v_proj",
+      "model.layers.25.self_attn.o_proj",
+      "model.layers.25.self_attn.gate_proj",
+      "model.layers.25.mlp.router.gate",
+      "model.layers.26.self_attn.q_proj",
+      "model.layers.26.self_attn.k_proj",
+      "model.layers.26.self_attn.v_proj",
+      "model.layers.26.self_attn.o_proj",
+      "model.layers.26.self_attn.gate_proj",
+      "model.layers.26.mlp.router.gate",
+      "model.layers.27.self_attn.q_proj",
+      "model.layers.27.self_attn.k_proj",
+      "model.layers.27.self_attn.v_proj",
+      "model.layers.27.self_attn.o_proj",
+      "model.layers.27.self_attn.gate_proj",
+      "model.layers.27.mlp.router.gate",
+      "model.layers.28.self_attn.q_proj",
+      "model.layers.28.self_attn.k_proj",
+      "model.layers.28.self_attn.v_proj",
+      "model.layers.28.self_attn.o_proj",
+      "model.layers.28.self_attn.gate_proj",
+      "model.layers.28.mlp.router.gate",
+      "model.layers.29.self_attn.q_proj",
+      "model.layers.29.self_attn.k_proj",
+      "model.layers.29.self_attn.v_proj",
+      "model.layers.29.self_attn.o_proj",
+      "model.layers.29.self_attn.gate_proj",
+      "model.layers.29.mlp.router.gate",
+      "model.layers.30.self_attn.q_proj",
+      "model.layers.30.self_attn.k_proj",
+      "model.layers.30.self_attn.v_proj",
+      "model.layers.30.self_attn.o_proj",
+      "model.layers.30.self_attn.gate_proj",
+      "model.layers.30.mlp.router.gate",
+      "model.layers.31.self_attn.q_proj",
+      "model.layers.31.self_attn.k_proj",
+      "model.layers.31.self_attn.v_proj",
+      "model.layers.31.self_attn.o_proj",
+      "model.layers.31.self_attn.gate_proj",
+      "model.layers.31.mlp.router.gate",
+      "model.layers.32.self_attn.q_proj",
+      "model.layers.32.self_attn.k_proj",
+      "model.layers.32.self_attn.v_proj",
+      "model.layers.32.self_attn.o_proj",
+      "model.layers.32.self_attn.gate_proj",
+      "model.layers.32.mlp.router.gate",
+      "model.layers.33.self_attn.q_proj",
+      "model.layers.33.self_attn.k_proj",
+      "model.layers.33.self_attn.v_proj",
+      "model.layers.33.self_attn.o_proj",
+      "model.layers.33.self_attn.gate_proj",
+      "model.layers.33.mlp.router.gate",
+      "model.layers.34.self_attn.q_proj",
+      "model.layers.34.self_attn.k_proj",
+      "model.layers.34.self_attn.v_proj",
+      "model.layers.34.self_attn.o_proj",
+      "model.layers.34.self_attn.gate_proj",
+      "model.layers.34.mlp.router.gate",
+      "model.layers.35.self_attn.q_proj",
+      "model.layers.35.self_attn.k_proj",
+      "model.layers.35.self_attn.v_proj",
+      "model.layers.35.self_attn.o_proj",
+      "model.layers.35.self_attn.gate_proj",
+      "model.layers.35.mlp.router.gate",
+      "model.layers.36.self_attn.q_proj",
+      "model.layers.36.self_attn.k_proj",
+      "model.layers.36.self_attn.v_proj",
+      "model.layers.36.self_attn.o_proj",
+      "model.layers.36.self_attn.gate_proj",
+      "model.layers.36.mlp.router.gate",
+      "model.layers.37.self_attn.q_proj",
+      "model.layers.37.self_attn.k_proj",
+      "model.layers.37.self_attn.v_proj",
+      "model.layers.37.self_attn.o_proj",
+      "model.layers.37.self_attn.gate_proj",
+      "model.layers.37.mlp.router.gate",
+      "model.layers.38.self_attn.q_proj",
+      "model.layers.38.self_attn.k_proj",
+      "model.layers.38.self_attn.v_proj",
+      "model.layers.38.self_attn.o_proj",
+      "model.layers.38.self_attn.gate_proj",
+      "model.layers.38.mlp.router.gate",
+      "model.layers.39.self_attn.q_proj",
+      "model.layers.39.self_attn.k_proj",
+      "model.layers.39.self_attn.v_proj",
+      "model.layers.39.self_attn.o_proj",
+      "model.layers.39.self_attn.gate_proj",
+      "model.layers.39.mlp.router.gate",
+      "model.layers.40.self_attn.q_proj",
+      "model.layers.40.self_attn.k_proj",
+      "model.layers.40.self_attn.v_proj",
+      "model.layers.40.self_attn.o_proj",
+      "model.layers.40.self_attn.gate_proj",
+      "model.layers.40.mlp.router.gate",
+      "model.layers.41.self_attn.q_proj",
+      "model.layers.41.self_attn.k_proj",
+      "model.layers.41.self_attn.v_proj",
+      "model.layers.41.self_attn.o_proj",
+      "model.layers.41.self_attn.gate_proj",
+      "model.layers.41.mlp.router.gate",
+      "model.layers.42.self_attn.q_proj",
+      "model.layers.42.self_attn.k_proj",
+      "model.layers.42.self_attn.v_proj",
+      "model.layers.42.self_attn.o_proj",
+      "model.layers.42.self_attn.gate_proj",
+      "model.layers.42.mlp.router.gate",
+      "model.layers.43.self_attn.q_proj",
+      "model.layers.43.self_attn.k_proj",
+      "model.layers.43.self_attn.v_proj",
+      "model.layers.43.self_attn.o_proj",
+      "model.layers.43.self_attn.gate_proj",
+      "model.layers.43.mlp.router.gate",
+      "model.layers.44.self_attn.q_proj",
+      "model.layers.44.self_attn.k_proj",
+      "model.layers.44.self_attn.v_proj",
+      "model.layers.44.self_attn.o_proj",
+      "model.layers.44.self_attn.gate_proj",
+      "model.layers.44.mlp.router.gate",
+      "model.layers.45.self_attn.q_proj",
+      "model.layers.45.self_attn.k_proj",
+      "model.layers.45.self_attn.v_proj",
+      "model.layers.45.self_attn.o_proj",
+      "model.layers.45.self_attn.gate_proj",
+      "model.layers.45.mlp.router.gate",
+      "model.layers.46.self_attn.q_proj",
+      "model.layers.46.self_attn.k_proj",
+      "model.layers.46.self_attn.v_proj",
+      "model.layers.46.self_attn.o_proj",
+      "model.layers.46.self_attn.gate_proj",
+      "model.layers.46.mlp.router.gate",
+      "model.layers.47.self_attn.q_proj",
+      "model.layers.47.self_attn.k_proj",
+      "model.layers.47.self_attn.v_proj",
+      "model.layers.47.self_attn.o_proj",
+      "model.layers.47.self_attn.gate_proj",
+      "model.layers.47.mlp.router.gate",
+      "model.layers.48.self_attn.q_proj",
+      "model.layers.48.self_attn.k_proj",
+      "model.layers.48.self_attn.v_proj",
+      "model.layers.48.self_attn.o_proj",
+      "model.layers.48.self_attn.gate_proj",
+      "model.layers.48.mlp.router.gate",
+      "model.layers.49.self_attn.q_proj",
+      "model.layers.49.self_attn.k_proj",
+      "model.layers.49.self_attn.v_proj",
+      "model.layers.49.self_attn.o_proj",
+      "model.layers.49.self_attn.gate_proj",
+      "model.layers.49.mlp.router.gate",
+      "model.layers.50.self_attn.q_proj",
+      "model.layers.50.self_attn.k_proj",
+      "model.layers.50.self_attn.v_proj",
+      "model.layers.50.self_attn.o_proj",
+      "model.layers.50.self_attn.gate_proj",
+      "model.layers.50.mlp.router.gate",
+      "model.layers.51.self_attn.q_proj",
+      "model.layers.51.self_attn.k_proj",
+      "model.layers.51.self_attn.v_proj",
+      "model.layers.51.self_attn.o_proj",
+      "model.layers.51.self_attn.gate_proj",
+      "model.layers.51.mlp.router.gate",
+      "model.layers.52.self_attn.q_proj",
+      "model.layers.52.self_attn.k_proj",
+      "model.layers.52.self_attn.v_proj",
+      "model.layers.52.self_attn.o_proj",
+      "model.layers.52.self_attn.gate_proj",
+      "model.layers.52.mlp.router.gate",
+      "model.layers.53.self_attn.q_proj",
+      "model.layers.53.self_attn.k_proj",
+      "model.layers.53.self_attn.v_proj",
+      "model.layers.53.self_attn.o_proj",
+      "model.layers.53.self_attn.gate_proj",
+      "model.layers.53.mlp.router.gate",
+      "model.layers.54.self_attn.q_proj",
+      "model.layers.54.self_attn.k_proj",
+      "model.layers.54.self_attn.v_proj",
+      "model.layers.54.self_attn.o_proj",
+      "model.layers.54.self_attn.gate_proj",
+      "model.layers.54.mlp.router.gate",
+      "model.layers.55.self_attn.q_proj",
+      "model.layers.55.self_attn.k_proj",
+      "model.layers.55.self_attn.v_proj",
+      "model.layers.55.self_attn.o_proj",
+      "model.layers.55.self_attn.gate_proj",
+      "model.layers.55.mlp.router.gate",
+      "model.layers.56.self_attn.q_proj",
+      "model.layers.56.self_attn.k_proj",
+      "model.layers.56.self_attn.v_proj",
+      "model.layers.56.self_attn.o_proj",
+      "model.layers.56.self_attn.gate_proj",
+      "model.layers.56.mlp.router.gate",
+      "model.layers.57.self_attn.q_proj",
+      "model.layers.57.self_attn.k_proj",
+      "model.layers.57.self_attn.v_proj",
+      "model.layers.57.self_attn.o_proj",
+      "model.layers.57.self_attn.gate_proj",
+      "model.layers.57.mlp.router.gate",
+      "model.layers.58.self_attn.q_proj",
+      "model.layers.58.self_attn.k_proj",
+      "model.layers.58.self_attn.v_proj",
+      "model.layers.58.self_attn.o_proj",
+      "model.layers.58.self_attn.gate_proj",
+      "model.layers.58.mlp.router.gate",
+      "model.layers.59.self_attn.q_proj",
+      "model.layers.59.self_attn.k_proj",
+      "model.layers.59.self_attn.v_proj",
+      "model.layers.59.self_attn.o_proj",
+      "model.layers.59.self_attn.gate_proj",
+      "model.layers.59.mlp.router.gate",
+      "lm_head"
+    ],
+    "kv_cache_scheme": null,
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed",
+    "sparsity_config": {},
+    "transform_config": {},
+    "version": "0.14.0"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000,
+  "route_norm": true,
+  "route_scale": 2.448,
+  "score_func": "sigmoid",
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "topk_group": 1,
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_grouped_mm": true,
+  "vocab_size": 200192
+}

configuration_afmoe.py

@@ -0,0 +1,133 @@
+# coding=utf-8
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from transformers.configuration_utils import PretrainedConfig
+from transformers.modeling_rope_utils import rope_config_validation
+from transformers.configuration_utils import layer_type_validation
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+class AfmoeConfig(PretrainedConfig):
+    """
+    n_group (`int`, *optional*, defaults to 1):
+            Number of groups for routed experts.
+    topk_group (`int`, *optional*, defaults to 1):
+        Number of selected groups for each token(for each token, ensuring the selected experts is only within `topk_group` groups).
+    """
+    model_type = "afmoe"
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+
+    def __init__(
+        self,
+        num_hidden_layers: int = 32,
+        vocab_size: int = 200192,
+        hidden_size: int = 2048,
+        intermediate_size: int = 6144,
+        moe_intermediate_size=1408,
+        num_dense_layers=1,
+        num_attention_heads=16,
+        num_key_value_heads=None,
+        head_dim=128,
+        hidden_act="silu",
+        max_position_embeddings=16384,
+        initializer_range=0.02,
+        rms_norm_eps=1e-5,
+        use_cache=True,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        num_experts=64,
+        num_experts_per_tok=6,
+        num_shared_experts=2,
+        num_expert_groups=1,
+        num_limited_groups=1,
+        score_func="sigmoid",
+        route_norm=True,
+        route_scale=1.0,
+        global_attn_every_n_layers=4,
+        sliding_window=1024,
+        mup_enabled=False,
+        layer_types=None,
+        attention_dropout: float = 0.0,
+        n_group: int = 1,
+        topk_group: int = 1,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_dense_layers = num_dense_layers
+        self.num_attention_heads = num_attention_heads
+        self.head_dim = head_dim
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+
+
+        # MoE specific
+        self.moe_intermediate_size = moe_intermediate_size
+        self.num_experts_per_tok = num_experts_per_tok
+        self.n_group = n_group
+        self.topk_group = topk_group
+        self.num_experts = num_experts
+        self.num_shared_experts = num_shared_experts
+        self.num_expert_groups = num_expert_groups
+        self.num_limited_groups = num_limited_groups
+        self.score_func = score_func
+        self.route_norm = route_norm
+        self.route_scale = route_scale
+
+
+        # Attention specific
+        self.attention_dropout = attention_dropout
+        self.global_attn_every_n_layers = global_attn_every_n_layers
+        self.sliding_window = sliding_window
+        self.layer_types = layer_types
+        if self.layer_types is None:
+            self.layer_types = [
+                "sliding_attention" if bool((i + 1) % global_attn_every_n_layers) else "full_attention" for i in range(self.num_hidden_layers)
+            ]
+        layer_type_validation(self.layer_types)
+
+        # muP specific
+        self.mup_enabled = mup_enabled
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+
+
+        # Validate rope configs
+        if self.rope_scaling is not None and "type" in self.rope_scaling:
+            self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+        rope_config_validation(self)
+
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+
+__all__ = ["AfmoeConfig"]

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "do_sample": true,
+  "eos_token_id": 3,
+  "pad_token_id": 12,
+  "temperature": 0.8,
+  "top_p": 0.8,
+  "transformers_version": "4.57.6"
+}

model-00002-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32a18d0dd75fe2c2b5df5ff40f6f0428cd90eb1681c8861ccec96f4cf795dead
+size 4993010240

model-00003-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b156805b23deeffe0be50154690039412b8b0ae61cb3dff038edec45caf524b
+size 4997737720

model-00004-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9271ba1753b5a980f646024d67ed500efee02c97e258bab245e4b63192f66606
+size 4997737816

model-00005-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7133af971736fc62e46748a963f2e8064884b3d6fcde75045069d43593255294
+size 4986706032

model-00007-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:488c721d1e2c44735cc0283b33a88ce230b62c567824334bd8836b594a95673e
+size 4997739824

model-00009-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d782411caf60c5ecf0d9da8057ce1fab40f11040942006ff9e4bf42c5955482b
+size 4993010824

model-00010-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e378a634c7ed99ca5497ce28a915bf6e47430ec5ea3275858e1806531b10ef2
+size 4997738792

model-00011-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40c8b9186ae9113c46820aacb387432daf6aaf28df54469e6a1b422c48441228
+size 4997739272

model-00012-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72b38889df4ebdba8a44c574293af942ae8687737102e0166a2eaa13ad005912
+size 4993010896

model-00013-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a76bc2f719f553dc4ff19329074cd9eb2b5ace9dabc7828d0988375cce7791e
+size 4997738760

model-00017-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acd5833d26d5dd7181385b1b6ce2c13e366059f060e277dda5ec26c6ed5a2c77
+size 4997739224

model-00021-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78329a9e7d9c8859518446eb31e981e480afdfbed1366269a8e0cb88c7d04564
+size 4993011008

model-00023-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cdc0b84e5026628a1959509bfb2fcd7ba140949e8aab6454c54f8086c441507
+size 4997739144

model-00025-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20483ca7e8c51af45aec715e108c4cffaa4ce46b3e0fa25997c21543adb9a482
+size 4997738760

model-00026-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16637170984f217fa1e7049710cb659b26035049af1b30c8dc781c8ac8be29f0
+size 4997739112

model-00028-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0186ff433240e0b7efc44d5ba4c57e2cbc37acb9889834f9f5bf9035ca653da8
+size 4997738760

model-00029-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98c9dcff7ef969e1873d670b44bacdd077902f6aa571af0a65e0dc3b31e854d3
+size 4997739072

model-00032-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc96c5834415bcf79000d860ac4f49f69824949c953f8f869144d4bde0528cd4
+size 4997739032

model-00033-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:085456c10dbaa210cbae28bb81bb3865d64701b4db71c4c7c8b0ab52bcd8557b
+size 4993011160

model-00034-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07011b261b19ea30f01fe8811c496ea7ad30b827c5b99f89d7cfdf41ff9e7003
+size 4997738768

model-00036-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08b66a2065cbb57f9f90b18d68acb200d10df27b13d2c8777695dae65230e430
+size 4993011200

model-00037-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f50481424ed34d6f07666159cffa345a7cda8c3e578578c5f8e7db208d9ced5
+size 4997738768

model-00038-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02a91a789bb6b129c15c3f0fa35a14e14d68a3470ce6d9bed99cc3d1be5dbe0c
+size 4997738960

model-00042-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d1b6c4504546d2533dc36c36026f051816f29c7a2d145a755b8d21a8d159806
+size 4993011280

model-00045-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:401c752c03eecaf08e687902472173e454f482c0ce157d86f0886f63e4fb67a4
+size 4983572728

model-00046-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1c27097ae5952749930b249a1bdffd4b5c4cc27aafd00b4425fb356f68d5c0a
+size 4997738776

model-00049-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:959f00bed59b83daa7b55eb18674e13b515b46cac81f9fb5f642ba7608dd51ff
+size 4993010784

model-00052-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:077ab336e9bfc0204462f0e56d514ffed612fc8606802d95db74c4a02c584102
+size 4993010856

model-00053-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bf1822d01dd98208d07affcf1e4f4e1c516705afe85196e0b973b19d9a369e7
+size 4997738768

model-00054-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f31120c1568007a0624dd92316574793d2261604b1e0ed687d10545b6a0533a0
+size 4997739272

model-00055-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6831b68e33589f769016b91b59d21f09fbbbc2fbfdd48dfdd1c0b7a6bd417a7c
+size 4993010920

model-00059-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6a77002859e4e3f41e0124ef1ed91c89e47e577a5526b0ad07b1f814bb40e8c
+size 4997738760

model-00061-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19479762194866893aff2df4d999587c24a80dc6191eab3f9d2e310a1fd99365
+size 4993010992

model-00063-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:943921a2dbe92055bd957e86c4c44df9765545ea78112c3f2c9804d0737b4582
+size 4997739168

model-00064-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd60dbbf5a6948d770df1bda91a1560cef03b9b27581bc27ae61e14e4f5b7f33
+size 4993011032

model-00066-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88e09c3d43399cbde32193f494545b71d72b296336a0d7d6ba97548da2236799
+size 4997739128

model-00069-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87641ec7d6ee5ff271c23388876270ea4c28b2574bc88b22ea0a617c414c34d8
+size 4997739088

model-00072-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:691328fec81a94c2b5349197748c60cb0821840f232dbb048a7ed7f35f11177e
+size 4997739048

model-00079-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:820c83fed28d86790fd8f3131dfbb75263962a79d6ef92c7fe5ee6643ca148d5
+size 4993011216

model-00080-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d5b7dc48f07b71998497905cff7c7d17ea3896bdc3fb89f2a28bc033680843b
+size 4997738768

model-00081-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8fd8937dd2c7a96e7a9a471aab8a13d84c5b7290a0b64cf89f8344f41bb130c
+size 4071002704

recipe.yaml

@@ -0,0 +1,7 @@
+default_stage:
+  default_modifiers:
+    QuantizationModifier:
+      targets: [Linear]
+      ignore: ['re:.*mlp\.router\.gate.*', lm_head, 're:.*norm.*', 're:.*self_attn.*']
+      scheme: FP8_BLOCK
+      bypass_divisibility_checks: false

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d864fe0d9c300d44c54006f5960548946d507d8ec05a082a3bff3e49de58208
+size 14614721