Update minimized Kimi-K2.5 MXFP4 model

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text
+figures/demo_video.mp4 filter=lfs diff=lfs merge=lfs -text

README.md

@@ -16,8 +16,9 @@ base_model:
 - **Operating System(s):** Linux
 - **Inference Engine:** [vLLM](https://docs.vllm.ai/en/latest/)
 - **Model Optimizer:** [AMD-Quark](https://quark.docs.amd.com/latest/index.html) (V0.11.1)
-  - **Weight quantization:** MOE-only, OCP MXFP4, Static 
-  - **Activation quantization:** MOE-only, OCP MXFP4, Dynamic
+  - **Quantized layers:** `layers.0.mlp`, `experts` and `shared_experts`
+  - **Weight quantization:** OCP MXFP4, Static 
+  - **Activation quantization:** OCP MXFP4, Dynamic
 - **Calibration Dataset:** [Pile](https://huggingface.co/datasets/mit-han-lab/pile-val-backup)
 
 This model was built with Kimi-K2.5 model by applying [AMD-Quark](https://quark.docs.amd.com/latest/index.html) for MXFP4 quantization.
@@ -29,7 +30,7 @@ The model was quantized from [moonshotai/Kimi-K2.5](https://huggingface.co/moons
 **Quantization scripts:**
 ```
 cd Quark/examples/torch/language_modeling/llm_ptq/
-exclude_layers="*self_attn* *mlp.gate *lm_head *mlp.gate_proj *mlp.up_proj *mlp.down_proj *shared_experts* *mm_projector* *vision_tower*"
+exclude_layers="*self_attn* *mlp.gate *lm_head *mm_projector* *vision_tower*"
 
 python quantize_quark.py \
     --model_dir moonshotai/Kimi-K2.5 \
@@ -65,9 +66,9 @@ The model was evaluated on GSM8K benchmarks.
    </td>
    <td>94.09
    </td>
-   <td>93.25
+   <td>93.1
    </td>
-   <td>99.1%
+   <td>98.95%
    </td>
   </tr>
 </table>

chat_template.jinja

@@ -5,7 +5,7 @@
     {%- elif c is not none -%}
       {% for content in c -%}
         {% if content['type'] == 'image' or content['type'] == 'image_url' -%}
-          <|media_start|>image<|media_content|><|media_pad|><|media_end|>
+          <|media_begin|>image<|media_content|><|media_pad|><|media_end|>
         {% elif content['type'] == 'video' or content['type']== 'video_url'-%}
           <|kimi_k25_video_placeholder|>
         {% else -%}
@@ -57,10 +57,6 @@
     <|im_system|>tool_declare<|im_middle|>{{ tools | tojson(separators=(',', ':')) }}<|im_end|>
   {%- endif -%}
 {%- endif -%}
-
-{%- if messages|length == 0 or messages[0]['role'] != 'system' -%}
-  <|im_system|>system<|im_middle|>You are Kimi, an AI assistant created by Moonshot AI.<|im_end|>
-{%- endif -%}
   
 {%- for message in hist_msgs -%}
   {{set_roles(message)}}

config.json

@@ -14,6 +14,111 @@
   "media_placeholder_token_id": 163605,
   "model_type": "kimi_k25",
   "pad_token_id": 163839,
+  "quantization_config": {
+    "algo_config": null,
+    "exclude": [
+      "language_model.lm_head",
+      "language_model.model.layers.0.self_attn.kv_a_proj_with_mqa",
+      "language_model.model.layers.0.self_attn.kv_b_proj",
+      "language_model.model.layers.0.self_attn.o_proj",
+      "language_model.model.layers.0.self_attn.q_a_proj",
+      "language_model.model.layers.0.self_attn.q_b_proj",
+      "language_model.model.layers.1.mlp.gate",
+      "language_model.model.layers.1.self_attn.kv_a_proj_with_mqa",
+      "language_model.model.layers.1.self_attn.kv_b_proj",
+      "language_model.model.layers.1.self_attn.o_proj",
+      "language_model.model.layers.1.self_attn.q_a_proj",
+      "language_model.model.layers.1.self_attn.q_b_proj",
+      "language_model.model.layers.2.mlp.gate",
+      "language_model.model.layers.2.self_attn.kv_a_proj_with_mqa",
+      "language_model.model.layers.2.self_attn.kv_b_proj",
+      "language_model.model.layers.2.self_attn.o_proj",
+      "language_model.model.layers.2.self_attn.q_a_proj",
+      "language_model.model.layers.2.self_attn.q_b_proj",
+      "language_model.model.layers.3.mlp.gate",
+      "language_model.model.layers.3.self_attn.kv_a_proj_with_mqa",
+      "language_model.model.layers.3.self_attn.kv_b_proj",
+      "language_model.model.layers.3.self_attn.o_proj",
+      "language_model.model.layers.3.self_attn.q_a_proj",
+      "language_model.model.layers.3.self_attn.q_b_proj",
+      "mm_projector.proj.0",
+      "mm_projector.proj.2",
+      "vision_tower.encoder.blocks.0.mlp.fc0",
+      "vision_tower.encoder.blocks.0.mlp.fc1",
+      "vision_tower.encoder.blocks.0.norm0",
+      "vision_tower.encoder.blocks.0.norm1",
+      "vision_tower.encoder.blocks.0.wo",
+      "vision_tower.encoder.blocks.0.wqkv",
+      "vision_tower.encoder.blocks.1.mlp.fc0",
+      "vision_tower.encoder.blocks.1.mlp.fc1",
+      "vision_tower.encoder.blocks.1.norm0",
+      "vision_tower.encoder.blocks.1.norm1",
+      "vision_tower.encoder.blocks.1.wo",
+      "vision_tower.encoder.blocks.1.wqkv",
+      "vision_tower.encoder.blocks.2.mlp.fc0",
+      "vision_tower.encoder.blocks.2.mlp.fc1",
+      "vision_tower.encoder.blocks.2.norm0",
+      "vision_tower.encoder.blocks.2.norm1",
+      "vision_tower.encoder.blocks.2.wo",
+      "vision_tower.encoder.blocks.2.wqkv",
+      "vision_tower.encoder.blocks.3.mlp.fc0",
+      "vision_tower.encoder.blocks.3.mlp.fc1",
+      "vision_tower.encoder.blocks.3.norm0",
+      "vision_tower.encoder.blocks.3.norm1",
+      "vision_tower.encoder.blocks.3.wo",
+      "vision_tower.encoder.blocks.3.wqkv"
+    ],
+    "export": {
+      "kv_cache_group": [],
+      "min_kv_scale": 0.0,
+      "pack_method": "reorder",
+      "weight_format": "real_quantized",
+      "weight_merge_groups": null
+    },
+    "global_quant_config": {
+      "bias": null,
+      "input_tensors": {
+        "ch_axis": -1,
+        "dtype": "fp4",
+        "group_size": 32,
+        "is_dynamic": true,
+        "is_scale_quant": false,
+        "mx_element_dtype": null,
+        "observer_cls": "PerBlockMXObserver",
+        "qscheme": "per_group",
+        "round_method": "half_even",
+        "scale_calculation_mode": "even",
+        "scale_format": "e8m0",
+        "scale_type": "float",
+        "symmetric": null
+      },
+      "output_tensors": null,
+      "target_device": null,
+      "weight": {
+        "ch_axis": -1,
+        "dtype": "fp4",
+        "group_size": 32,
+        "is_dynamic": false,
+        "is_scale_quant": false,
+        "mx_element_dtype": null,
+        "observer_cls": "PerBlockMXObserver",
+        "qscheme": "per_group",
+        "round_method": "half_even",
+        "scale_calculation_mode": "even",
+        "scale_format": "e8m0",
+        "scale_type": "float",
+        "symmetric": null
+      }
+    },
+    "kv_cache_post_rope": false,
+    "kv_cache_quant_config": {},
+    "layer_quant_config": {},
+    "layer_type_quant_config": {},
+    "quant_method": "quark",
+    "quant_mode": "eager_mode",
+    "softmax_quant_spec": null,
+    "version": "0.11.2+b560ff9e7f9"
+  },
   "text_config": {
     "_name_or_path": "",
     "add_cross_attention": false,
@@ -69,16 +174,16 @@
     "moe_intermediate_size": 2048,
     "moe_layer_freq": 1,
     "n_group": 1,
-    "n_routed_experts": 8,
+    "n_routed_experts": 384,
     "n_shared_experts": 1,
     "no_repeat_ngram_size": 0,
     "norm_topk_prob": true,
-    "num_attention_heads": 128,
+    "num_attention_heads": 64,
     "num_beam_groups": 1,
     "num_beams": 1,
     "num_experts_per_tok": 8,
     "num_hidden_layers": 4,
-    "num_key_value_heads": 128,
+    "num_key_value_heads": 64,
     "num_nextn_predict_layers": 0,
     "num_return_sequences": 1,
     "output_attentions": false,
@@ -154,67 +259,6 @@
     "vt_hidden_size": 1152,
     "vt_intermediate_size": 4304,
     "vt_num_attention_heads": 16,
-    "vt_num_hidden_layers": 27
-  },
-  "quantization_config": {
-    "global_quant_config": {
-      "input_tensors": {
-        "dtype": "fp4",
-        "is_dynamic": true,
-        "qscheme": "per_group",
-        "ch_axis": -1,
-        "group_size": 32,
-        "symmetric": null,
-        "round_method": "half_even",
-        "scale_type": "float",
-        "scale_format": "e8m0",
-        "scale_calculation_mode": "even",
-        "mx_element_dtype": null,
-        "observer_cls": "PerBlockMXObserver",
-        "is_scale_quant": false
-      },
-      "output_tensors": null,
-      "weight": {
-        "dtype": "fp4",
-        "is_dynamic": false,
-        "qscheme": "per_group",
-        "ch_axis": -1,
-        "group_size": 32,
-        "symmetric": null,
-        "round_method": "half_even",
-        "scale_type": "float",
-        "scale_format": "e8m0",
-        "scale_calculation_mode": "even",
-        "mx_element_dtype": null,
-        "observer_cls": "PerBlockMXObserver",
-        "is_scale_quant": false
-      },
-      "bias": null,
-      "target_device": null
-    },
-    "exclude": [
-      "lm_head",
-      "re:.*self_attn.*",
-      "re:.*shared_experts.*",
-      "re:.*mlp\\.(gate|up|gate_up|down)_proj.*",
-      "re:mm_projector.*",
-      "re:vision_tower.*"
-    ],
-    "algo_config": null,
-    "softmax_quant_spec": null,
-    "quant_method": "quark",
-    "layer_type_quant_config": {},
-    "layer_quant_config": {},
-    "kv_cache_quant_config": {},
-    "kv_cache_post_rope": false,
-    "quant_mode": "eager_mode",
-    "version": "0.11+4a34634b4a",
-    "export": {
-      "kv_cache_group": [],
-      "min_kv_scale": 0.0,
-      "pack_method": "reorder",
-      "weight_format": "real_quantized",
-      "weight_merge_groups": null
-    }
+    "vt_num_hidden_layers": 4
   }
 }

docs/deploy_guidance.md

@@ -0,0 +1,82 @@
+# Kimi-K2.5 Deployment Guide
+
+> [!Note]
+> This guide only provides some examples of deployment commands for Kimi-K2.5, which may not be the optimal configuration. Since inference engines are still being updated frequenty,  please continue to follow the guidance from their homepage if you want to achieve better inference performance.
+
+> kimi_k2 reasoning parser and other related features have been merged into vLLM/sglang and will be available in the next release. For now, please use the nightly build Docker image.
+## vLLM Deployment
+
+This model is available in nightly vLLM wheel:
+```
+uv pip install -U vllm \
+    --torch-backend=auto \
+    --extra-index-url https://wheels.vllm.ai/nightly
+```
+
+Here is the example to serve this model on a H200 single node with TP8 via vLLM:
+```bash
+vllm serve $MODEL_PATH -tp 8 --mm-encoder-tp-mode data --trust-remote-code --tool-call-parser kimi_k2 --reasoning-parser kimi_k2
+```
+**Key notes**
+- `--tool-call-parser kimi_k2`: Required for enabling tool calling
+- `--reasoning-parser kimi_k2`: Kimi-K2.5 enables thinking mode by default. Make sure to pass this for correct reasoning processing.
+
+## SGLang Deployment
+
+This model is available in SGLang latest main:
+
+```
+pip install "sglang @ git+https://github.com/sgl-project/sglang.git#subdirectory=python"
+pip install nvidia-cudnn-cu12==9.16.0.29
+```
+
+Similarly, here is the example for it to run with TP8 on H200 in a single node via SGLang:
+``` bash
+sglang serve --model-path $MODEL_PATH --tp 8 --trust-remote-code --tool-call-parser kimi_k2 --reasoning-parser kimi_k2
+```
+**Key parameter notes:**
+- `--tool-call-parser kimi_k2`: Required when enabling tool usage.
+- `--reasoning-parser kimi_k2`: Required for correctly processing reasoning content.
+
+## KTransformers Deployment
+### KTransformers+SGLang Inference Deployment
+Launch with KTransformers + SGLang for CPU+GPU heterogeneous inference:
+
+```
+python -m sglang.launch_server \
+  --model path/to/Kimi-K2.5/ \
+  --kt-amx-weight-path path/to/Kimi-K2.5/ \
+  --kt-cpuinfer 64 \
+  --kt-threadpool-count 2 \
+  --kt-num-gpu-experts 180 \
+  --kt-amx-method AMXINT4 \
+  --trust-remote-code \
+  --mem-fraction-static 0.98 \
+  --chunked-prefill-size 16384 \
+  --max-running-requests 48 \
+  --max-total-tokens 50000 \
+  --tensor-parallel-size 8 \
+  --enable-p2p-check \
+  --disable-shared-experts-fusion
+```
+
+Achieves 640.12 tokens/s Prefill and 24.51 tokens/s Decode (48-way concurrency) on 8× NVIDIA L20 + 2× Intel 6454S.
+
+More details: https://github.com/kvcache-ai/ktransformers/blob/main/doc/en/Kimi-K2.5.md .
+
+### KTransformers+LLaMA-Factory Fine-tuning Deployment
+
+You can use below command to run LoRA SFT with KT+llamafactory.
+
+```
+# For LoRA SFT
+USE_KT=1 llamafactory-cli train examples/train_lora/kimik2_lora_sft_kt.yaml
+# For Chat with model after LoRA SFT
+llamafactory-cli chat examples/inference/kimik2_lora_sft_kt.yaml
+# For API with model after LoRA SFT
+llamafactory-cli api examples/inference/kimik2_lora_sft_kt.yaml
+```
+
+This achieves end-to-end LoRA SFT Throughput: 44.55 token/s on 2× NVIDIA 4090 + Intel 8488C with 1.97T RAM and 200G swap memory.
+
+More details refer to https://github.com/kvcache-ai/ktransformers/blob/main/doc/en/SFT_Installation_Guide_KimiK2.5.md .

figures/demo_video.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09b4d925aa0a7c712feef50765355f0625d8f6d46ea302fd98db9609e9070047
+size 270100

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42a2d99bc0c34e386ae682ed749b0ef89e5264d58b9c69e5bfcfdece27595d2d
+size 5365266456

model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdfe4ab0dec82144dd4b1f0225bf86cb3558e20801b1a1e1f699fd5926c483c7
+size 5362966080

model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcb7312dd70da4a4685363a9783a73a97c1603ad7b3e9d3118e7cd44bfc538f2
+size 5365756248

model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acfeed6e455d2fe5eeda2d23f8d143eff89a170b818ae13caaaa3e90a451d5f0
+size 5362966096

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c243fa18c930a5318e9a84e1710011a300d63f207bdc2e05beaefa6a15be4dcd
+size 5365756088

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23b4b78a4734bb16f27716de9299859572a559f4aa294bd8b851bcd7faea2762
+size 3586057136

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b8ef9813f27fb085b5e0a887631630ba3a4260496e049ea70b82dafbedaaa78
+size 2590102248

model.safetensors.index.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9225e4226cd4201ae1d9aaeda7e72a3b06fd0698dd212fb77df9cb6d904376d
+size 758863

quark_profile.yaml

@@ -0,0 +1,78 @@
+# Quark Profiling Results
+
+memory_usage:
+- step: "Start"
+  timestamp: 1775015372.9551535
+  relative_time_secs: 0.0
+  cpu_memory_mb: 3967.14
+  gpu_memory_mb: 774.32
+  disk_read_mb: 0.0
+  disk_write_mb: 0.0
+- step: "File-to-File Quantization Start"
+  timestamp: 1775015373.1001534
+  relative_time_secs: 0.14499998092651367
+  cpu_memory_mb: 3967.14
+  gpu_memory_mb: 774.32
+  disk_read_mb: 0.0
+  disk_write_mb: 0.0
+- step: "File-to-File Quantization Start"
+  timestamp: 1775015373.207735
+  relative_time_secs: 0.2525815963745117
+  cpu_memory_mb: 3967.14
+  gpu_memory_mb: 774.32
+  disk_read_mb: 0.0
+  disk_write_mb: 0.0
+- step: "File-to-File Quantization End"
+  timestamp: 1775015373.3390446
+  relative_time_secs: 0.38389110565185547
+  cpu_memory_mb: 3967.14
+  gpu_memory_mb: 774.32
+  disk_read_mb: 0.0
+  disk_write_mb: 0.0
+- step: "File-to-File Quantization End"
+  timestamp: 1775015373.47948
+  relative_time_secs: 0.5243265628814697
+  cpu_memory_mb: 3967.14
+  gpu_memory_mb: 774.32
+  disk_read_mb: 0.0
+  disk_write_mb: 0.0
+- step: "End"
+  timestamp: 1775015373.6052163
+  relative_time_secs: 0.6500627994537354
+  cpu_memory_mb: 3967.14
+  gpu_memory_mb: 774.32
+  disk_read_mb: 0.0
+  disk_write_mb: 0.0
+
+# Summary Metrics
+total_quantization_time_seconds: 0.6501
+peak_memory_mb: 3967.14
+peak_gpu_memory_mb: 774.32
+total_disk_read_mb: 0.0
+total_disk_write_mb: 0.0
+
+# Metric Definitions:
+#
+# Checkpoint Metrics (per record):
+# - step: Name of the profiling checkpoint. Common steps include:
+#    - "Start": Initial state when profiling begins
+#    - "Model Loaded": After loading the ONNX model into memory
+#    - "Pre-process Start/End": Before and after model preprocessing
+#    - "Calibration Start/End": Before and after calibration data collection
+#    - "Quantization (MatMulNBits) Start/End": MatMulNBits quantization phase
+#    - "Quantization (Static) Start/End": Static quantization phase
+#    - "Post-process Start/End": Before and after post-processing
+#    - "Fast Finetune Start/End": Before and after fast finetuning (if enabled)
+# - timestamp: Unix timestamp (seconds since epoch) when this measurement was taken. Useful for correlating with external logs or events.
+# - relative_time_secs: Time elapsed (in seconds) since the "Start" step. Useful for understanding the duration of each phase relative to the beginning of profiling.
+# - cpu_memory_mb: Current Resident Set Size (RSS) in megabytes at this step. This includes memory from the main process and all child processes. RSS represents the portion of memory held in RAM (not swapped out).
+# - gpu_memory_mb: Current GPU memory usage in megabytes. This represents actual GPU memory used by the process, including allocations from PyTorch, ONNX Runtime, TensorRT, and other frameworks. Only available when PyTorch with CUDA/ROCm is installed and GPU is available.
+# - disk_read_mb: Cumulative disk bytes read (in megabytes) since the start of profiling. Measured relative to the baseline captured at the 'Start' checkpoint, including I/O from the main process and all child processes. Only available when psutil is installed and the OS exposes per-process I/O counters (Linux /proc/<pid>/io, Windows; not available on macOS without root).
+# - disk_write_mb: Cumulative disk bytes written (in megabytes) since the start of profiling. Measured relative to the baseline captured at the 'Start' checkpoint, including I/O from the main process and all child processes. Only available when psutil is installed and the OS exposes per-process I/O counters (Linux /proc/<pid>/io, Windows; not available on macOS without root).
+#
+# Summary Metrics (overall):
+# - total_quantization_time_seconds: Total elapsed time (in seconds) from the start of profiling to the end of the quantization process.
+# - peak_memory_mb: Peak resident set size (RSS) in megabytes for the main process during the entire profiling session. On Linux, this is read from VmHWM (high water mark) in /proc/<pid>/status. On Windows, this is the peak working set size. This metric may not be available on all platforms.
+# - peak_gpu_memory_mb: Peak GPU memory usage in megabytes during the entire profiling session. This is the maximum GPU memory used, including allocations from PyTorch, ONNX Runtime, TensorRT, and other frameworks. Only available when PyTorch with CUDA/ROCm is installed and GPU is available.
+# - total_disk_read_mb: Total disk bytes read (in megabytes) during the entire profiling session. Computed as the difference between the final and baseline cumulative read counters, including I/O from the main process and all child processes. Only available when psutil is installed and the OS exposes per-process I/O counters (Linux /proc/<pid>/io, Windows; not available on macOS without root).
+# - total_disk_write_mb: Total disk bytes written (in megabytes) during the entire profiling session. Computed as the difference between the final and baseline cumulative write counters, including I/O from the main process and all child processes. Only available when psutil is installed and the OS exposes per-process I/O counters (Linux /proc/<pid>/io, Windows; not available on macOS without root).

tokenization_kimi.py

@@ -9,11 +9,7 @@ import tiktoken
 from tiktoken.load import load_tiktoken_bpe
 from tokenizers import AddedToken
 
-try:
-    from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
-except:
-    from transformers.convert_slow_tokenizer import bytes_to_unicode
-
+from transformers.convert_slow_tokenizer import bytes_to_unicode
 from transformers.tokenization_utils import PreTrainedTokenizer
 
 from .tool_declaration_ts import encode_tools_to_typescript_style