Super-squash branch 'main' using huggingface_hub

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.gguf filter=lfs diff=lfs merge=lfs -text

IQ4_XS/Step-3.7-Flash-IQ4_XS-00001-of-00004.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a231fe5ab197cbb320c879f3577eaa819a5576ed9485b3adcadc4ed7aed1897
+size 5227136

IQ4_XS/Step-3.7-Flash-IQ4_XS-00002-of-00004.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1536e96c4d97b43f3041f6d0752bad480ab5b8fe4224cea303610da65b88a8e
+size 49971170976

IQ4_XS/Step-3.7-Flash-IQ4_XS-00003-of-00004.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f62dbebc2568ca3f1bd22e22fb708412bb51244c9d3d23207f9aa70f654dbef
+size 49333737984

IQ4_XS/Step-3.7-Flash-IQ4_XS-00004-of-00004.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e856252a87585693b769a752e744df636f61d8b9347362a0b4c8bc9c65505f8
+size 7424323584

README.md

@@ -0,0 +1,71 @@
+---
+quantized_by: tarruda
+pipeline_tag: text-generation
+base_model: stepfun-ai/Step-3.7-Flash
+base_model_relation: quantized
+license: apache-2.0
+license_link: https://www.apache.org/licenses/LICENSE-2.0.txt
+tags:
+- gguf
+- imatrix
+- conversational
+- multimodal
+- vision
+- llama.cpp
+---
+
+# Step 3.7 Flash GGUF
+
+My custom IQ4_XS GGUF quantization for
+[stepfun-ai/Step-3.7-Flash](https://huggingface.co/stepfun-ai/Step-3.7-Flash)
+
+I've also modified the chat template also adds a `preserve_thinking` option,
+which preserves thinking across user turns and can improve the experience when
+prompt processing speed is a bottleneck.
+
+## Quant Recipes
+
+| Recipe | Quant Size | Default type | Tensor-specific overrides |
+| --- | --- | --- | --- |
+| `IQ4_XS` | 101784.88 MiB (4.34 BPW) | `Q6_K` | `ffn_down_exps=iq4_xs`, `ffn_gate_exps=iq4_xs`, `ffn_up_exps=iq4_xs` |
+
+## Related Files
+
+| File | Description |
+| --- | --- |
+| `Step-3.7-Flash-MTP-Q8_0.gguf` | Q8_0 MTP weights |
+| `Step-3.7-Flash-mmproj-BF16.gguf` | BF16 multimodal projector |
+| `Step-3.7-Flash-mmproj-F16.gguf` | F16 multimodal projector |
+| `Step-3.7-Flash-mmproj-Q8_0.gguf` | Q8_0 multimodal projector |
+
+## Usage
+
+Here's an example script:
+
+```bash
+#!/bin/sh -e
+
+model="./IQ4_XS/Step-3.7-Flash-IQ4_XS-00001-of-00004.gguf"
+mmproj="./Step-3.7-Flash-mmproj-Q8_0.gguf"
+mtp=./Step-3.7-Flash-MTP-Q8_0.gguf
+ 
+ctx=262144
+parallel=1
+
+ctx_size=$((ctx * parallel))
+
+reasoning_budget_message="...
+
+Actually, I will stop now.
+
+Let me provide the user with a comprehensive answer."
+
+llama-server --no-mmap --no-warmup --model $model --mmproj $mmproj \
+  --ctx-size $ctx_size -np $parallel --temp 1.0 --top-p 0.95 \
+  --repeat-penalty 1.0 --presence-penalty 0.0 \
+  --reasoning-budget-message "$reasoning_budget_message" \
+  --reasoning-preserve \
+  --spec-type draft-mtp -md $mtp --spec-draft-n-max 3 --spec-draft-p-min 0.65 \
+  -ctxcp 8 --checkpoint-min-step 512 \
+  --cache-ram 4096
+```

Step-3.7-Flash-MTP-Q8_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0bf32474ec4b491e267b2344022d4a795dbe7647e64a6b349ecedbab46fce79
+size 3707271936

Step-3.7-Flash-mmproj-BF16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:123026a9c9d9221af27b6d798fda36c04721fd9ad0dd81d9f3791a5100b9ce9f
+size 4399308608

Step-3.7-Flash-mmproj-F16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1695d4541d727a67604ea8f2070b383fff0767e49abe76b4a51632f3b2a52c9
+size 3972828992

Step-3.7-Flash-mmproj-Q8_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fb4b625241216272b388fd13ab5afc7409d3728481d8efc5c2841c25ac006d3
+size 2747064128

chat_template.jinja

@@ -0,0 +1,91 @@
+{% macro render_message_content(message) %}{% if message.content is none %}{{- '' }}{% elif message.content is string %}{{- message.content }}{% elif message.content is mapping %}{{- message.content['value'] if 'value' in message.content else message.content['text'] }}{% elif message.content is iterable %}{% set ns = namespace(needs_text_separator=false) %}{% for item in message.content %}{% if item.type == 'text' %}{% if ns.needs_text_separator %}{{- ' ' }}{% endif %}{{- item['value'] if 'value' in item else item['text'] }}{% set ns.needs_text_separator = true %}{% elif item.type == 'image' %}<im_patch>{% set ns.needs_text_separator = false %}{% endif %}{% endfor %}{% endif %}{% endmacro %}
+{{bos_token}}{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if reasoning_effort is defined %}
+        {{- "Reasoning: " + reasoning_effort + '\n\n' }}
+    {%- endif %}
+    {%- if messages[0].role == 'system' %}
+        {{- render_message_content(messages[0]) + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou have access to the following functions in JSONSchema format:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson(ensure_ascii=False) }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...>\n...\n</function> block must be nested within <tool_call>\n...\n</tool_call> XML tags\n- Required parameters MUST be specified\n</IMPORTANT><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' }}
+        {%- if reasoning_effort is defined %}
+            {{- "Reasoning: " + reasoning_effort + '\n\n' }}
+        {%- endif %}
+        {{- render_message_content(messages[0]) + '<|im_end|>\n' }}
+    {%- elif reasoning_effort is defined %}
+        {{- '<|im_start|>system\n' + "Reasoning: " + reasoning_effort + '\n\n' + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and render_message_content(message) is string and not(render_message_content(message).startswith('<tool_response>') and render_message_content(message).endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_message_content(message) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {%- set role_name = 'observation' if (message.role == "system" and not loop.first and message.name == 'observation') else message.role %}
+        {{- '<|im_start|>' + role_name + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- else %}
+                {%- set reasoning_content = '' %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- set content = content|trim %}
+        {%- if (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- if tool_call.arguments is defined %}
+                    {%- set arguments = tool_call.arguments | fromjson if tool_call.arguments is string else tool_call.arguments %}
+                    {%- for args_name, args_value in arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson(ensure_ascii=False) | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>tool_response\n' }}
+        {%- endif %}
+        {{- '<tool_response>' }}
+        {{- content }}
+        {{- '</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n<think>\n' }}
+{%- endif %}

imatrix.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f94ca213e4560d30b492b332128527c6808041ec3526df6c2816884eb107203
+size 465998112

scripts/convert-to-gguf.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Validate that exactly 2 arguments are provided
+if [ $# -ne 2 ]; then
+  echo "Error: Exactly 2 arguments required."
+  echo "Usage: $0 <llama_cpp_dir> <original_safetensors_path>"
+  echo "Example: $0 ~/code/llama.cpp ../../Qwen/Qwen3.5-397B-A10B"
+  exit 1
+fi
+
+# Assign arguments to variables for clarity
+LLAMA_CPP_DIR="$1"
+ORIGINAL_SAFETENSORS_PATH="$2"
+CONVERSION_MODEL_PATH="$ORIGINAL_SAFETENSORS_PATH"
+
+# Validate that the llama.cpp directory exists
+if [ ! -d "$LLAMA_CPP_DIR" ]; then
+  echo "Error: llama.cpp directory not found: $LLAMA_CPP_DIR"
+  exit 1
+fi
+
+# Construct the path to the conversion script
+CONVERT_TO_GGUF_PATH="$LLAMA_CPP_DIR/convert_hf_to_gguf.py"
+
+# Validate that the Python script exists
+if [ ! -f "$CONVERT_TO_GGUF_PATH" ]; then
+  echo "Error: Python script not found: $CONVERT_TO_GGUF_PATH"
+  exit 1
+fi
+
+# Validate that the model path exists
+if [ ! -e "$ORIGINAL_SAFETENSORS_PATH" ]; then
+  echo "Error: Model path not found: $ORIGINAL_SAFETENSORS_PATH"
+  PARENT_DIR="$(dirname "$ORIGINAL_SAFETENSORS_PATH")"
+  BASE_NAME="$(basename "$ORIGINAL_SAFETENSORS_PATH")"
+  if [ -d "$PARENT_DIR" ]; then
+    CANDIDATE="$(find "$PARENT_DIR" -maxdepth 1 -type d -name "${BASE_NAME}*" -print -quit 2>/dev/null || true)"
+    if [ -n "$CANDIDATE" ]; then
+      echo "Did you mean: $CANDIDATE"
+    fi
+  fi
+  exit 1
+fi
+
+ORIGINAL_SAFETENSORS_PATH="$(cd "$(dirname "$ORIGINAL_SAFETENSORS_PATH")" && pwd)/$(basename "$ORIGINAL_SAFETENSORS_PATH")"
+CONVERSION_MODEL_PATH="$ORIGINAL_SAFETENSORS_PATH"
+MODEL_NAME="$(basename "$ORIGINAL_SAFETENSORS_PATH")"
+# Get the directory where the script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CONVERTER_SHIM="$SCRIPT_DIR/convert_hf_to_gguf_step37.py"
+if [ ! -f "$CONVERTER_SHIM" ]; then
+  echo "Error: converter shim not found: $CONVERTER_SHIM"
+  exit 1
+fi
+
+# Resolve GGUF_OUT_DIR absolutely (parent of script directory)
+# We cd into the parent and run pwd to get the absolute path
+if [ ! -d "$SCRIPT_DIR/.." ]; then
+  echo "Error: Could not resolve parent directory for output."
+  exit 1
+fi
+GGUF_OUT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+OVERLAY_PARENT="$GGUF_OUT_DIR/.conversion-model"
+OVERLAY_MODEL_PATH="$OVERLAY_PARENT/$MODEL_NAME"
+
+# Define BF16_DIR inside GGUF_OUT_DIR
+BF16_DIR="$GGUF_OUT_DIR/BF16"
+mkdir -pv "$BF16_DIR"
+
+# Activate virtual environment inside llama.cpp directory
+if [ ! -f "$LLAMA_CPP_DIR/.venv/bin/activate" ]; then
+  echo "Error: Python virtual environment not found: $LLAMA_CPP_DIR/.venv"
+  exit 1
+fi
+cd "$LLAMA_CPP_DIR"
+source .venv/bin/activate
+PYTHON_BIN="$(command -v python)"
+cd - >/dev/null
+
+run_converter() {
+  "$PYTHON_BIN" "$CONVERTER_SHIM" "$CONVERT_TO_GGUF_PATH" "$@"
+}
+
+# Execute the Python script with the provided arguments
+# Main conversion output goes to BF16_DIR
+
+run_converter --no-mtp --no-tensor-first-split --split-max-size 50G --outfile "$BF16_DIR/${MODEL_NAME}-BF16.gguf" "$CONVERSION_MODEL_PATH"
+
+run_converter --mtp --outfile "$GGUF_OUT_DIR/${MODEL_NAME}-MTP-Q8_0.gguf" --outtype q8_0 "$ORIGINAL_SAFETENSORS_PATH"
+
+# # mmproj output goes to GGUF_OUT_DIR
+run_converter --mmproj --outtype bf16 --outfile "$GGUF_OUT_DIR/${MODEL_NAME}-mmproj-BF16.gguf" "$CONVERSION_MODEL_PATH"
+# # also do FP16 and Q8_0 versions
+run_converter --mmproj --outtype f16 --outfile "$GGUF_OUT_DIR/${MODEL_NAME}-mmproj-F16.gguf" "$CONVERSION_MODEL_PATH"
+run_converter --mmproj --outtype q8_0 --outfile "$GGUF_OUT_DIR/${MODEL_NAME}-mmproj-Q8_0.gguf" "$CONVERSION_MODEL_PATH"
+
+rm -rf "$OVERLAY_PARENT"

scripts/convert_hf_to_gguf_step37.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import ast
+import importlib.util
+import inspect
+import sys
+import textwrap
+from hashlib import sha256
+from pathlib import Path
+from types import ModuleType
+from typing import Any, Callable
+
+
+STEP37_TOKENIZER_HASH = "5841594bd6a8eeecd7207aeec6570831cc97ffaeba51e908bdaf560113177bae"
+STEP37_TOKENIZER_PRE = "deepseek-v3"
+
+
+def load_module(path: Path) -> ModuleType:
+    spec = importlib.util.spec_from_file_location("_llama_convert_hf_to_gguf", path)
+    if spec is None or spec.loader is None:
+        raise RuntimeError(f"Could not load converter module from {path}")
+
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def checker_text(original: Callable[..., str]) -> str | None:
+    try:
+        source = textwrap.dedent(inspect.getsource(original))
+        tree = ast.parse(source)
+    except Exception:
+        return None
+
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Assign):
+            continue
+        if any(isinstance(target, ast.Name) and target.id == "chktxt" for target in node.targets):
+            try:
+                return ast.literal_eval(node.value)
+            except Exception:
+                return None
+
+    return None
+
+
+def patch_step37_tokenizer() -> None:
+    from conversion import base
+
+    original = base.TextModel.get_vocab_base_pre
+    chktxt = checker_text(original)
+
+    def patched_get_vocab_base_pre(self: Any, tokenizer: Any) -> str:
+        if chktxt is not None:
+            try:
+                chkhsh = sha256(str(tokenizer.encode(chktxt)).encode()).hexdigest()
+            except Exception:
+                chkhsh = None
+
+            if chkhsh == STEP37_TOKENIZER_HASH:
+                return STEP37_TOKENIZER_PRE
+
+        return original(self, tokenizer)
+
+    base.TextModel.get_vocab_base_pre = patched_get_vocab_base_pre
+
+
+def main() -> None:
+    if len(sys.argv) < 2:
+        raise SystemExit("Usage: convert_hf_to_gguf_step37.py <convert_hf_to_gguf.py> [converter args...]")
+
+    convert_path = Path(sys.argv[1]).expanduser().resolve()
+    if not convert_path.is_file():
+        raise SystemExit(f"Converter not found: {convert_path}")
+
+    sys.path.insert(0, str(convert_path.parent))
+    module = load_module(convert_path)
+    patch_step37_tokenizer()
+
+    sys.argv = [str(convert_path), *sys.argv[2:]]
+    module.main()
+
+
+if __name__ == "__main__":
+    main()

scripts/generate-imatrix.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Validate that exactly 2 arguments are provided
+if [ $# -ne 2 ]; then
+  echo "Error: Exactly 2 arguments required."
+  echo "Usage: $0 <llama_cpp_dir> <gguf>"
+  echo "Example: $0 ~/code/llama.cpp BF16/Qwen3.5-35B-A3B-BF16-00001-of-00003.gguf"
+  exit 1
+fi
+
+# Assign arguments to variables for clarity
+LLAMA_CPP_DIR="$1"
+GGUF_PATH="$2"
+
+# Validate that the llama.cpp directory exists
+if [ ! -d "$LLAMA_CPP_DIR" ]; then
+  echo "Error: llama.cpp directory not found: $LLAMA_CPP_DIR"
+  exit 1
+fi
+
+
+# Validate that the Python script exists
+if [ ! -e "$GGUF_PATH" ]; then
+  echo "Error: GGUF not found: $GGUF_PATH"
+  exit 1
+fi
+
+CALIBRATION_DATASET_PATH=$HOME/calibration-data.txt
+
+if [ ! -e $CALIBRATION_DATASET_PATH ]; then
+  # download the calibration dataset
+  curl -L https://gist.githubusercontent.com/ubergarm/edfeb3ff9c6ec8b49e88cdf627b0711a/raw/ba5b01b6960a86874592f5913e283746ff734483/ubergarm-imatrix-calibration-corpus-v02.txt > $CALIBRATION_DATASET_PATH
+fi
+
+# Get the directory where the script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+IMATRIX_OUT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+IMATRIX_PATH="$IMATRIX_OUT_DIR/imatrix.gguf"
+
+if [ -e "$IMATRIX_PATH" ]; then
+  echo "Error: imatrix already exists: $IMATRIX_PATH"
+  exit 1
+fi
+
+$LLAMA_CPP_DIR/build/bin/llama-imatrix \
+  -fit off \
+  --model "$GGUF_PATH"\
+  -f $CALIBRATION_DATASET_PATH \
+  -o $IMATRIX_PATH \
+  --ctx-size 512 \
+  -ub 4096 -b 4096 \
+  --no-mmap

scripts/quantize.sh

@@ -0,0 +1,285 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+shopt -s extglob
+
+common_tensor_types="
+  "
+
+TYPE_DEFAULT_FALLBACK=Q4_K
+
+recipes=(
+  "
+  RECIPE_NAME=IQ4_XS
+  TYPE_DEFAULT=q6_k
+
+  ffn_down_exps=iq4_xs
+  ffn_gate_exps=iq4_xs
+  ffn_up_exps=iq4_xs
+  "
+
+)
+
+recipe_line() {
+  local line="$1"
+  printf '%s' "${line##+([[:space:]])}"
+}
+
+recipe_value() {
+  local recipe="$1"
+  local key="$2"
+  local line
+
+  while IFS= read -r line || [ -n "$line" ]; do
+    line="$(recipe_line "$line")"
+    case "$line" in
+      "${key}="*)
+        printf '%s' "${line#*=}"
+        return 0
+        ;;
+    esac
+  done <<<"$recipe"
+
+  return 1
+}
+
+recipe_type_default() {
+  local recipe="$1"
+
+  recipe_value "$recipe" TYPE_DEFAULT || printf '%s' "$TYPE_DEFAULT_FALLBACK"
+}
+
+print_recipes() {
+  local recipe recipe_name type_default
+
+  for recipe in "${recipes[@]}"; do
+    recipe_name="$(recipe_value "$recipe" RECIPE_NAME)"
+    type_default="$(recipe_type_default "$recipe")"
+
+    printf '  %s:\n' "$recipe_name"
+    printf '    TYPE_DEFAULT=%s\n' "$type_default"
+    printf '\n'
+  done
+}
+
+SELECTED_RECIPE_NAME=
+SELECTED_TYPE_DEFAULT=
+SELECTED_RECIPE=
+
+load_recipe() {
+  local requested_recipe="$1"
+  local recipe recipe_name
+
+  for recipe in "${recipes[@]}"; do
+    recipe_name="$(recipe_value "$recipe" RECIPE_NAME)"
+    if [ "$recipe_name" = "$requested_recipe" ]; then
+      SELECTED_RECIPE_NAME="$recipe_name"
+      SELECTED_TYPE_DEFAULT="$(recipe_type_default "$recipe")"
+      SELECTED_RECIPE="$recipe"
+      return 0
+    fi
+  done
+
+  return 1
+}
+
+append_tensor_type_lines() {
+  local block="$1"
+  local output_path="$2"
+  local skip_recipe_metadata="${3:-false}"
+  local line
+
+  while IFS= read -r line || [ -n "$line" ]; do
+    line="$(recipe_line "$line")"
+    if [ "$skip_recipe_metadata" = true ]; then
+      case "$line" in
+        RECIPE_NAME=* | TYPE_DEFAULT=*)
+          continue
+          ;;
+      esac
+    fi
+
+    if [ -z "$line" ]; then
+      continue
+    fi
+
+    printf '%s\n' "$line" >>"$output_path"
+  done <<<"$block"
+}
+
+write_tensor_type_file() {
+  local recipe="$1"
+  local output_path="$2"
+
+  : >"$output_path"
+  append_tensor_type_lines "$common_tensor_types" "$output_path"
+  append_tensor_type_lines "$recipe" "$output_path" true
+}
+
+usage() {
+  cat <<'EOF'
+Usage: quantize.sh <llama_cpp_dir> <recipe_name> [--dry-run] [--split]
+
+Outputs are written relative to the current directory.
+
+Environment overrides:
+  INPUT_GGUF=<path>     Input GGUF. Defaults to the first *BF16*.gguf in BF16/.
+  IMATRIX_PATH=<path>   Importance matrix path. Defaults to imatrix.gguf.
+
+Available recipes:
+EOF
+  print_recipes
+  cat <<'EOF'
+Examples:
+  ./scripts/quantize.sh ~/code/llama.cpp IQ3_XXS --dry-run
+  ./scripts/quantize.sh ~/code/llama.cpp IQ3_XXS
+  ./scripts/quantize.sh ~/code/llama.cpp IQ3_XXS --split
+EOF
+}
+
+if [ $# -lt 2 ]; then
+  usage
+  exit 1
+fi
+
+LLAMA_CPP_DIR="$1"
+REQUESTED_RECIPE="$2"
+DRY_RUN=false
+SPLIT=false
+shift 2
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --dry-run)
+      DRY_RUN=true
+      ;;
+    --split)
+      SPLIT=true
+      ;;
+    *)
+      usage
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+BF16_DIR="$PROJECT_DIR/BF16"
+SPLIT_SCRIPT="$SCRIPT_DIR/split.sh"
+
+if ! load_recipe "$REQUESTED_RECIPE"; then
+  echo "Error: unknown recipe: $REQUESTED_RECIPE" >&2
+  usage >&2
+  exit 1
+fi
+
+if [ ! -d "$LLAMA_CPP_DIR" ]; then
+  echo "Error: llama.cpp directory not found: $LLAMA_CPP_DIR" >&2
+  exit 1
+fi
+
+QUANTIZE_BIN="$LLAMA_CPP_DIR/build/bin/llama-quantize"
+SPLIT_BIN="$LLAMA_CPP_DIR/build/bin/llama-gguf-split"
+
+if [ ! -x "$QUANTIZE_BIN" ]; then
+  echo "Error: llama-quantize binary not found: $QUANTIZE_BIN" >&2
+  exit 1
+fi
+
+if [ "$SPLIT" = true ] && [ ! -x "$SPLIT_SCRIPT" ]; then
+  echo "Error: split script not found or not executable: $SPLIT_SCRIPT" >&2
+  exit 1
+fi
+
+if [ "$SPLIT" = true ] && [ ! -x "$SPLIT_BIN" ]; then
+  echo "Error: llama-gguf-split binary not found: $SPLIT_BIN" >&2
+  exit 1
+fi
+
+if [ -z "${INPUT_GGUF-}" ]; then
+  if [ ! -d "$BF16_DIR" ]; then
+    echo "Error: BF16 directory not found: $BF16_DIR" >&2
+    exit 1
+  fi
+
+  INPUT_GGUF="$(find "$BF16_DIR" -maxdepth 1 -name "*BF16*.gguf" -type f | sort | head -n 1)"
+fi
+
+if [ -z "$INPUT_GGUF" ] || [ ! -e "$INPUT_GGUF" ]; then
+  echo "Error: input GGUF not found: ${INPUT_GGUF:-<none>}" >&2
+  exit 1
+fi
+
+IMATRIX_PATH="${IMATRIX_PATH:-$PROJECT_DIR/imatrix.gguf}"
+if [ ! -e "$IMATRIX_PATH" ]; then
+  echo "Error: imatrix file not found: $IMATRIX_PATH" >&2
+  exit 1
+fi
+
+cd "$LLAMA_CPP_DIR"
+if [ -f .venv/bin/activate ]; then
+  # shellcheck disable=SC1091
+  source .venv/bin/activate
+fi
+cd - >/dev/null
+
+OUTPUT_BASE_DIR="$(pwd)"
+INPUT_BASENAME="$(basename "$INPUT_GGUF")"
+MODEL_NAME="$(printf '%s\n' "$INPUT_BASENAME" | sed -E 's/-BF16(-[0-9]+-of-[0-9]+)?\.gguf$//')"
+INTERMEDIATE_OUTPUT="$OUTPUT_BASE_DIR/${MODEL_NAME}-${SELECTED_RECIPE_NAME}.gguf"
+OUTPUT_DIR="$OUTPUT_BASE_DIR/$SELECTED_RECIPE_NAME"
+
+if [ "$DRY_RUN" = false ] && [ -e "$INTERMEDIATE_OUTPUT" ]; then
+  echo "Error: quantized output already exists: $INTERMEDIATE_OUTPUT" >&2
+  exit 1
+fi
+
+TENSOR_TYPE_FILE="$(mktemp "${TMPDIR:-/tmp}/nex-n2-pro-tensor-types.XXXXXX")"
+cleanup() {
+  rm -f "$TENSOR_TYPE_FILE"
+}
+trap cleanup EXIT
+
+write_tensor_type_file "$SELECTED_RECIPE" "$TENSOR_TYPE_FILE"
+
+echo "Input: $INPUT_GGUF"
+echo "Recipe name: $SELECTED_RECIPE_NAME"
+echo "Tensor type file: $TENSOR_TYPE_FILE"
+echo "Default type: $SELECTED_TYPE_DEFAULT"
+echo "imatrix: $IMATRIX_PATH"
+echo "Output base: $OUTPUT_BASE_DIR"
+echo "Split output: $SPLIT"
+
+if [ "$DRY_RUN" = true ]; then
+  "$QUANTIZE_BIN" \
+    --dry-run \
+    --allow-requantize \
+    --tensor-type-file "$TENSOR_TYPE_FILE" \
+    --imatrix "$IMATRIX_PATH" \
+    "$INPUT_GGUF" \
+    "$INTERMEDIATE_OUTPUT" \
+    "$SELECTED_TYPE_DEFAULT"
+else
+  "$QUANTIZE_BIN" \
+    --allow-requantize \
+    --tensor-type-file "$TENSOR_TYPE_FILE" \
+    --imatrix "$IMATRIX_PATH" \
+    "$INPUT_GGUF" \
+    "$INTERMEDIATE_OUTPUT" \
+    "$SELECTED_TYPE_DEFAULT"
+fi
+
+if [ "$DRY_RUN" = false ]; then
+  if [ "$SPLIT" = true ]; then
+    "$SPLIT_SCRIPT" \
+      "$LLAMA_CPP_DIR" \
+      "$INTERMEDIATE_OUTPUT" \
+      "$OUTPUT_DIR" \
+      "${MODEL_NAME}-${SELECTED_RECIPE_NAME}" \
+      --remove-input
+  else
+    echo "Quantization complete. Output saved to: $INTERMEDIATE_OUTPUT"
+  fi
+fi

scripts/report-imatrix-activations.py

@@ -0,0 +1,661 @@
+#!/usr/bin/env python3
+"""Report FFN activation importance from a llama.cpp imatrix GGUF.
+
+The script ranks FFN down and gate/up activation streams from imatrix tensors.
+For MoE tensors, it also tries to infer the routed expert count and report the
+highest-activation block/expert pairs.
+"""
+
+from __future__ import annotations
+
+import argparse
+import math
+import re
+import sys
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+import numpy as np
+
+try:
+    from gguf import GGUFReader
+except ImportError as exc:  # pragma: no cover - exercised by users without gguf.
+    raise SystemExit(
+        "Could not import gguf. Run this with a Python environment that has "
+        "llama.cpp's gguf package installed, for example:\n"
+        "  ~/code/llama.cpp/.venv/bin/python scripts/report-imatrix-activations.py imatrix.gguf"
+    ) from exc
+
+
+BLOCK_PATTERNS = (
+    re.compile(r"^blk\.(\d+)\.(.+)\.weight$"),
+    re.compile(r"^layers\.(\d+)\.(.+)\.weight$"),
+    re.compile(r"^model\.layers\.(\d+)\.(.+)\.weight$"),
+)
+
+
+@dataclass(frozen=True)
+class Candidate:
+    base: str
+    tensor_name: str
+    count_name: str
+    block: int | None
+    module: str
+    kind: str
+    projection: str
+    family: str
+    n: int
+
+
+@dataclass
+class Row:
+    block: int | None
+    family: str
+    kind: str
+    projection: str
+    tensor: str
+    count: float
+    n: int
+    mean_mse: float
+    rms: float
+    p95: float
+    p99: float
+    max_value: float
+    gt1: float
+    gt4: float
+
+
+@dataclass
+class ExpertRow:
+    block: int
+    expert: int
+    family: str
+    tensor: str
+    count: float
+    n: int
+    mean_mse: float
+    rms: float
+    p95: float
+    p99: float
+    max_value: float
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Rank FFN down and gate/up activation streams from an imatrix GGUF.",
+    )
+    parser.add_argument("imatrix", type=Path, help="Path to an imatrix .gguf file")
+    parser.add_argument(
+        "--top",
+        type=int,
+        default=15,
+        help="Rows to show in the top tensor-stream table. Default: 15",
+    )
+    parser.add_argument(
+        "--family-top",
+        type=int,
+        default=5,
+        help="Rows to show per tensor family. Default: 5",
+    )
+    parser.add_argument(
+        "--top-experts",
+        type=int,
+        default=25,
+        help="Rows to show in the combined routed expert table. Default: 25",
+    )
+    parser.add_argument(
+        "--top-expert-streams",
+        type=int,
+        default=15,
+        help="Rows to show in the per-stream routed expert table. Default: 15",
+    )
+    parser.add_argument(
+        "--last-blocks",
+        type=int,
+        default=10,
+        help="Number of final MoE blocks to show in the compact table. Default: 10",
+    )
+    parser.add_argument(
+        "--experts",
+        type=int,
+        default=None,
+        help="Override routed expert count. Useful when it cannot be inferred.",
+    )
+    parser.add_argument(
+        "--no-experts",
+        action="store_true",
+        help="Skip per-expert routed rankings.",
+    )
+    parser.add_argument(
+        "--include-router-input",
+        action="store_true",
+        help="Include ffn_gate_inp/router input tensors when present.",
+    )
+    return parser.parse_args()
+
+
+def field_type_name(field: object, index: int) -> str:
+    try:
+        return field.types[index].name
+    except Exception:
+        return ""
+
+
+def part_to_list(part: object) -> object:
+    return part.tolist() if hasattr(part, "tolist") else part
+
+
+def decode_bytes(values: object) -> str:
+    if isinstance(values, list):
+        return bytes(values).decode("utf-8", errors="replace")
+    if isinstance(values, np.ndarray):
+        return bytes(values.tolist()).decode("utf-8", errors="replace")
+    if isinstance(values, bytes):
+        return values.decode("utf-8", errors="replace")
+    return str(values)
+
+
+def decode_field(field: object) -> object:
+    """Best-effort conversion of a GGUF ReaderField into a printable value."""
+    if not getattr(field, "parts", None):
+        return ""
+
+    if field_type_name(field, 0) == "STRING":
+        return decode_bytes(part_to_list(field.parts[-1]))
+
+    if field_type_name(field, 0) == "ARRAY" and field_type_name(field, 1) == "STRING":
+        # llama.cpp imatrix files usually store a single dataset string.
+        return decode_bytes(part_to_list(field.parts[-1]))
+
+    value = part_to_list(field.parts[-1])
+    if isinstance(value, list) and len(value) == 1:
+        return value[0]
+    return value
+
+
+def parse_block_module(base: str) -> tuple[int | None, str]:
+    for pattern in BLOCK_PATTERNS:
+        match = pattern.match(base)
+        if match:
+            return int(match.group(1)), match.group(2)
+    if base.endswith(".weight"):
+        return None, base[: -len(".weight")]
+    return None, base
+
+
+def classify_module(module: str, include_router_input: bool = False) -> tuple[str, str, str] | None:
+    """Return (kind, projection, family) for FFN modules that this report handles."""
+    if module == "ffn_gate_inp" or module.endswith(".ffn_gate_inp"):
+        if not include_router_input:
+            return None
+        return "router", "gate input", "router gate input"
+
+    if "ffn_down" in module:
+        projection = "down"
+    elif "ffn_gate" in module or "ffn_up" in module:
+        projection = "gate/up"
+    else:
+        return None
+
+    if "_exps" in module or ".experts." in module or "_experts" in module:
+        kind = "routed expert"
+    elif "_shexp" in module or "shared_expert" in module or "shared_experts" in module:
+        kind = "shared expert"
+    else:
+        kind = "dense"
+
+    return kind, projection, f"{kind} {projection}"
+
+
+def collect_lengths(reader: GGUFReader) -> dict[tuple[int | None, str], int]:
+    lengths: dict[tuple[int | None, str], int] = {}
+    for tensor in reader.tensors:
+        if not tensor.name.endswith(".in_sum2"):
+            continue
+        base = tensor.name[: -len(".in_sum2")]
+        block, module = parse_block_module(base)
+        lengths[(block, module)] = int(np.asarray(tensor.data).size)
+    return lengths
+
+
+def collect_candidates(reader: GGUFReader, include_router_input: bool) -> list[Candidate]:
+    tensor_names = {tensor.name for tensor in reader.tensors}
+    candidates: list[Candidate] = []
+
+    for tensor in reader.tensors:
+        if not tensor.name.endswith(".in_sum2"):
+            continue
+
+        base = tensor.name[: -len(".in_sum2")]
+        count_name = base + ".counts"
+        if count_name not in tensor_names:
+            continue
+
+        block, module = parse_block_module(base)
+        classified = classify_module(module, include_router_input=include_router_input)
+        if classified is None:
+            continue
+
+        kind, projection, family = classified
+        candidates.append(
+            Candidate(
+                base=base,
+                tensor_name=tensor.name,
+                count_name=count_name,
+                block=block,
+                module=module,
+                kind=kind,
+                projection=projection,
+                family=family,
+                n=int(np.asarray(tensor.data).size),
+            )
+        )
+
+    return candidates
+
+
+def infer_expert_count(candidates: Iterable[Candidate], lengths: dict[tuple[int | None, str], int]) -> tuple[int | None, Counter[int]]:
+    candidates = list(candidates)
+    gate_ref_modules = {
+        "attn_gate",
+        "attn_k",
+        "attn_q",
+        "attn_v",
+        "ffn_gate",
+        "ffn_up",
+        "ffn_gate_shexp",
+        "ffn_up_shexp",
+    }
+    down_ref_modules = {
+        "ffn_down",
+        "ffn_down_shexp",
+    }
+
+    gate_refs: dict[int | None, set[int]] = defaultdict(set)
+    down_refs: dict[int | None, set[int]] = defaultdict(set)
+    for (block, module), size in lengths.items():
+        if module in gate_ref_modules:
+            gate_refs[block].add(size)
+        if module in down_ref_modules:
+            down_refs[block].add(size)
+
+    # Prefer dimensions discovered from the FFN candidates themselves. This
+    # keeps inference useful for MoE GGUFs that have shared experts, dense
+    # leading layers, or non-standard attention tensor availability.
+    for candidate in candidates:
+        if candidate.block is None or candidate.kind == "routed expert":
+            continue
+        if candidate.projection == "gate/up":
+            gate_refs[candidate.block].add(candidate.n)
+        elif candidate.projection == "down":
+            down_refs[candidate.block].add(candidate.n)
+
+    votes: Counter[int] = Counter()
+    for candidate in candidates:
+        if candidate.kind != "routed expert" or candidate.block is None:
+            continue
+        refs = gate_refs[candidate.block] if candidate.projection == "gate/up" else down_refs[candidate.block]
+        for ref in refs:
+            if ref <= 0 or candidate.n % ref != 0:
+                continue
+            expert_count = candidate.n // ref
+            if 1 < expert_count <= 4096:
+                votes[expert_count] += 1
+
+    if not votes:
+        return None, votes
+    return votes.most_common(1)[0][0], votes
+
+
+def deduplicate_gate_up(candidates: Iterable[Candidate]) -> list[Candidate]:
+    """Keep one tensor for each shared gate/up input stream."""
+    by_key: dict[tuple[int | None, str], Candidate] = {}
+    for candidate in candidates:
+        key = (candidate.block, candidate.family)
+        current = by_key.get(key)
+        if current is None:
+            by_key[key] = candidate
+            continue
+
+        # Prefer gate over up because it is the conventional representative.
+        if "ffn_gate" in candidate.module and "ffn_up" in current.module:
+            by_key[key] = candidate
+
+    return list(by_key.values())
+
+
+def stats(values: np.ndarray) -> tuple[float, float, float, float, float, float, float]:
+    mean_mse = float(np.mean(values))
+    rms = math.sqrt(mean_mse) if mean_mse >= 0 and math.isfinite(mean_mse) else float("nan")
+    return (
+        mean_mse,
+        rms,
+        float(np.percentile(values, 95)),
+        float(np.percentile(values, 99)),
+        float(np.max(values)),
+        float(np.mean(values > 1.0)),
+        float(np.mean(values > 4.0)),
+    )
+
+
+def analyze(reader: GGUFReader, args: argparse.Namespace) -> tuple[list[Row], list[ExpertRow], int | None, Counter[int]]:
+    tensor_by_name = {tensor.name: tensor for tensor in reader.tensors}
+    lengths = collect_lengths(reader)
+    candidates = collect_candidates(reader, include_router_input=args.include_router_input)
+    inferred_experts, expert_votes = infer_expert_count(candidates, lengths)
+    expert_count = args.experts if args.experts is not None else inferred_experts
+    candidates = deduplicate_gate_up(candidates)
+
+    rows: list[Row] = []
+    expert_rows: list[ExpertRow] = []
+
+    for candidate in candidates:
+        count = float(np.asarray(tensor_by_name[candidate.count_name].data).reshape(-1)[0])
+        raw = np.asarray(tensor_by_name[candidate.tensor_name].data, dtype=np.float64)
+        values = raw / count if count else np.full_like(raw, np.nan, dtype=np.float64)
+        mean_mse, rms, p95, p99, max_value, gt1, gt4 = stats(values)
+        rows.append(
+            Row(
+                block=candidate.block,
+                family=candidate.family,
+                kind=candidate.kind,
+                projection=candidate.projection,
+                tensor=candidate.base,
+                count=count,
+                n=candidate.n,
+                mean_mse=mean_mse,
+                rms=rms,
+                p95=p95,
+                p99=p99,
+                max_value=max_value,
+                gt1=gt1,
+                gt4=gt4,
+            )
+        )
+
+        if (
+            not args.no_experts
+            and expert_count
+            and candidate.kind == "routed expert"
+            and candidate.block is not None
+            and candidate.n % expert_count == 0
+        ):
+            per_expert = values.reshape(expert_count, candidate.n // expert_count)
+            for expert_id, expert_values in enumerate(per_expert):
+                exp_mean, exp_rms, exp_p95, exp_p99, exp_max, _, _ = stats(expert_values)
+                expert_rows.append(
+                    ExpertRow(
+                        block=candidate.block,
+                        expert=expert_id,
+                        family=candidate.family,
+                        tensor=candidate.base,
+                        count=count,
+                        n=int(expert_values.size),
+                        mean_mse=exp_mean,
+                        rms=exp_rms,
+                        p95=exp_p95,
+                        p99=exp_p99,
+                        max_value=exp_max,
+                    )
+                )
+
+    return rows, expert_rows, expert_count, expert_votes
+
+
+def fmt(value: object) -> str:
+    if isinstance(value, float):
+        if math.isnan(value):
+            return ""
+        return f"{value:.4g}"
+    if isinstance(value, int):
+        return str(value)
+    return str(value)
+
+
+def pct(value: float) -> str:
+    return f"{100.0 * value:.1f}%"
+
+
+def block_label(block: int | None) -> str:
+    return "?" if block is None else str(block)
+
+
+def print_table(headers: list[str], rows: Iterable[list[object]]) -> None:
+    print("| " + " | ".join(headers) + " |")
+    print("| " + " | ".join("---" for _ in headers) + " |")
+    for row in rows:
+        print("| " + " | ".join(fmt(value) for value in row) + " |")
+
+
+def metadata(reader: GGUFReader) -> dict[str, object]:
+    wanted = ("general.type", "imatrix.datasets", "imatrix.chunk_count", "imatrix.chunk_size")
+    return {key: decode_field(reader.fields[key]) for key in wanted if key in reader.fields}
+
+
+def aggregate_experts(expert_rows: Iterable[ExpertRow]) -> list[dict[str, object]]:
+    acc: dict[tuple[int, int], dict[str, object]] = {}
+    for row in expert_rows:
+        key = (row.block, row.expert)
+        item = acc.setdefault(
+            key,
+            {
+                "block": row.block,
+                "expert": row.expert,
+                "weighted_mse_sum": 0.0,
+                "n": 0,
+                "max_stream_rms": 0.0,
+                "streams": [],
+            },
+        )
+        item["weighted_mse_sum"] = float(item["weighted_mse_sum"]) + row.mean_mse * row.n
+        item["n"] = int(item["n"]) + row.n
+        item["max_stream_rms"] = max(float(item["max_stream_rms"]), row.rms)
+        item["streams"].append(row.family)
+
+    result: list[dict[str, object]] = []
+    for item in acc.values():
+        mean_mse = float(item["weighted_mse_sum"]) / int(item["n"])
+        result.append(
+            {
+                "block": item["block"],
+                "expert": item["expert"],
+                "rms": math.sqrt(mean_mse),
+                "mean_mse": mean_mse,
+                "max_stream_rms": item["max_stream_rms"],
+                "streams": ", ".join(sorted(set(item["streams"]))),
+            }
+        )
+    return result
+
+
+def render(path: Path, reader: GGUFReader, rows: list[Row], expert_rows: list[ExpertRow], expert_count: int | None, expert_votes: Counter[int], args: argparse.Namespace) -> None:
+    print(f"# {path.name}")
+    print()
+    print("Score: `RMS = sqrt(mean(in_sum2 / counts))`.")
+    print()
+
+    meta = metadata(reader)
+    if meta:
+        print("Metadata:")
+        print()
+        print_table(["Key", "Value"], ([key, value] for key, value in meta.items()))
+        print()
+
+    blocks = sorted(row.block for row in rows if row.block is not None)
+    if blocks:
+        print(f"Blocks covered: `{blocks[0]}-{blocks[-1]}`")
+        print()
+
+    if expert_count:
+        source = "command line" if args.experts is not None else "inferred from tensor lengths"
+        print(f"Routed expert count: `{expert_count}` ({source}).")
+        if expert_votes and args.experts is None:
+            vote_text = ", ".join(f"{count}: {votes}" for count, votes in expert_votes.most_common(5))
+            print(f"Expert-count inference votes: `{vote_text}`.")
+        print()
+    elif not args.no_experts:
+        print("Routed expert count could not be inferred. Use `--experts N` to enable per-expert rankings.")
+        print()
+
+    rows_sorted = sorted(rows, key=lambda row: row.rms, reverse=True)
+    if rows_sorted:
+        print(f"Top {min(args.top, len(rows_sorted))} `down` / `gate/up` tensor streams:")
+        print()
+        print_table(
+            ["Block", "Family", "Tensor", "RMS", "Mean MSE", "P95", "Max", "Count"],
+            (
+                [
+                    block_label(row.block),
+                    row.family,
+                    f"`{row.tensor}`",
+                    row.rms,
+                    row.mean_mse,
+                    row.p95,
+                    row.max_value,
+                    int(row.count),
+                ]
+                for row in rows_sorted[: args.top]
+            ),
+        )
+        print()
+
+    if expert_rows:
+        print(
+            "Per-expert rankings assume llama.cpp-style routed expert tensors are flattened "
+            "with each expert stored contiguously. Counts are tensor-level counts, so these "
+            "rank aggregate activation contribution per expert stream."
+        )
+        print()
+
+        aggregated = sorted(aggregate_experts(expert_rows), key=lambda row: float(row["rms"]), reverse=True)
+        print(f"Top {min(args.top_experts, len(aggregated))} routed experts, combined across routed down and gate/up:")
+        print()
+        print_table(
+            ["Block", "Expert", "Combined RMS", "Mean MSE", "Max stream RMS", "Streams"],
+            (
+                [
+                    row["block"],
+                    row["expert"],
+                    row["rms"],
+                    row["mean_mse"],
+                    row["max_stream_rms"],
+                    row["streams"],
+                ]
+                for row in aggregated[: args.top_experts]
+            ),
+        )
+        print()
+
+        expert_streams = sorted(expert_rows, key=lambda row: row.rms, reverse=True)
+        print(f"Top {min(args.top_expert_streams, len(expert_streams))} routed expert streams:")
+        print()
+        print_table(
+            ["Block", "Expert", "Family", "Tensor", "RMS", "Mean MSE", "P95", "Max", "Count"],
+            (
+                [
+                    row.block,
+                    row.expert,
+                    row.family,
+                    f"`{row.tensor}`",
+                    row.rms,
+                    row.mean_mse,
+                    row.p95,
+                    row.max_value,
+                    int(row.count),
+                ]
+                for row in expert_streams[: args.top_expert_streams]
+            ),
+        )
+        print()
+
+    families = [
+        "dense down",
+        "dense gate/up",
+        "routed expert down",
+        "routed expert gate/up",
+        "shared expert down",
+        "shared expert gate/up",
+        "router gate input",
+    ]
+    print(f"Top {args.family_top} by family:")
+    print()
+    family_rows: list[list[object]] = []
+    for family in families:
+        subset = [row for row in rows if row.family == family]
+        for row in sorted(subset, key=lambda item: item.rms, reverse=True)[: args.family_top]:
+            family_rows.append(
+                [
+                    family,
+                    block_label(row.block),
+                    f"`{row.tensor}`",
+                    row.rms,
+                    row.mean_mse,
+                    row.p95,
+                    row.p99,
+                    row.max_value,
+                    pct(row.gt1),
+                    pct(row.gt4),
+                    int(row.count),
+                ]
+            )
+    print_table(
+        ["Family", "Block", "Tensor", "RMS", "Mean MSE", "P95", "P99", "Max", ">1", ">4", "Count"],
+        family_rows,
+    )
+    print()
+
+    moe_blocks = sorted(
+        {
+            row.block
+            for row in rows
+            if row.block is not None and row.kind in {"routed expert", "shared expert"}
+        }
+    )
+    if args.last_blocks > 0 and moe_blocks:
+        selected_blocks = moe_blocks[-args.last_blocks :]
+        by_block_family = {(row.block, row.family): row for row in rows}
+        print(f"Last {len(selected_blocks)} MoE block RMS:")
+        print()
+        compact_rows: list[list[object]] = []
+        for block in selected_blocks:
+            compact_rows.append(
+                [
+                    block,
+                    by_block_family.get((block, "shared expert down"), "").rms
+                    if (block, "shared expert down") in by_block_family
+                    else "",
+                    by_block_family.get((block, "shared expert gate/up"), "").rms
+                    if (block, "shared expert gate/up") in by_block_family
+                    else "",
+                    by_block_family.get((block, "routed expert down"), "").rms
+                    if (block, "routed expert down") in by_block_family
+                    else "",
+                    by_block_family.get((block, "routed expert gate/up"), "").rms
+                    if (block, "routed expert gate/up") in by_block_family
+                    else "",
+                ]
+            )
+        print_table(["Block", "Shared down", "Shared gate/up", "Routed down", "Routed gate/up"], compact_rows)
+        print()
+
+
+def main() -> int:
+    args = parse_args()
+    if not args.imatrix.exists():
+        print(f"error: file not found: {args.imatrix}", file=sys.stderr)
+        return 2
+
+    reader = GGUFReader(str(args.imatrix))
+    rows, expert_rows, expert_count, expert_votes = analyze(reader, args)
+    if not rows:
+        print("error: no FFN down/gate/up imatrix tensors were found", file=sys.stderr)
+        return 1
+
+    render(args.imatrix, reader, rows, expert_rows, expert_count, expert_votes, args)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/split.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage: split.sh <llama_cpp_dir> <input_gguf> <output_dir> [output_prefix] [--remove-input]
+
+Environment overrides:
+  SPLIT_MAX_SIZE=50G    Maximum size for each split file.
+
+Examples:
+  ./scripts/split.sh ~/code/llama.cpp model.gguf ./Q5_K_S
+  ./scripts/split.sh ~/code/llama.cpp model.gguf ./Q5_K_S model-Q5_K_S --remove-input
+EOF
+}
+
+if [ $# -lt 3 ]; then
+  usage
+  exit 1
+fi
+
+LLAMA_CPP_DIR="$1"
+INPUT_GGUF="$2"
+OUTPUT_DIR="$3"
+OUTPUT_PREFIX_NAME=
+REMOVE_INPUT=false
+shift 3
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --remove-input)
+      REMOVE_INPUT=true
+      ;;
+    --*)
+      usage
+      exit 1
+      ;;
+    *)
+      if [ -n "$OUTPUT_PREFIX_NAME" ]; then
+        usage
+        exit 1
+      fi
+      OUTPUT_PREFIX_NAME="$1"
+      ;;
+  esac
+  shift
+done
+
+if [ ! -d "$LLAMA_CPP_DIR" ]; then
+  echo "Error: llama.cpp directory not found: $LLAMA_CPP_DIR" >&2
+  exit 1
+fi
+
+if [ ! -e "$INPUT_GGUF" ]; then
+  echo "Error: input GGUF not found: $INPUT_GGUF" >&2
+  exit 1
+fi
+
+SPLIT_BIN="$LLAMA_CPP_DIR/build/bin/llama-gguf-split"
+if [ ! -x "$SPLIT_BIN" ]; then
+  echo "Error: llama-gguf-split binary not found: $SPLIT_BIN" >&2
+  exit 1
+fi
+
+if [ -z "$OUTPUT_PREFIX_NAME" ]; then
+  input_basename="$(basename "$INPUT_GGUF")"
+  OUTPUT_PREFIX_NAME="${input_basename%.gguf}"
+fi
+
+SPLIT_MAX_SIZE="${SPLIT_MAX_SIZE:-50G}"
+mkdir -p "$OUTPUT_DIR"
+OUTPUT_PREFIX="$OUTPUT_DIR/$OUTPUT_PREFIX_NAME"
+
+"$SPLIT_BIN" \
+  --split-max-size "$SPLIT_MAX_SIZE" \
+  --no-tensor-first-split \
+  "$INPUT_GGUF" \
+  "$OUTPUT_PREFIX"
+
+if [ "$REMOVE_INPUT" = true ]; then
+  rm -f "$INPUT_GGUF"
+fi
+
+echo "Split complete. Output saved to: $OUTPUT_DIR"