add gguf/hy4-preview-patch/Hy4-preview-STQ1_0.tensortypes from AngelSlim/Hy4-preview-GGUF

gguf/hy4-preview-patch/Hy4-preview-STQ1_0.tensortypes

@@ -0,0 +1,211 @@
+# Rung G: rung F 的变体 -- 把 IQ1_S 全部替换为 STQ1_0 (1.3125 bpw)。
+#
+# 与 rung F 的唯一差别: 那 29 层 (58 个张量) 的 gate/up 从 IQ1_S (1.5625) 降到 STQ1_0 (1.3125)。
+# 48 层的 IQ2_XXS 与所有非专家张量完全不变。
+#
+# STQ1_0 用的是本 fork 的 imatrix-aware PTQ 编码器 (commit f68f3f35a), 不是上游 PR 的原版:
+#   1. 加权最小二乘 scale: d = sum(w*sel*x)/sum(w*sel^2)  取代上游的 d = amax
+#      -- 这是主要收益来源, 实测 -89.7% 加权 SSD (amax 会把 d 钉在 256 个权重里的最大离群值上)
+#   2. imatrix-aware 强制零位置: 每 4 lane 一组, 零掉使 w[j]*(x[j]^2-(|x[j]|-d)^2) 最小的那个,
+#      即比较"增量"代价, 而非上游的 argmin|x|
+#   两者交替 3 轮 (coordinate descent), 权重 w[j]=quant_weights[j]*sqrt(sigma2+x[j]^2), sigma2 块内局部。
+#   相对已修好的 LS scale, imatrix 项再带来 -4.1% 残差下降。
+#
+# 格式本身与上游 PR #22836 字节一致: {-d,0,+d}, 每 4 lane 强制一个零 (3:4 稀疏), stride-16 分组,
+# 32 字节码本, 每 256 权重一个 fp16 scale。
+#
+# 部署: STQ1_0 已有 CUDA kernel (commit 31c23dd25, mmvq + dequant, 已注册进 MUL_MAT 与
+# MUL_MAT_ID -- 后者是 MoE 路径, 这 58 个张量全部走它)。rung G 与 E/F 一样全程留在 GPU 路径上。
+# 实测 pp512 从 20.80 t/s (无 kernel, CPU 回落) 提升到 204.56 t/s (9.83x); 加载日志里
+# CPU_Mapped model buffer 从 106835 MiB 降到 398 MiB, graph splits 从 67 降到 9。
+# ⚠ 但构建必须真的带上 kernel: `nm -D build-cuda/bin/libggml-cuda.so | grep -ci stq1_0` 必须 > 0。
+# efae5ba56 及更早的 build 没有 kernel, 其 STQ1_0 速度数字全部是 CPU 回落态, 不代表格式本身。
+#
+# ⚠ 已知精度上限: 21_补偿性PTQ_可行性分析.md 实测, 3:4 强制零这一条约束本身就占了约 33% 的可改进
+# 误差, 且穷举零位置搜索相对本编码器的贪心规则收益为 +0.00% (即搜索已最优)。所以若 rung G 精度不足,
+# 原因在格式而非编码器, 只能换格式或走 QAT。
+#
+# ORDERING IS LOAD-BEARING: 逐层规则必须排在通用 ffn_*_exps 规则之前 (regex_search, 首个匹配生效)。
+
+blk.1.ffn_gate_exps=iq2_xxs
+blk.1.ffn_up_exps=iq2_xxs
+blk.12.ffn_gate_exps=iq2_xxs
+blk.12.ffn_up_exps=iq2_xxs
+blk.17.ffn_gate_exps=iq2_xxs
+blk.17.ffn_up_exps=iq2_xxs
+blk.18.ffn_gate_exps=iq2_xxs
+blk.18.ffn_up_exps=iq2_xxs
+blk.19.ffn_gate_exps=iq2_xxs
+blk.19.ffn_up_exps=iq2_xxs
+blk.25.ffn_gate_exps=iq2_xxs
+blk.25.ffn_up_exps=iq2_xxs
+blk.26.ffn_gate_exps=iq2_xxs
+blk.26.ffn_up_exps=iq2_xxs
+blk.27.ffn_gate_exps=iq2_xxs
+blk.27.ffn_up_exps=iq2_xxs
+blk.28.ffn_gate_exps=iq2_xxs
+blk.28.ffn_up_exps=iq2_xxs
+blk.30.ffn_gate_exps=iq2_xxs
+blk.30.ffn_up_exps=iq2_xxs
+blk.39.ffn_gate_exps=iq2_xxs
+blk.39.ffn_up_exps=iq2_xxs
+blk.40.ffn_gate_exps=iq2_xxs
+blk.40.ffn_up_exps=iq2_xxs
+blk.42.ffn_gate_exps=iq2_xxs
+blk.42.ffn_up_exps=iq2_xxs
+blk.43.ffn_gate_exps=iq2_xxs
+blk.43.ffn_up_exps=iq2_xxs
+blk.44.ffn_gate_exps=iq2_xxs
+blk.44.ffn_up_exps=iq2_xxs
+blk.45.ffn_gate_exps=iq2_xxs
+blk.45.ffn_up_exps=iq2_xxs
+blk.46.ffn_gate_exps=iq2_xxs
+blk.46.ffn_up_exps=iq2_xxs
+blk.47.ffn_gate_exps=iq2_xxs
+blk.47.ffn_up_exps=iq2_xxs
+blk.48.ffn_gate_exps=iq2_xxs
+blk.48.ffn_up_exps=iq2_xxs
+blk.49.ffn_gate_exps=iq2_xxs
+blk.49.ffn_up_exps=iq2_xxs
+blk.50.ffn_gate_exps=iq2_xxs
+blk.50.ffn_up_exps=iq2_xxs
+blk.51.ffn_gate_exps=iq2_xxs
+blk.51.ffn_up_exps=iq2_xxs
+blk.52.ffn_gate_exps=iq2_xxs
+blk.52.ffn_up_exps=iq2_xxs
+blk.53.ffn_gate_exps=iq2_xxs
+blk.53.ffn_up_exps=iq2_xxs
+blk.54.ffn_gate_exps=iq2_xxs
+blk.54.ffn_up_exps=iq2_xxs
+blk.55.ffn_gate_exps=iq2_xxs
+blk.55.ffn_up_exps=iq2_xxs
+blk.56.ffn_gate_exps=iq2_xxs
+blk.56.ffn_up_exps=iq2_xxs
+blk.57.ffn_gate_exps=iq2_xxs
+blk.57.ffn_up_exps=iq2_xxs
+blk.58.ffn_gate_exps=iq2_xxs
+blk.58.ffn_up_exps=iq2_xxs
+blk.59.ffn_gate_exps=iq2_xxs
+blk.59.ffn_up_exps=iq2_xxs
+blk.60.ffn_gate_exps=iq2_xxs
+blk.60.ffn_up_exps=iq2_xxs
+blk.61.ffn_gate_exps=iq2_xxs
+blk.61.ffn_up_exps=iq2_xxs
+blk.62.ffn_gate_exps=iq2_xxs
+blk.62.ffn_up_exps=iq2_xxs
+blk.63.ffn_gate_exps=iq2_xxs
+blk.63.ffn_up_exps=iq2_xxs
+blk.64.ffn_gate_exps=iq2_xxs
+blk.64.ffn_up_exps=iq2_xxs
+blk.65.ffn_gate_exps=iq2_xxs
+blk.65.ffn_up_exps=iq2_xxs
+blk.66.ffn_gate_exps=iq2_xxs
+blk.66.ffn_up_exps=iq2_xxs
+blk.67.ffn_gate_exps=iq2_xxs
+blk.67.ffn_up_exps=iq2_xxs
+blk.68.ffn_gate_exps=iq2_xxs
+blk.68.ffn_up_exps=iq2_xxs
+blk.69.ffn_gate_exps=iq2_xxs
+blk.69.ffn_up_exps=iq2_xxs
+blk.70.ffn_gate_exps=iq2_xxs
+blk.70.ffn_up_exps=iq2_xxs
+blk.71.ffn_gate_exps=iq2_xxs
+blk.71.ffn_up_exps=iq2_xxs
+blk.72.ffn_gate_exps=iq2_xxs
+blk.72.ffn_up_exps=iq2_xxs
+blk.73.ffn_gate_exps=iq2_xxs
+blk.73.ffn_up_exps=iq2_xxs
+blk.74.ffn_gate_exps=iq2_xxs
+blk.74.ffn_up_exps=iq2_xxs
+blk.75.ffn_gate_exps=iq2_xxs
+blk.75.ffn_up_exps=iq2_xxs
+blk.76.ffn_gate_exps=iq2_xxs
+blk.76.ffn_up_exps=iq2_xxs
+blk.77.ffn_gate_exps=iq2_xxs
+blk.77.ffn_up_exps=iq2_xxs
+
+# all remaining routed gate/up drop to IQ1_S (layers 2-11,13-16,20-24,29,31-38,41)
+ffn_gate_exps=stq1_0
+ffn_up_exps=stq1_0
+
+# ---------------- base recipe (hyv4_iq1.tensortypes, unmodified) ----------------
+# IQ1_M mixed-precision recipe for HY4, aligned to the Unsloth GLM-5.2 UD-IQ1_M baseline.
+#
+# Baseline: unsloth/GLM-5.2-GGUF UD-IQ1_M (212.80 GiB, 2.425 bpw overall). GLM-5.2 is arch
+# glm-dsa -- the same family as HY4 (MLA + 256 routed experts top-8 + a DSA indexer with
+# index_topk=2048, indexer head_count=32, key_length=128), which makes its per-tensor choices
+# directly transferable. Types below were read out of the UD-IQ1_M GGUF headers, not from prose.
+#
+# ORDERING IS LOAD-BEARING. Overrides are matched with regex_search and the FIRST match wins
+# (llama-quant.cpp llama_tensor_get_type), so specific rules must precede general ones. Two
+# real traps: "attn_q_b.weight" also matches "blk.N.indexer.attn_q_b.weight", and a bare
+# "ffn_down_exps" would swallow the per-layer bumps below.
+
+# ---- kept in F32: iHC mixing matrices + router (reference _keep_in_fp32_modules) ----
+hc_attn_fn=f32
+hc_ffn_fn=f32
+output_hc_fn=f32
+ffn_gate_inp=f32
+
+# ---- DSA indexer: q8_0 / f32 ----
+# Identical to GLM UD-IQ1_M (indexer.attn_q_b + indexer.attn_k = Q8_0, proj + k_norm = F32).
+# llama-quant.cpp has no indexer guard, so without these they take the IQ1_M base type -- 1.75
+# bpw on the tensors that decide which index_topk (2048) tokens every query may attend to, i.e.
+# the gate on the entire long-context path. All 105 indexer tensors are ~0.4 GiB in bf16, so the
+# floor costs almost nothing. MUST precede the attn_* rules below.
+indexer.attn_q_b=q8_0
+indexer.attn_k=q8_0
+indexer.proj=f32
+
+# ---- MLA q/k/v projections: q8_0 (GLM baseline) ----
+# tensor_get_category() matches "attn_v.weight"/"attn_kv_b.weight" by exact substring, so HY4's
+# *split* k_b/v_b/q_b names fall into category OTHER and get no automatic bump at all. GLM keeps
+# all four of these at Q8_0; these are only ~1.6% of parameters, so match it.
+# (k_b has ncols=192: not divisible by 256, but Q8_0's block is 32, so no fallback occurs.)
+attn_q_b.weight=q8_0
+attn_k_b=q8_0
+attn_v_b=q8_0
+attn_kv_a_mqa=q8_0
+
+# ---- attention output / q_a / gate: q5_k (GLM baseline) ----
+# attn_output is the one llama.cpp actively gets wrong here: llama_tensor_get_type_impl bumps it
+# to Q5_K only when n_expert == 8, otherwise IQ1_M -> IQ2_XXS. HY4 has 256 experts, so it took
+# IQ2_XXS (2.06 bpw) on all 78 layers while GLM ships Q5_K. attn_gate has no GLM analogue; it is
+# the same shape and sits on the same residual path as attn_output, so it gets the same floor.
+attn_q_a.weight=q5_k
+attn_output=q5_k
+attn_gate=q5_k
+
+# ---- routed experts: down gets two extra levels over gate/up ----
+# This is the core of the Unsloth recipe. down_proj reads from and writes straight back into the
+# residual stream, so its quantization error is not attenuated by a following gate -- GLM puts
+# gate/up at IQ1_M/IQ2_XXS but every down_exps at IQ3_XXS, and lifts the final layers further.
+# The last-3-layer bump mirrors GLM's IQ4_XS at its last three MoE layers. (GLM also bumps one
+# mid-stack layer; that index does not transfer across a different depth, so it is not mirrored.)
+# These per-layer rules MUST precede the generic ffn_down_exps rule.
+blk.75.ffn_down_exps=iq4_xs
+blk.76.ffn_down_exps=iq4_xs
+blk.77.ffn_down_exps=iq4_xs
+ffn_down_exps=iq3_xxs
+# gate/up_exps stay on the IQ1_M base type -- this is the one deliberate deviation from GLM,
+# which alternates roughly half of them up to IQ2_XXS. Mirroring that would add ~9 GiB and its
+# per-layer pattern looks imatrix-derived rather than structural.
+
+# ---- shared experts: q6_k ----
+# GLM uses Q5_K for gate/up_shexp and Q6_K for down_shexp; Q6_K throughout is strictly safer and
+# costs 0.24 GiB, so the higher floor is kept.
+ffn_gate_shexp=q6_k
+ffn_up_shexp=q6_k
+ffn_down_shexp=q6_k
+
+# ---- dense layer-0 MLP: q6_k (GLM: Q5_K gate/up, Q6_K down on its 3 dense layers) ----
+blk.0.ffn_gate.weight=q6_k
+blk.0.ffn_up.weight=q6_k
+blk.0.ffn_down.weight=q6_k
+
+# ---- token embeddings: q4_k (GLM baseline; llama.cpp defaults IQ1_M -> Q2_K) ----
+token_embd=q4_k
+
+# output.weight (lm_head) is NOT listed here: --leave-output-tensor keeps it at the F32 written
+# by the converter, mirroring the reference's enable_lm_head_fp32 matmul. That is 2.77 GiB versus
+# GLM's Q4_K, i.e. deliberately more conservative than the baseline.