SMQ set
Browse files- smq.py +282 -0
- smq_gguf_convert_live.py +1480 -0
- smq_qat.py +0 -0
- train_smq_qat.py +189 -0
smq.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SMQ solver pieces under the matched metric: code projection + closed-form (d, m).
|
| 2 |
+
|
| 3 |
+
E = sum_i w_i e_i^2 + sum_{k=1..8} lambda_k * sum_j G_kj^2, e = x - xhat
|
| 4 |
+
|
| 5 |
+
The whole matched metric is a SINGLE quadratic form. Each fold level k is a
|
| 6 |
+
strided group-sum operator A_k (element i belongs to group i mod (256/2^k)),
|
| 7 |
+
and sum_j G_kj^2 = ||A_k e||^2 = e' A_k' A_k e, so
|
| 8 |
+
|
| 9 |
+
E = e' M e, M = diag(w) + L, L = sum_k lambda_k A_k' A_k (fixed 256x256)
|
| 10 |
+
|
| 11 |
+
L is precomputed once per (lam, dtype, device) by _fold_gram and every
|
| 12 |
+
objective evaluation, gradient tile, and Hessian/Gram entry is then just a
|
| 13 |
+
bilinear form with L (one matmul), with no per-level fold loops.
|
| 14 |
+
|
| 15 |
+
Per pass: project every code at once (Newton form, e_i = x_i - s_i q_i +
|
| 16 |
+
m lm_i, s_i = d ls_i, Lam = sum_k lambda_k). Since (L e)_i equals
|
| 17 |
+
sum_k lambda_k G_k(i), the per-element coupled gradient tile is e @ L:
|
| 18 |
+
|
| 19 |
+
q_i <- clip(round(q_i + (w_i e_i + (e @ L)_i) / ((w_i + Lam) s_i)), 0, 3)
|
| 20 |
+
|
| 21 |
+
After the pass, e is rebuilt from the new codes and the next pass re-evaluates.
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
QK = 256
|
| 27 |
+
NSUB = 16
|
| 28 |
+
_TINY = 1.0e-30
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _fold_gram(lam):
|
| 32 |
+
"""Single-matrix form of the spectral penalty: L = sum_k lambda_k A_k' A_k.
|
| 33 |
+
|
| 34 |
+
A_k is the strided fold operator with (A_k e)_r = sum over elements i with
|
| 35 |
+
i mod (256/2^k) == r, so (A_k' A_k)[i, j] = [i mod (256/2^k) == j mod
|
| 36 |
+
(256/2^k)]. The full matched metric of the module docstring is then
|
| 37 |
+
E = e' (diag(w) + L) e — one quadratic form, no fold loops.
|
| 38 |
+
"""
|
| 39 |
+
import torch
|
| 40 |
+
|
| 41 |
+
ids = torch.arange(QK, device=lam.device)
|
| 42 |
+
L = torch.zeros(QK, QK, dtype=lam.dtype, device=lam.device)
|
| 43 |
+
for k in range(1, 9):
|
| 44 |
+
r = ids.remainder(QK >> k)
|
| 45 |
+
L = L + lam[k - 1] * (r[:, None] == r[None, :]).to(L.dtype)
|
| 46 |
+
return L
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _objective_quad(e, w, L):
|
| 50 |
+
"""Matched metric E per block as one quadratic form: (w e^2) + e' L e -> (n,)."""
|
| 51 |
+
return (w * e * e).sum(dim=1) + (e @ L * e).sum(dim=1)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _codes_pass(x, w, e, d, m, ls_e, lm_e, q, L, lam_sum):
|
| 55 |
+
"""One code-projection pass over all 256 elements. Returns (q, e)."""
|
| 56 |
+
import torch
|
| 57 |
+
|
| 58 |
+
S = e @ L # == sum_k lambda_k G_k(i) tiled per element
|
| 59 |
+
step = d[:, None] * ls_e
|
| 60 |
+
curv = w + lam_sum
|
| 61 |
+
good = (step > _TINY) & (curv > _TINY)
|
| 62 |
+
safe = torch.where(good, curv * step, torch.ones_like(step))
|
| 63 |
+
ratio = q + (w * e + S) / safe
|
| 64 |
+
q_new = torch.where(good, torch.clamp(torch.round(ratio), 0.0, 3.0), q)
|
| 65 |
+
e_new = x - d[:, None] * ls_e * q_new + m[:, None] * lm_e
|
| 66 |
+
return q_new, e_new
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _dm_system(x, w, a, b, L):
|
| 70 |
+
"""2x2 matched-metric WLS system for (d, m), one quadratic form, no folds.
|
| 71 |
+
|
| 72 |
+
a = ls_e * q (grid slope), b = lm_e (grid offset, r_i = 1); returns
|
| 73 |
+
H11, H12, H22, g1, g2 for E = th' H th - 2 th' g + c, th = (d, m).
|
| 74 |
+
Every fold-level term is a bilinear form u' L v of the single matrix L:
|
| 75 |
+
sum_k lam_k (A_k u) . (A_k v) = u' L v.
|
| 76 |
+
"""
|
| 77 |
+
H11 = (w * a * a).sum(1) + (a @ L * a).sum(1)
|
| 78 |
+
H12 = -((w * a * b).sum(1) + (a @ L * b).sum(1))
|
| 79 |
+
H22 = (w * b * b).sum(1) + (b @ L * b).sum(1)
|
| 80 |
+
g1 = (w * x * a).sum(1) + (x @ L * a).sum(1)
|
| 81 |
+
g2 = -((w * x * b).sum(1) + (x @ L * b).sum(1))
|
| 82 |
+
return H11, H12, H22, g1, g2
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _solve2(H11, H12, H22, g1, g2):
|
| 86 |
+
"""Solve a batch of 2x2 systems H th = g with a validity mask."""
|
| 87 |
+
import torch
|
| 88 |
+
|
| 89 |
+
det = H11 * H22 - H12 * H12
|
| 90 |
+
ok = (det > _TINY) & torch.isfinite(det) & torch.isfinite(g1) & torch.isfinite(g2)
|
| 91 |
+
det_s = torch.where(ok, det, torch.ones_like(det))
|
| 92 |
+
v1 = (H22 * g1 - H12 * g2) / det_s
|
| 93 |
+
v2 = (H11 * g2 - H12 * g1) / det_s
|
| 94 |
+
ok = ok & torch.isfinite(v1) & torch.isfinite(v2)
|
| 95 |
+
return ok, v1, v2
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def _quad2(H11, H12, H22, g1, g2, d, m):
|
| 99 |
+
"""Constant-free value of the 2-D quadratic represented by _dm_system."""
|
| 100 |
+
return (
|
| 101 |
+
H11 * d * d + 2.0 * H12 * d * m + H22 * m * m
|
| 102 |
+
- 2.0 * g1 * d - 2.0 * g2 * m
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _solve2_constrained(
|
| 107 |
+
H11, H12, H22, g1, g2, d_current, m_current, d_floor=_TINY
|
| 108 |
+
):
|
| 109 |
+
"""Minimize the convex 2-D quadratic subject to d >= d_floor and m >= 0.
|
| 110 |
+
|
| 111 |
+
The candidate set consists of the feasible unconstrained optimum, both
|
| 112 |
+
one-dimensional boundary optima, the corner (included by clamping the
|
| 113 |
+
boundary solutions), and the current feasible point as a numerical
|
| 114 |
+
fallback. Comparing the original quadratic makes this valid for singular
|
| 115 |
+
positive-semidefinite systems as well as nonsingular systems.
|
| 116 |
+
"""
|
| 117 |
+
import torch
|
| 118 |
+
|
| 119 |
+
d0 = torch.clamp(torch.nan_to_num(d_current, nan=d_floor, posinf=65504.0, neginf=d_floor), min=d_floor)
|
| 120 |
+
m0 = torch.clamp(torch.nan_to_num(m_current, nan=0.0, posinf=65504.0, neginf=0.0), min=0.0)
|
| 121 |
+
best_d, best_m = d0, m0
|
| 122 |
+
best_v = _quad2(H11, H12, H22, g1, g2, best_d, best_m)
|
| 123 |
+
best_v = torch.where(torch.isfinite(best_v), best_v, torch.full_like(best_v, float("inf")))
|
| 124 |
+
|
| 125 |
+
# Interior stationary point.
|
| 126 |
+
ok_i, d_i, m_i = _solve2(H11, H12, H22, g1, g2)
|
| 127 |
+
ok_i = ok_i & (d_i >= d_floor) & (m_i >= 0.0)
|
| 128 |
+
v_i = _quad2(H11, H12, H22, g1, g2, d_i, m_i)
|
| 129 |
+
ok_i = ok_i & torch.isfinite(v_i)
|
| 130 |
+
take = ok_i & (v_i < best_v)
|
| 131 |
+
best_d = torch.where(take, d_i, best_d)
|
| 132 |
+
best_m = torch.where(take, m_i, best_m)
|
| 133 |
+
best_v = torch.where(take, v_i, best_v)
|
| 134 |
+
|
| 135 |
+
# Boundary m = 0: optimize d on its feasible half-line.
|
| 136 |
+
h11_ok = torch.isfinite(H11) & (H11 > _TINY)
|
| 137 |
+
d_m0 = torch.where(h11_ok, torch.clamp(g1 / torch.where(h11_ok, H11, torch.ones_like(H11)), min=d_floor), d0)
|
| 138 |
+
m_m0 = torch.zeros_like(m0)
|
| 139 |
+
v_m0 = _quad2(H11, H12, H22, g1, g2, d_m0, m_m0)
|
| 140 |
+
ok_m0 = torch.isfinite(v_m0) & torch.isfinite(d_m0)
|
| 141 |
+
take = ok_m0 & (v_m0 < best_v)
|
| 142 |
+
best_d = torch.where(take, d_m0, best_d)
|
| 143 |
+
best_m = torch.where(take, m_m0, best_m)
|
| 144 |
+
best_v = torch.where(take, v_m0, best_v)
|
| 145 |
+
|
| 146 |
+
# Boundary d = d_floor: optimize the nonnegative offset.
|
| 147 |
+
h22_ok = torch.isfinite(H22) & (H22 > _TINY)
|
| 148 |
+
h22_safe = torch.where(h22_ok, H22, torch.ones_like(H22))
|
| 149 |
+
m_df = torch.where(
|
| 150 |
+
h22_ok,
|
| 151 |
+
torch.clamp((g2 - H12 * d_floor) / h22_safe, min=0.0),
|
| 152 |
+
m0,
|
| 153 |
+
)
|
| 154 |
+
d_df = torch.full_like(d0, float(d_floor))
|
| 155 |
+
v_df = _quad2(H11, H12, H22, g1, g2, d_df, m_df)
|
| 156 |
+
ok_df = torch.isfinite(v_df) & torch.isfinite(m_df)
|
| 157 |
+
take = ok_df & (v_df < best_v)
|
| 158 |
+
best_d = torch.where(take, d_df, best_d)
|
| 159 |
+
best_m = torch.where(take, m_df, best_m)
|
| 160 |
+
|
| 161 |
+
return best_d, best_m
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def _dm_resolve(x, w, q, ls_e, lm_e, d, m, L):
|
| 165 |
+
"""Closed-form matched-metric (d, m) for ALL blocks at once, no iteration.
|
| 166 |
+
|
| 167 |
+
With codes and (ls, lm) fixed, e_i = x_i - d a_i + m b_i is affine in
|
| 168 |
+
(d, m), a_i = ls_i q_i, b_i = lm_i, so the single quadratic E = e' M e
|
| 169 |
+
(M = diag(w) + L, all eight fold levels included) is an exact 2x2
|
| 170 |
+
quadratic in th = (d, m): E = th' H th - 2 th' g + c. The solution is
|
| 171 |
+
th = H^-1 g (Cramer's rule), constrained to d > 0, m >= 0.
|
| 172 |
+
Returns continuous (d, m); the caller rounds to fp16 and guards on E.
|
| 173 |
+
"""
|
| 174 |
+
a = ls_e * q
|
| 175 |
+
H11, H12, H22, g1, g2 = _dm_system(x, w, a, lm_e, L)
|
| 176 |
+
return _solve2_constrained(H11, H12, H22, g1, g2, d, m)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def _integer_lattice_min_2d(H11, H12, H22, g1, g2, current_s, current_t, upper=15):
|
| 180 |
+
"""Exact minimum of a convex 2-D quadratic on {0..upper}^2.
|
| 181 |
+
|
| 182 |
+
Enumerate the 16 (or ``upper+1``) possible first coordinates. For each,
|
| 183 |
+
the conditional optimum in t is one-dimensional and convex, so its integer
|
| 184 |
+
minimum is floor(t*) or ceil(t*) (with clipping at the boundaries). This
|
| 185 |
+
gives at most 2*(upper+1) candidates instead of a full square lattice scan.
|
| 186 |
+
"""
|
| 187 |
+
import torch
|
| 188 |
+
|
| 189 |
+
cur_s = torch.clamp(torch.round(current_s), 0.0, float(upper))
|
| 190 |
+
cur_t = torch.clamp(torch.round(current_t), 0.0, float(upper))
|
| 191 |
+
s_grid = torch.arange(upper + 1, device=H11.device, dtype=H11.dtype).view(1, -1)
|
| 192 |
+
s_grid = s_grid.expand(H11.shape[0], -1)
|
| 193 |
+
h22_ok = torch.isfinite(H22) & (H22 > _TINY)
|
| 194 |
+
h22_safe = torch.where(h22_ok, H22, torch.ones_like(H22))
|
| 195 |
+
t_star = torch.clamp(
|
| 196 |
+
(g2[:, None] - H12[:, None] * s_grid) / h22_safe[:, None],
|
| 197 |
+
0.0, float(upper),
|
| 198 |
+
)
|
| 199 |
+
t_lo = torch.floor(t_star)
|
| 200 |
+
t_hi = torch.ceil(t_star)
|
| 201 |
+
# For a PSD quadratic H22=0 implies H12=0; t is then unidentifiable.
|
| 202 |
+
t_lo = torch.where(h22_ok[:, None], t_lo, cur_t[:, None])
|
| 203 |
+
t_hi = torch.where(h22_ok[:, None], t_hi, cur_t[:, None])
|
| 204 |
+
s_candidates = torch.cat((s_grid, s_grid), dim=1)
|
| 205 |
+
t_candidates = torch.cat((t_lo, t_hi), dim=1)
|
| 206 |
+
values = _quad2(
|
| 207 |
+
H11[:, None], H12[:, None], H22[:, None],
|
| 208 |
+
g1[:, None], g2[:, None], s_candidates, t_candidates,
|
| 209 |
+
)
|
| 210 |
+
values = torch.where(torch.isfinite(values), values, torch.full_like(values, float("inf")))
|
| 211 |
+
best_idx = torch.argmin(values, dim=1, keepdim=True)
|
| 212 |
+
best_val = values.gather(1, best_idx).squeeze(1)
|
| 213 |
+
best_s = s_candidates.gather(1, best_idx).squeeze(1)
|
| 214 |
+
best_t = t_candidates.gather(1, best_idx).squeeze(1)
|
| 215 |
+
|
| 216 |
+
# Retain the current lattice point if roundoff makes its evaluated value
|
| 217 |
+
# lower than the analytically enumerated conditional minima.
|
| 218 |
+
cur_val = _quad2(H11, H12, H22, g1, g2, cur_s, cur_t)
|
| 219 |
+
keep_cur = (~torch.isfinite(best_val)) | (cur_val < best_val)
|
| 220 |
+
return torch.where(keep_cur, cur_s, best_s), torch.where(keep_cur, cur_t, best_t), torch.minimum(cur_val, best_val)
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _ls_lm_resolve(x, w, e, d, m, q, ls, lm, lam):
|
| 224 |
+
"""Closed-form matched-metric (ls, lm) for ALL 16 sub-blocks of ALL blocks at once.
|
| 225 |
+
|
| 226 |
+
For sub-block j, e_i = x_i - Ls alpha_i + Lm beta (alpha_i = d q_i, beta = m)
|
| 227 |
+
is affine in theta = (Ls, Lm). A level-k group g meets sub-block j in A_g =
|
| 228 |
+
sum alpha (and B_g = |g n j| m) and its group sum is T_g - Ls A_g + Lm B_g,
|
| 229 |
+
where T_g = G_g + Ls_cur A_g - Lm_cur B_g carries every other sub-block's
|
| 230 |
+
current contribution. So the matched E restricted to sub-block j is an exact
|
| 231 |
+
2x2 quadratic th' H th - 2 th' g + c with
|
| 232 |
+
H11 = sum w alpha^2 + sum_k lam_k sum_g A_g^2 H12 = -(sum w alpha beta + sum_k lam_k sum_g A_g B_g)
|
| 233 |
+
H22 = sum w beta^2 + sum_k lam_k sum_g B_g^2
|
| 234 |
+
g1 = sum w x alpha + sum_k lam_k sum_g T_g A_g g2 = -(sum w x beta + sum_k lam_k sum_g T_g B_g)
|
| 235 |
+
Levels 1-4 (stride >= 16): each group holds one element of j. Levels 5-8
|
| 236 |
+
(stride 8, 4, 2, 1): each group holds 16/stride elements of j.
|
| 237 |
+
The integer minimum on {0..15}^2 is then enumerated exactly (no 256-point scan).
|
| 238 |
+
All sub-blocks are solved against the CURRENT state of the others.
|
| 239 |
+
The metric is still the single quadratic E = e' (diag(w) + L) e; this solver
|
| 240 |
+
keeps the explicit fold form because its per-sub-block "state of the others"
|
| 241 |
+
correction T_g does not reduce to cheap bilinear forms with L.
|
| 242 |
+
"""
|
| 243 |
+
import torch
|
| 244 |
+
|
| 245 |
+
n = x.shape[0]
|
| 246 |
+
al = (d[:, None] * q).reshape(n, NSUB, 16)
|
| 247 |
+
be = m[:, None, None]
|
| 248 |
+
xs = x.reshape(n, NSUB, 16)
|
| 249 |
+
ws = w.reshape(n, NSUB, 16)
|
| 250 |
+
H11 = (ws * al * al).sum(2)
|
| 251 |
+
H12 = -(ws * al * be).sum(2)
|
| 252 |
+
H22 = (ws * be * be).sum(2)
|
| 253 |
+
g1 = (ws * xs * al).sum(2)
|
| 254 |
+
g2 = -(ws * xs * be).sum(2)
|
| 255 |
+
|
| 256 |
+
ids = torch.arange(QK, device=x.device).reshape(NSUB, 16)
|
| 257 |
+
ls3 = ls[:, :, None]
|
| 258 |
+
lm3 = lm[:, :, None]
|
| 259 |
+
for k in range(1, 9):
|
| 260 |
+
S = QK >> k
|
| 261 |
+
Gk = e.reshape(n, 1 << k, S).sum(1) # (n, S) group sums, indexed by residue
|
| 262 |
+
if S >= 16:
|
| 263 |
+
A = al
|
| 264 |
+
B = be + torch.zeros_like(al)
|
| 265 |
+
Gc = Gk[:, ids % S] # (n, 16, 16): group sum of each element's own group
|
| 266 |
+
else:
|
| 267 |
+
A = al.reshape(n, NSUB, 16 // S, S).sum(2) # (n, 16, S)
|
| 268 |
+
B = be * (16 // S) + torch.zeros_like(A)
|
| 269 |
+
Gc = Gk[:, None, :] + torch.zeros_like(A)
|
| 270 |
+
T = Gc + ls3 * A - lm3 * B
|
| 271 |
+
c = lam[k - 1]
|
| 272 |
+
H11 = H11 + c * (A * A).sum(2)
|
| 273 |
+
H12 = H12 - c * (A * B).sum(2)
|
| 274 |
+
H22 = H22 + c * (B * B).sum(2)
|
| 275 |
+
g1 = g1 + c * (T * A).sum(2)
|
| 276 |
+
g2 = g2 - c * (T * B).sum(2)
|
| 277 |
+
|
| 278 |
+
s_new, t_new, _ = _integer_lattice_min_2d(
|
| 279 |
+
H11.reshape(-1), H12.reshape(-1), H22.reshape(-1), g1.reshape(-1), g2.reshape(-1),
|
| 280 |
+
ls.reshape(-1), lm.reshape(-1),
|
| 281 |
+
)
|
| 282 |
+
return s_new.reshape(n, NSUB), t_new.reshape(n, NSUB)
|
smq_gguf_convert_live.py
ADDED
|
@@ -0,0 +1,1480 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Factor GGUF weights directly into Q2_K by closed-form quadratic decomposition.
|
| 3 |
+
|
| 4 |
+
Every eligible tensor is processed in ONE vectorized NumPy pass, with no
|
| 5 |
+
iteration and no objective sweeps:
|
| 6 |
+
|
| 7 |
+
1. RTN grid (algebraic, closed form): per 16-wide sub-block, step and offset
|
| 8 |
+
from min/max; per-block d = max step / 15, dmin = max offset / 15 (fp16);
|
| 9 |
+
codes q = clip(round((x + o) / s), 0, 3).
|
| 10 |
+
2. Quadratic decomposition: with those codes fixed, each sub-block vector is
|
| 11 |
+
projected in closed form onto span{q, 1} by solving the 2x2 normal
|
| 12 |
+
equations of sum_i (x_i - s q_i + o)^2 (the exact least-squares
|
| 13 |
+
factorization x ~ s*q - o; degenerate all-equal-code sub-blocks fall back
|
| 14 |
+
to the RTN grid).
|
| 15 |
+
3. The fitted (s, o) are snapped to the Q2_K integer/fp16 parameter lattice
|
| 16 |
+
(d = max s / 15, ls = round(s/d), dmin = max o / 15, lm = round(o/dmin))
|
| 17 |
+
and the codes are re-derived once against that final grid.
|
| 18 |
+
4. Per block, the decomposition is kept only if its squared error does not
|
| 19 |
+
exceed the RTN reconstruction's, so the output is never worse than RTN.
|
| 20 |
+
|
| 21 |
+
An optional llama.cpp importance matrix (.dat) supplies a per-weight factor w:
|
| 22 |
+
the projection then minimizes sum_i w_i (x_i - s q_i + o)^2 (same closed
|
| 23 |
+
form, weighted 2x2 normal equations), the per-block guard compares weighted
|
| 24 |
+
errors, and the reported metrics become importance-weighted. Tensors absent
|
| 25 |
+
from the matrix are quantized with uniform weights.
|
| 26 |
+
|
| 27 |
+
Packing, reconstruction metrics (SMQ and RTN baselines) and optional .npy
|
| 28 |
+
reconstructions all happen inside the same pass; there are no extra stages.
|
| 29 |
+
|
| 30 |
+
Requirements:
|
| 31 |
+
pip install numpy
|
| 32 |
+
# Install gguf-py from the same llama.cpp checkout used by your runtime:
|
| 33 |
+
pip install -e /path/to/llama.cpp/gguf-py
|
| 34 |
+
|
| 35 |
+
Examples:
|
| 36 |
+
python smq_gguf_convert.py model-f16.gguf --output-gguf model-q2k.gguf
|
| 37 |
+
python smq_gguf_convert.py model-f16.gguf --imatrix imatrix.dat --output-gguf model-q2k.gguf
|
| 38 |
+
python smq_gguf_convert.py model.gguf --list-tensors
|
| 39 |
+
python smq_gguf_convert.py model.gguf --tensor blk.57.attn_q.weight --output-gguf test.gguf
|
| 40 |
+
python smq_gguf_convert.py --self-test
|
| 41 |
+
|
| 42 |
+
Notes:
|
| 43 |
+
* GGUF tensor shapes in ReaderTensor.shape are GGML dimensions (ne[0],...);
|
| 44 |
+
NumPy's physical row-major view is the reversed shape. Q2_K blocks must be
|
| 45 |
+
aligned within each ne[0] row, so eligibility checks shape[0], not shape[-1].
|
| 46 |
+
* Eligibility is conservative: only weight matrices (two or more GGML
|
| 47 |
+
dimensions, none of them 1) whose ne[0] is a multiple of 256 are quantized.
|
| 48 |
+
1-D elementwise parameters (norms, biases, ...) are always copied
|
| 49 |
+
unchanged: llama.cpp applies them with ggml_mul/ggml_add, which require
|
| 50 |
+
floating-point operands ("binary_op: unsupported types: dst: f32,
|
| 51 |
+
src0: f32, src1: q2_K" if a norm weight is quantized).
|
| 52 |
+
* Q2_K uses 256 weights per block and 84 bytes per block: 16 packed
|
| 53 |
+
scale/min bytes, 64 code bytes, and two FP16 superblock parameters.
|
| 54 |
+
"""
|
| 55 |
+
|
| 56 |
+
from __future__ import annotations
|
| 57 |
+
|
| 58 |
+
import argparse
|
| 59 |
+
import csv
|
| 60 |
+
import json
|
| 61 |
+
import math
|
| 62 |
+
import os
|
| 63 |
+
import re
|
| 64 |
+
import struct
|
| 65 |
+
import sys
|
| 66 |
+
import tempfile
|
| 67 |
+
import threading
|
| 68 |
+
import time
|
| 69 |
+
from contextlib import contextmanager
|
| 70 |
+
from pathlib import Path
|
| 71 |
+
from typing import Any
|
| 72 |
+
|
| 73 |
+
import numpy as np
|
| 74 |
+
|
| 75 |
+
QK = 256
|
| 76 |
+
NSUB = 16
|
| 77 |
+
_TINY = 1.0e-30
|
| 78 |
+
Q2K_BYTES = 84
|
| 79 |
+
Q2K_FILE_TYPE = 10 # LLAMA_FTYPE_MOSTLY_Q2_K
|
| 80 |
+
GGML_QUANT_VERSION = 2
|
| 81 |
+
|
| 82 |
+
# Sub-block reductions (16-wide inner axis) are scalar loops in numpy and
|
| 83 |
+
# dominate runtime, while torch's CPU kernels vectorize them ~10x faster. We
|
| 84 |
+
# only ever use torch here as a drop-in reducer on numpy buffers (zero-copy
|
| 85 |
+
# views, no torch tensors escape), so it stays an optional accelerator.
|
| 86 |
+
_TORCH = None
|
| 87 |
+
_TORCH_READY = False
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _torch_cpu():
|
| 91 |
+
global _TORCH, _TORCH_READY
|
| 92 |
+
if not _TORCH_READY:
|
| 93 |
+
_TORCH_READY = True
|
| 94 |
+
try:
|
| 95 |
+
import torch # noqa: PLC0415 - optional accelerator, imported lazily
|
| 96 |
+
|
| 97 |
+
_TORCH = torch
|
| 98 |
+
except Exception: # pragma: no cover - numpy fallback
|
| 99 |
+
_TORCH = None
|
| 100 |
+
return _TORCH
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _aminmax_last(x3: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
| 104 |
+
"""Per-block (min, max) over each 16-wide sub-block, shape (n, 16)."""
|
| 105 |
+
t = _torch_cpu()
|
| 106 |
+
if t is not None:
|
| 107 |
+
lo, hi = t.aminmax(t.from_numpy(x3), dim=2)
|
| 108 |
+
return lo.numpy(), hi.numpy()
|
| 109 |
+
return x3.min(axis=2), x3.max(axis=2)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def _sum_last(x3: np.ndarray) -> np.ndarray:
|
| 113 |
+
t = _torch_cpu()
|
| 114 |
+
if t is not None:
|
| 115 |
+
return t.sum(t.from_numpy(x3), dim=2).numpy()
|
| 116 |
+
return x3.sum(axis=2)
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _sq_sum_last(x3: np.ndarray) -> np.ndarray:
|
| 120 |
+
t = _torch_cpu()
|
| 121 |
+
if t is not None:
|
| 122 |
+
tt = t.from_numpy(x3)
|
| 123 |
+
return t.einsum("bij,bij->bi", tt, tt).numpy()
|
| 124 |
+
return (x3 * x3).sum(axis=2)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _dot_last(x3: np.ndarray, y3: np.ndarray) -> np.ndarray:
|
| 128 |
+
t = _torch_cpu()
|
| 129 |
+
if t is not None:
|
| 130 |
+
return t.einsum("bij,bij->bi", t.from_numpy(x3), t.from_numpy(y3)).numpy()
|
| 131 |
+
return (x3 * y3).sum(axis=2)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def _abs_sum_max(diff: np.ndarray, w: np.ndarray | None = None) -> tuple[float, float]:
|
| 135 |
+
t = _torch_cpu()
|
| 136 |
+
d3 = diff if w is None else diff.reshape(-1, NSUB, 16)
|
| 137 |
+
if t is not None:
|
| 138 |
+
td = t.from_numpy(d3)
|
| 139 |
+
ad = t.abs(td)
|
| 140 |
+
total = ad if w is None else ad * t.from_numpy(w)
|
| 141 |
+
return float(total.sum()), float(ad.max())
|
| 142 |
+
ad = np.abs(d3)
|
| 143 |
+
if w is None:
|
| 144 |
+
return float(ad.sum(dtype=np.float64)), float(ad.max())
|
| 145 |
+
return float((ad * w).sum(dtype=np.float64)), float(ad.max())
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _einsum_last(*arrs: np.ndarray) -> np.ndarray:
|
| 149 |
+
"""Per-sub-block sum of the elementwise product of the given (n,16,16)s.
|
| 150 |
+
|
| 151 |
+
With one argument this is a plain sum over the 16-wide axis. The multi-
|
| 152 |
+
operand form is used for weighted moments (e.g. w*x*q) and stays fused in
|
| 153 |
+
torch so no full-size temporary is materialized.
|
| 154 |
+
"""
|
| 155 |
+
t = _torch_cpu()
|
| 156 |
+
if t is not None:
|
| 157 |
+
tensors = [t.from_numpy(a) for a in arrs]
|
| 158 |
+
return t.einsum(",".join("bij" for _ in tensors) + "->bi", *tensors).numpy()
|
| 159 |
+
acc = np.array(arrs[0], dtype=np.float32, copy=True)
|
| 160 |
+
for a in arrs[1:]:
|
| 161 |
+
acc *= a
|
| 162 |
+
return acc.sum(axis=2)
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
IMATRIX_MAGIC = 0x696D6174 # legacy "imat" .dat
|
| 166 |
+
GGUF_MAGIC = 0x46554747 # "GGUF"
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
class FullWeights:
|
| 170 |
+
"""Per-element importance with one value per tensor element."""
|
| 171 |
+
|
| 172 |
+
__slots__ = ("flat",)
|
| 173 |
+
|
| 174 |
+
def __init__(self, flat: np.ndarray):
|
| 175 |
+
self.flat = flat
|
| 176 |
+
|
| 177 |
+
def chunk(self, c0: int, c1: int, qk: int = QK) -> np.ndarray:
|
| 178 |
+
return np.array(
|
| 179 |
+
self.flat[c0 * qk:c1 * qk].reshape(c1 - c0, qk), dtype=np.float32, copy=True
|
| 180 |
+
)
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
class ChannelWeights:
|
| 184 |
+
"""Per-input-channel importance, repeated across every output row.
|
| 185 |
+
|
| 186 |
+
Newer llama.cpp importance matrices store one value per input channel
|
| 187 |
+
(``<tensor>.in_sum2``) instead of one per element. Q2_K blocks are 256
|
| 188 |
+
consecutive channels along the contiguous (ne[0]) axis, so a block's
|
| 189 |
+
weight profile is the same slice in every row and can be tiled on demand
|
| 190 |
+
without materializing the full per-element array.
|
| 191 |
+
"""
|
| 192 |
+
|
| 193 |
+
__slots__ = ("values", "ne0")
|
| 194 |
+
|
| 195 |
+
def __init__(self, values: np.ndarray, ne0: int):
|
| 196 |
+
self.values = np.ascontiguousarray(values, dtype=np.float32)
|
| 197 |
+
self.ne0 = int(ne0)
|
| 198 |
+
|
| 199 |
+
def chunk(self, c0: int, c1: int, qk: int = QK) -> np.ndarray:
|
| 200 |
+
count = (c1 - c0) * qk
|
| 201 |
+
off = (c0 * qk) % self.ne0
|
| 202 |
+
v = self.values if off == 0 else np.concatenate((self.values[off:], self.values[:off]))
|
| 203 |
+
reps = (count + self.ne0 - 1) // self.ne0
|
| 204 |
+
return np.tile(v, reps)[:count].reshape(c1 - c0, qk).astype(np.float32, copy=False)
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
class ImatrixFile:
|
| 208 |
+
"""Importance matrix reader supporting both llama.cpp formats.
|
| 209 |
+
|
| 210 |
+
* Legacy ``.dat``: int32 magic "imat", int32 version, int32 n_entries, then
|
| 211 |
+
per entry name_len/name/ncall/nval/values. Memory-mapped so huge files
|
| 212 |
+
are not loaded wholesale.
|
| 213 |
+
* GGUF (``general.type = imatrix``): per-tensor ``<name>.in_sum2`` vectors
|
| 214 |
+
(one value per ne[0] input channel) and ``<name>.counts`` scalars. Values
|
| 215 |
+
are divided by their token count to give mean-squared activations.
|
| 216 |
+
|
| 217 |
+
``weights(name, ggml_shape)`` returns None when the tensor is absent, or a
|
| 218 |
+
FullWeights/ChannelWeights object that yields per-block weights for a chunk.
|
| 219 |
+
"""
|
| 220 |
+
|
| 221 |
+
def __init__(self, path: Path):
|
| 222 |
+
self.path = Path(path)
|
| 223 |
+
with open(self.path, "rb") as handle:
|
| 224 |
+
head = handle.read(4)
|
| 225 |
+
if len(head) < 4:
|
| 226 |
+
raise ValueError(f"{self.path}: file is too short to be an importance matrix")
|
| 227 |
+
self.kind = "gguf" if struct.unpack("<I", head)[0] == GGUF_MAGIC else "dat"
|
| 228 |
+
if self.kind == "gguf":
|
| 229 |
+
self._init_gguf()
|
| 230 |
+
else:
|
| 231 |
+
self._init_dat()
|
| 232 |
+
|
| 233 |
+
# -- legacy .dat -------------------------------------------------------
|
| 234 |
+
def _init_dat(self) -> None:
|
| 235 |
+
self._mm = np.memmap(self.path, dtype=np.uint8, mode="r")
|
| 236 |
+
self.entries: dict[str, tuple[int, int, int]] = {}
|
| 237 |
+
mm = self._mm
|
| 238 |
+
pos = 0
|
| 239 |
+
|
| 240 |
+
def take(n: int) -> bytes:
|
| 241 |
+
nonlocal pos
|
| 242 |
+
if pos + n > mm.size:
|
| 243 |
+
raise ValueError(f"truncated imatrix file {self.path}")
|
| 244 |
+
chunk = mm[pos:pos + n].tobytes()
|
| 245 |
+
pos += n
|
| 246 |
+
return chunk
|
| 247 |
+
|
| 248 |
+
magic, version, n_entries = struct.unpack("<iii", take(12))
|
| 249 |
+
if magic != IMATRIX_MAGIC:
|
| 250 |
+
raise ValueError(f"{self.path}: bad imatrix magic 0x{magic:08x}")
|
| 251 |
+
if version not in (1, 2):
|
| 252 |
+
raise ValueError(f"{self.path}: unsupported imatrix version {version}")
|
| 253 |
+
if n_entries < 0:
|
| 254 |
+
raise ValueError(f"{self.path}: invalid imatrix entry count {n_entries}")
|
| 255 |
+
for _ in range(n_entries):
|
| 256 |
+
(name_len,) = struct.unpack("<i", take(4))
|
| 257 |
+
name = take(name_len).decode("utf-8", "replace")
|
| 258 |
+
ncall, nval = struct.unpack("<ii", take(8))
|
| 259 |
+
if nval < 0:
|
| 260 |
+
raise ValueError(f"{self.path}: invalid value count {nval} for {name!r}")
|
| 261 |
+
offset = pos
|
| 262 |
+
pos += 4 * nval
|
| 263 |
+
self.entries[name] = (offset, nval, ncall)
|
| 264 |
+
self._entry_count = len(self.entries)
|
| 265 |
+
|
| 266 |
+
# -- GGUF --------------------------------------------------------------
|
| 267 |
+
def _init_gguf(self) -> None:
|
| 268 |
+
_gguf, GGUFReader, *_rest = load_gguf()
|
| 269 |
+
self._reader = GGUFReader(str(self.path))
|
| 270 |
+
type_field = self._reader.fields.get("general.type")
|
| 271 |
+
if type_field is None or str(type_field.contents()) != "imatrix":
|
| 272 |
+
raise ValueError(
|
| 273 |
+
f"{self.path}: GGUF is not an importance matrix (general.type != 'imatrix')"
|
| 274 |
+
)
|
| 275 |
+
self._in: dict[str, Any] = {}
|
| 276 |
+
self._counts: dict[str, Any] = {}
|
| 277 |
+
for tensor in self._reader.tensors:
|
| 278 |
+
if tensor.name.endswith(".in_sum2"):
|
| 279 |
+
self._in[tensor.name[: -len(".in_sum2")]] = tensor
|
| 280 |
+
elif tensor.name.endswith(".counts"):
|
| 281 |
+
self._counts[tensor.name[: -len(".counts")]] = tensor
|
| 282 |
+
if not self._in:
|
| 283 |
+
raise ValueError(f"{self.path}: imatrix GGUF contains no <tensor>.in_sum2 entries")
|
| 284 |
+
self._entry_count = len(self._in)
|
| 285 |
+
|
| 286 |
+
# -- public ------------------------------------------------------------
|
| 287 |
+
@property
|
| 288 |
+
def entry_count(self) -> int:
|
| 289 |
+
return self._entry_count
|
| 290 |
+
|
| 291 |
+
def names(self) -> list[str]:
|
| 292 |
+
if self.kind == "gguf":
|
| 293 |
+
return sorted(self._in)
|
| 294 |
+
return list(self.entries)
|
| 295 |
+
|
| 296 |
+
def weights(self, name: str, ggml_shape: tuple[int, ...]):
|
| 297 |
+
ne0 = int(ggml_shape[0]) if ggml_shape else 0
|
| 298 |
+
total = int(np.prod(ggml_shape, dtype=np.int64)) if ggml_shape else 0
|
| 299 |
+
if self.kind == "dat":
|
| 300 |
+
entry = self.entries.get(name)
|
| 301 |
+
if entry is None:
|
| 302 |
+
return None
|
| 303 |
+
offset, nval, _ncall = entry
|
| 304 |
+
values = np.frombuffer(self._mm, dtype="<f4", count=nval, offset=offset)
|
| 305 |
+
else:
|
| 306 |
+
tensor = self._in.get(name)
|
| 307 |
+
if tensor is None:
|
| 308 |
+
return None
|
| 309 |
+
values = np.asarray(tensor.data, dtype=np.float32).reshape(-1)
|
| 310 |
+
counts = self._counts.get(name)
|
| 311 |
+
if counts is not None:
|
| 312 |
+
ncall = float(np.asarray(counts.data, dtype=np.float64).reshape(-1)[0])
|
| 313 |
+
if ncall > 0:
|
| 314 |
+
values = values / ncall
|
| 315 |
+
if total and values.size == total:
|
| 316 |
+
return FullWeights(values)
|
| 317 |
+
if ne0 and values.size == ne0:
|
| 318 |
+
return ChannelWeights(values, ne0)
|
| 319 |
+
raise ValueError(
|
| 320 |
+
f"imatrix entry for {name!r} has {values.size} values; "
|
| 321 |
+
f"expected {total} (per element) or ne[0]={ne0} (per channel)"
|
| 322 |
+
)
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
@contextmanager
|
| 326 |
+
def progress_stage(label: str, prefix: str, interval: float = 5.0, quiet: bool = False):
|
| 327 |
+
"""Print a stage start/end plus periodic heartbeats for long blocking work."""
|
| 328 |
+
started = time.perf_counter()
|
| 329 |
+
if quiet:
|
| 330 |
+
yield
|
| 331 |
+
return
|
| 332 |
+
|
| 333 |
+
stop_event = threading.Event()
|
| 334 |
+
|
| 335 |
+
def heartbeat() -> None:
|
| 336 |
+
while not stop_event.wait(max(0.5, float(interval))):
|
| 337 |
+
elapsed = time.perf_counter() - started
|
| 338 |
+
print(f"{prefix} {label}: still working ({elapsed:.0f}s elapsed)", flush=True)
|
| 339 |
+
|
| 340 |
+
print(f"{prefix} {label} ...", flush=True)
|
| 341 |
+
thread = threading.Thread(target=heartbeat, name="smq-progress-heartbeat", daemon=True)
|
| 342 |
+
thread.start()
|
| 343 |
+
try:
|
| 344 |
+
yield
|
| 345 |
+
finally:
|
| 346 |
+
stop_event.set()
|
| 347 |
+
thread.join(timeout=max(1.0, min(float(interval) + 1.0, 6.0)))
|
| 348 |
+
elapsed = time.perf_counter() - started
|
| 349 |
+
print(f"{prefix} {label}: done ({elapsed:.2f}s)", flush=True)
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
def load_gguf():
|
| 353 |
+
try:
|
| 354 |
+
import gguf
|
| 355 |
+
except ImportError as exc:
|
| 356 |
+
raise SystemExit(
|
| 357 |
+
"Missing llama.cpp gguf-py package. Install the gguf-py package from "
|
| 358 |
+
"your llama.cpp checkout, for example:\n"
|
| 359 |
+
" pip install -e /path/to/llama.cpp/gguf-py"
|
| 360 |
+
) from exc
|
| 361 |
+
|
| 362 |
+
GGUFReader = getattr(gguf, "GGUFReader", None)
|
| 363 |
+
if GGUFReader is None:
|
| 364 |
+
from gguf.gguf_reader import GGUFReader
|
| 365 |
+
|
| 366 |
+
dequantize_fn = getattr(gguf, "dequantize", None)
|
| 367 |
+
if dequantize_fn is None:
|
| 368 |
+
from gguf.quants import dequantize as dequantize_fn
|
| 369 |
+
|
| 370 |
+
GGUFWriter = getattr(gguf, "GGUFWriter", None)
|
| 371 |
+
if GGUFWriter is None:
|
| 372 |
+
from gguf.gguf_writer import GGUFWriter
|
| 373 |
+
|
| 374 |
+
try:
|
| 375 |
+
from gguf.constants import GGMLQuantizationType, GGUFValueType, GGUFEndian
|
| 376 |
+
except ImportError:
|
| 377 |
+
GGMLQuantizationType = gguf.GGMLQuantizationType
|
| 378 |
+
GGUFValueType = gguf.GGUFValueType
|
| 379 |
+
GGUFEndian = gguf.GGUFEndian
|
| 380 |
+
|
| 381 |
+
return gguf, GGUFReader, dequantize_fn, GGUFWriter, GGMLQuantizationType, GGUFValueType, GGUFEndian
|
| 382 |
+
|
| 383 |
+
|
| 384 |
+
def tensor_ggml_shape(tensor: Any) -> tuple[int, ...]:
|
| 385 |
+
"""GGML dimensions in stored order: (ne[0], ne[1], ...)."""
|
| 386 |
+
return tuple(int(v) for v in np.asarray(tensor.shape).tolist())
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
def tensor_numpy_shape(tensor: Any) -> tuple[int, ...]:
|
| 390 |
+
"""Physical C-order NumPy shape: (..., ne[1], ne[0])."""
|
| 391 |
+
return tuple(reversed(tensor_ggml_shape(tensor)))
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def _spool_dir_for(path: Path) -> str | None:
|
| 395 |
+
"""Pick a spool directory for gguf-py's temp file on the output filesystem.
|
| 396 |
+
|
| 397 |
+
gguf-py spools tensor payloads to a SpooledTemporaryFile, which lives under
|
| 398 |
+
the process temp dir (often a small tmpfs) once it exceeds 256 MiB. A
|
| 399 |
+
multi-GB Q2_K output overflows that, so keep the spool next to the output.
|
| 400 |
+
"""
|
| 401 |
+
try:
|
| 402 |
+
candidate = path.resolve().parent
|
| 403 |
+
probe = candidate / f".smq-spool-probe-{os.getpid()}"
|
| 404 |
+
probe.write_bytes(b"")
|
| 405 |
+
probe.unlink()
|
| 406 |
+
return str(candidate)
|
| 407 |
+
except OSError:
|
| 408 |
+
return None
|
| 409 |
+
|
| 410 |
+
|
| 411 |
+
def tensor_is_eligible(tensor: Any) -> tuple[bool, str]:
|
| 412 |
+
"""Q2_K eligibility: a weight matrix with ne[0] % 256 == 0.
|
| 413 |
+
|
| 414 |
+
"Weight matrix" means at least two GGML dimensions and no 1-sized
|
| 415 |
+
dimension. Everything else is left byte-identical (pass-through). In
|
| 416 |
+
particular, 1-D tensors are elementwise parameters (norms, biases, ...):
|
| 417 |
+
llama.cpp multiplies/adds them directly, and those ops require
|
| 418 |
+
floating-point operands, so quantizing them to Q2_K breaks inference with
|
| 419 |
+
"binary_op: unsupported types: dst: f32, src0: f32, src1: q2_K".
|
| 420 |
+
"""
|
| 421 |
+
shape = tensor_ggml_shape(tensor)
|
| 422 |
+
if not shape:
|
| 423 |
+
return False, "scalar"
|
| 424 |
+
ne0 = shape[0]
|
| 425 |
+
n = int(np.prod(shape, dtype=np.int64))
|
| 426 |
+
if n == 0:
|
| 427 |
+
return False, "empty tensor"
|
| 428 |
+
if ne0 % QK:
|
| 429 |
+
return False, f"GGML row length ne[0]={ne0} is not divisible by {QK}"
|
| 430 |
+
if n % QK:
|
| 431 |
+
return False, "element count is not divisible by 256"
|
| 432 |
+
if len(shape) < 2:
|
| 433 |
+
return False, "1-D elementwise parameter (norm/bias); must stay unquantized"
|
| 434 |
+
if any(int(d) == 1 for d in shape[1:]):
|
| 435 |
+
return False, "degenerate dimension (elementwise parameter); must stay unquantized"
|
| 436 |
+
return True, ""
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def load_tensor_f32(tensor: Any, dequantize_fn) -> np.ndarray:
|
| 440 |
+
"""Dequantize a GGUF tensor to its physical NumPy shape, preserving byte order."""
|
| 441 |
+
try:
|
| 442 |
+
arr = np.asarray(dequantize_fn(tensor.data, tensor.tensor_type), dtype=np.float32)
|
| 443 |
+
except Exception as exc:
|
| 444 |
+
raise RuntimeError(
|
| 445 |
+
f"could not dequantize tensor {tensor.name!r} ({tensor.tensor_type}): {exc}"
|
| 446 |
+
) from exc
|
| 447 |
+
expected_shape = tensor_numpy_shape(tensor)
|
| 448 |
+
expected_size = int(np.prod(expected_shape, dtype=np.int64))
|
| 449 |
+
if arr.size != expected_size:
|
| 450 |
+
raise ValueError(
|
| 451 |
+
f"dequantized size mismatch for {tensor.name}: got {arr.size}, "
|
| 452 |
+
f"expected {expected_size} for physical shape {expected_shape}"
|
| 453 |
+
)
|
| 454 |
+
arr = np.ascontiguousarray(arr.reshape(expected_shape), dtype=np.float32)
|
| 455 |
+
if not arr.flags.writeable: # dequantizers may hand back read-only views
|
| 456 |
+
arr = arr.copy()
|
| 457 |
+
if not np.isfinite(arr).all():
|
| 458 |
+
raise ValueError(f"tensor {tensor.name!r} contains NaN or infinity")
|
| 459 |
+
return arr
|
| 460 |
+
|
| 461 |
+
|
| 462 |
+
def _new_metric_accumulator() -> dict[str, float]:
|
| 463 |
+
return {
|
| 464 |
+
"n": 0.0,
|
| 465 |
+
"err2": 0.0,
|
| 466 |
+
"ref2": 0.0,
|
| 467 |
+
"xerr": 0.0,
|
| 468 |
+
"abs_err": 0.0,
|
| 469 |
+
"max_abs": 0.0,
|
| 470 |
+
}
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
def _accumulate_moments(
|
| 474 |
+
acc: dict[str, float],
|
| 475 |
+
n_elements: int,
|
| 476 |
+
err2_blocks: np.ndarray,
|
| 477 |
+
xerr_blocks: np.ndarray,
|
| 478 |
+
ref2_blocks: np.ndarray,
|
| 479 |
+
abs_err: float | None = None,
|
| 480 |
+
max_abs: float | None = None,
|
| 481 |
+
) -> None:
|
| 482 |
+
"""Accumulate metrics from per-sub-block moment sums (no residual arrays).
|
| 483 |
+
|
| 484 |
+
The residual metrics are recovered algebraically from second moments:
|
| 485 |
+
with diff = rec - ref, sum(diff^2) is the SSE, sum(ref*diff) = xerr, and
|
| 486 |
+
sum(ref^2) = ref2, so rmse/rel_rmse/cosine need no reconstruction pass.
|
| 487 |
+
``abs_err``/``max_abs`` are supplied only when a diff array exists.
|
| 488 |
+
"""
|
| 489 |
+
acc["n"] += float(n_elements)
|
| 490 |
+
acc["err2"] += float(np.asarray(err2_blocks, np.float64).sum())
|
| 491 |
+
acc["xerr"] += float(np.asarray(xerr_blocks, np.float64).sum())
|
| 492 |
+
acc["ref2"] += float(np.asarray(ref2_blocks, np.float64).sum())
|
| 493 |
+
if abs_err is not None:
|
| 494 |
+
acc["abs_err"] += float(abs_err)
|
| 495 |
+
if max_abs is not None:
|
| 496 |
+
acc["max_abs"] = max(acc["max_abs"], float(max_abs))
|
| 497 |
+
|
| 498 |
+
|
| 499 |
+
def _finish_metrics(acc: dict[str, float]) -> dict[str, float]:
|
| 500 |
+
n = max(acc["n"], 1.0)
|
| 501 |
+
rmse = math.sqrt(max(acc["err2"] / n, 0.0))
|
| 502 |
+
rms_ref = math.sqrt(max(acc["ref2"] / n, 0.0))
|
| 503 |
+
# rec = ref - err => sum(ref*rec) = ref2 - xerr, sum(rec^2) = ref2 - 2 xerr + err2
|
| 504 |
+
dot = acc["ref2"] - acc["xerr"]
|
| 505 |
+
rec2 = max(acc["ref2"] - 2.0 * acc["xerr"] + acc["err2"], 0.0)
|
| 506 |
+
denom = math.sqrt(max(acc["ref2"], 0.0) * rec2)
|
| 507 |
+
cosine = dot / denom if denom > 0 else (1.0 if acc["ref2"] == 0 and rec2 == 0 else 0.0)
|
| 508 |
+
return {
|
| 509 |
+
"rmse": rmse,
|
| 510 |
+
"rel_rmse": rmse / max(rms_ref, 1.0e-30),
|
| 511 |
+
"mae": acc["abs_err"] / n,
|
| 512 |
+
"max_abs": acc["max_abs"],
|
| 513 |
+
"cosine": cosine,
|
| 514 |
+
"rms_reference": rms_ref,
|
| 515 |
+
}
|
| 516 |
+
|
| 517 |
+
|
| 518 |
+
def _snap_fp16(v: np.ndarray, floor: float = 0.0, d_floor: float = 2.0 ** -24) -> np.ndarray:
|
| 519 |
+
"""Round to the fp16 storage lattice Q2_K uses for d / dmin."""
|
| 520 |
+
out = np.where(v > 0, v, floor).astype(np.float16).astype(np.float32)
|
| 521 |
+
return np.where(out <= 0, np.float32(d_floor), out)
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
def _reconstruct(d, dmin, ls, lm, q) -> np.ndarray:
|
| 525 |
+
"""Q2_K decode of one chunk of parameters -> (n, 256) float32."""
|
| 526 |
+
q = np.asarray(q, np.float32).reshape(len(np.asarray(d, np.float32)), QK)
|
| 527 |
+
return (
|
| 528 |
+
np.asarray(d, np.float32)[:, None]
|
| 529 |
+
* np.repeat(np.asarray(ls, np.float32), 16, axis=1)
|
| 530 |
+
* q
|
| 531 |
+
- np.asarray(dmin, np.float32)[:, None]
|
| 532 |
+
* np.repeat(np.asarray(lm, np.float32), 16, axis=1)
|
| 533 |
+
)
|
| 534 |
+
|
| 535 |
+
|
| 536 |
+
def _ls_grid_fit(
|
| 537 |
+
xb: np.ndarray,
|
| 538 |
+
q: np.ndarray,
|
| 539 |
+
step: np.ndarray,
|
| 540 |
+
lmin: np.ndarray,
|
| 541 |
+
*,
|
| 542 |
+
Sx: np.ndarray | None = None,
|
| 543 |
+
Sq: np.ndarray | None = None,
|
| 544 |
+
Sqq: np.ndarray | None = None,
|
| 545 |
+
Sxq: np.ndarray | None = None,
|
| 546 |
+
Sw: np.ndarray | None = None,
|
| 547 |
+
):
|
| 548 |
+
"""Exact closed-form least-squares factorization x ~ s*q - o per sub-block.
|
| 549 |
+
|
| 550 |
+
For fixed integer codes q this is the quadratic projection of each 16-wide
|
| 551 |
+
sub-block onto span{q, 1}: solving the 2x2 normal equations of
|
| 552 |
+
sum_i (x_i - s q_i + o)^2 gives
|
| 553 |
+
s = (Sw Sxq - Sq Sx) / (Sw Sqq - Sq^2), o = (Sq Sxq - Sqq Sx) / det,
|
| 554 |
+
with Sw = 16 for uniform weights (importance weighting only changes the
|
| 555 |
+
moments, not the shape of the solution). Q2_K can only subtract a
|
| 556 |
+
nonnegative offset, so o < 0 is clamped to zero and s is refit along that
|
| 557 |
+
boundary (s = Sxq / Sqq). Degenerate sub-blocks (all-equal codes, or weight
|
| 558 |
+
concentrated on one element) are unidentifiable and fall back to the RTN
|
| 559 |
+
grid. Returns (s, o), both nonnegative.
|
| 560 |
+
|
| 561 |
+
The moment sums may be passed in when the caller already has them.
|
| 562 |
+
"""
|
| 563 |
+
if Sq is None:
|
| 564 |
+
Sq = _sum_last(q)
|
| 565 |
+
if Sqq is None:
|
| 566 |
+
Sqq = _sq_sum_last(q)
|
| 567 |
+
if Sx is None:
|
| 568 |
+
Sx = _sum_last(xb)
|
| 569 |
+
if Sxq is None:
|
| 570 |
+
Sxq = _dot_last(xb, q)
|
| 571 |
+
Sqq = np.asarray(Sqq, np.float64)
|
| 572 |
+
Sq = np.asarray(Sq, np.float64)
|
| 573 |
+
Sx = np.asarray(Sx, np.float64)
|
| 574 |
+
Sxq = np.asarray(Sxq, np.float64)
|
| 575 |
+
if Sw is None:
|
| 576 |
+
Sw = np.float64(16.0)
|
| 577 |
+
dw = 16.0
|
| 578 |
+
else:
|
| 579 |
+
Sw = np.asarray(Sw, np.float64)
|
| 580 |
+
dw = np.maximum(Sw, 1.0e-30)
|
| 581 |
+
det = Sw * Sqq - Sq * Sq
|
| 582 |
+
good = det > 1.0e-3 * dw * dw
|
| 583 |
+
det_s = np.where(good, det, 1.0)
|
| 584 |
+
s = np.where(good, (Sw * Sxq - Sq * Sx) / det_s, step)
|
| 585 |
+
o = np.where(good, (Sq * Sxq - Sqq * Sx) / det_s, lmin)
|
| 586 |
+
neg = o < 0.0
|
| 587 |
+
s = np.where(neg & (Sqq > 1.0e-12), Sxq / np.where(Sqq > 1.0e-12, Sqq, 1.0), s)
|
| 588 |
+
o = np.where(neg, 0.0, o)
|
| 589 |
+
s = np.where(s > _TINY, s, step)
|
| 590 |
+
s = np.maximum(s, 0.0)
|
| 591 |
+
o = np.maximum(o, 0.0)
|
| 592 |
+
return s.astype(np.float32), o.astype(np.float32)
|
| 593 |
+
|
| 594 |
+
|
| 595 |
+
def _factorize_chunk(
|
| 596 |
+
xb: np.ndarray,
|
| 597 |
+
rtn_baseline: bool = True,
|
| 598 |
+
need_diff: bool = True,
|
| 599 |
+
w: np.ndarray | None = None,
|
| 600 |
+
) -> tuple[dict[str, np.ndarray], dict[str, np.ndarray] | None, dict[str, np.ndarray], np.ndarray | None]:
|
| 601 |
+
"""Closed-form quadratic factorization of one chunk of blocks.
|
| 602 |
+
|
| 603 |
+
xb: (nb, 16, 16) float32. Returns (params, rtn_params, extras, err_rtn).
|
| 604 |
+
|
| 605 |
+
The pipeline is two exact least-squares projections and one lattice snap:
|
| 606 |
+
1. RTN grid (min/max -> d/dmin fp16, ls/lm) gives codes q0;
|
| 607 |
+
2. q0 is projected onto span{q, 1} in closed form to get continuous
|
| 608 |
+
(s, o) = argmin sum_i w_i (x_i - s q0_i + o)^2 per sub-block;
|
| 609 |
+
3. (s, o) are snapped onto the Q2_K lattice, then the codes are re-derived
|
| 610 |
+
once against that final grid (q1) and the better of decomposition vs
|
| 611 |
+
RTN is kept per block (in the same w-weighted norm).
|
| 612 |
+
|
| 613 |
+
``w`` is an optional (nb, 16, 16) importance factor (uniform when None).
|
| 614 |
+
Residual SSEs and the cross term come from per-sub-block second moments
|
| 615 |
+
(Sw, Sxx, Sx, Sxq, Sq, Sqq), so neither residual array is materialized for
|
| 616 |
+
the guard or the metrics. ``extras`` carries those per-block SSE/xerr/ref2
|
| 617 |
+
moments, plus (diff, sq) only when ``need_diff`` is set.
|
| 618 |
+
"""
|
| 619 |
+
nb = xb.shape[0]
|
| 620 |
+
w3 = None if w is None else np.asarray(w, np.float32)
|
| 621 |
+
lo, hi = _aminmax_last(xb)
|
| 622 |
+
lmin = np.maximum(-lo, 0.0)
|
| 623 |
+
step = np.maximum(hi + lmin, 0.0) / 3.0
|
| 624 |
+
|
| 625 |
+
max_step = step.max(axis=1)
|
| 626 |
+
max_off = lmin.max(axis=1)
|
| 627 |
+
d = _snap_fp16(np.divide(max_step, 15.0, out=np.zeros_like(max_step), where=max_step > 0))
|
| 628 |
+
dmin = _snap_fp16(np.divide(max_off, 15.0, out=np.zeros_like(max_off), where=max_off > 0))
|
| 629 |
+
ls0 = np.rint(step / d[:, None]).clip(0, 15).astype(np.uint8)
|
| 630 |
+
lm0 = np.rint(
|
| 631 |
+
np.divide(lmin, dmin[:, None], out=np.zeros_like(lmin), where=dmin[:, None] > 0)
|
| 632 |
+
).clip(0, 15).astype(np.uint8)
|
| 633 |
+
|
| 634 |
+
s0 = d[:, None] * ls0 # (nb, 16) grid step per sub-block
|
| 635 |
+
o0 = dmin[:, None] * lm0
|
| 636 |
+
safe_s0 = np.where(s0 > _TINY, s0, 1.0e30)
|
| 637 |
+
q0 = xb + o0[:, :, None]
|
| 638 |
+
q0 /= safe_s0[:, :, None]
|
| 639 |
+
np.clip(q0, 0.0, 3.0, out=q0)
|
| 640 |
+
np.rint(q0, out=q0)
|
| 641 |
+
|
| 642 |
+
# Second moments of the reference; reused by every SSE and the metrics.
|
| 643 |
+
if w3 is None:
|
| 644 |
+
Sx = _sum_last(xb)
|
| 645 |
+
Sxx = _sq_sum_last(xb)
|
| 646 |
+
Sw = None
|
| 647 |
+
Sq0 = _sum_last(q0)
|
| 648 |
+
Sqq0 = _sq_sum_last(q0)
|
| 649 |
+
Sxq0 = _dot_last(xb, q0)
|
| 650 |
+
else:
|
| 651 |
+
Sx = _einsum_last(w3, xb)
|
| 652 |
+
Sxx = _einsum_last(w3, xb, xb)
|
| 653 |
+
Sw = _sum_last(w3)
|
| 654 |
+
Sq0 = _einsum_last(w3, q0)
|
| 655 |
+
Sqq0 = _einsum_last(w3, q0, q0)
|
| 656 |
+
Sxq0 = _einsum_last(w3, xb, q0)
|
| 657 |
+
|
| 658 |
+
if rtn_baseline:
|
| 659 |
+
err_rtn = _sse_from_moments(Sxx, Sx, Sxq0, Sq0, Sqq0, s0, o0, Sw=Sw)
|
| 660 |
+
else:
|
| 661 |
+
err_rtn = None
|
| 662 |
+
|
| 663 |
+
# Exact LS factorization x ~ s*q0 - o for the RTN codes, reusing moments.
|
| 664 |
+
s1, o1 = _ls_grid_fit(
|
| 665 |
+
xb, q0, step, lmin, Sx=Sx, Sq=Sq0, Sqq=Sqq0, Sxq=Sxq0, Sw=Sw
|
| 666 |
+
)
|
| 667 |
+
|
| 668 |
+
# Snap the fitted (s, o) onto the Q2_K integer/fp16 parameter lattice and
|
| 669 |
+
# re-derive the codes once against that final grid.
|
| 670 |
+
d2 = _snap_fp16(np.divide(s1.max(axis=1), 15.0, out=np.zeros(nb), where=s1.max(axis=1) > 0))
|
| 671 |
+
dmin2 = _snap_fp16(np.divide(o1.max(axis=1), 15.0, out=np.zeros(nb), where=o1.max(axis=1) > 0))
|
| 672 |
+
ls1 = np.rint(s1 / d2[:, None]).clip(0, 15).astype(np.uint8)
|
| 673 |
+
lm1 = np.rint(
|
| 674 |
+
np.divide(o1, dmin2[:, None], out=np.zeros_like(o1), where=dmin2[:, None] > 0)
|
| 675 |
+
).clip(0, 15).astype(np.uint8)
|
| 676 |
+
s2 = d2[:, None] * ls1
|
| 677 |
+
o2 = dmin2[:, None] * lm1
|
| 678 |
+
safe_s2 = np.where(s2 > _TINY, s2, 1.0e30)
|
| 679 |
+
q1 = xb + o2[:, :, None]
|
| 680 |
+
q1 /= safe_s2[:, :, None]
|
| 681 |
+
np.clip(q1, 0.0, 3.0, out=q1)
|
| 682 |
+
np.rint(q1, out=q1)
|
| 683 |
+
|
| 684 |
+
if w3 is None:
|
| 685 |
+
Sq1 = _sum_last(q1)
|
| 686 |
+
Sqq1 = _sq_sum_last(q1)
|
| 687 |
+
Sxq1 = _dot_last(xb, q1)
|
| 688 |
+
else:
|
| 689 |
+
Sq1 = _einsum_last(w3, q1)
|
| 690 |
+
Sqq1 = _einsum_last(w3, q1, q1)
|
| 691 |
+
Sxq1 = _einsum_last(w3, xb, q1)
|
| 692 |
+
err_dec = _sse_from_moments(Sxx, Sx, Sxq1, Sq1, Sqq1, s2, o2, Sw=Sw)
|
| 693 |
+
# sum(w * x * (rec - x)) = s*Sxq - o*Sx - Sxx, used for residual metrics.
|
| 694 |
+
xerr_dec = (
|
| 695 |
+
np.asarray(s2, np.float64) * Sxq1
|
| 696 |
+
- np.asarray(o2, np.float64) * Sx
|
| 697 |
+
- Sxx.astype(np.float64)
|
| 698 |
+
)
|
| 699 |
+
|
| 700 |
+
if rtn_baseline:
|
| 701 |
+
# The guard is per 256-weight block: compare the block-total SSEs.
|
| 702 |
+
take = err_dec.sum(axis=1) <= err_rtn.sum(axis=1)
|
| 703 |
+
else:
|
| 704 |
+
take = None
|
| 705 |
+
|
| 706 |
+
if take is None or bool(take.all()):
|
| 707 |
+
d_out, dmin_out = d2, dmin2
|
| 708 |
+
ls_out, lm_out = ls1, lm1
|
| 709 |
+
codes_out = q1.astype(np.uint8).reshape(nb, QK)
|
| 710 |
+
else:
|
| 711 |
+
d_out = np.where(take, d2, d)
|
| 712 |
+
dmin_out = np.where(take, dmin2, dmin)
|
| 713 |
+
ls_out = np.where(take[:, None], ls1, ls0)
|
| 714 |
+
lm_out = np.where(take[:, None], lm1, lm0)
|
| 715 |
+
codes_out = np.where(take[:, None, None], q1, q0).astype(np.uint8).reshape(nb, QK)
|
| 716 |
+
|
| 717 |
+
params = {
|
| 718 |
+
"d": np.asarray(d_out, np.float64),
|
| 719 |
+
"dmin": np.asarray(dmin_out, np.float64),
|
| 720 |
+
"ls": ls_out.astype(np.uint8),
|
| 721 |
+
"lm": lm_out.astype(np.uint8),
|
| 722 |
+
"codes": codes_out,
|
| 723 |
+
}
|
| 724 |
+
rtn_params = None if not rtn_baseline else {
|
| 725 |
+
"d": d.astype(np.float64),
|
| 726 |
+
"dmin": dmin.astype(np.float64),
|
| 727 |
+
"ls": ls0,
|
| 728 |
+
"lm": lm0,
|
| 729 |
+
"codes": q0.astype(np.uint8).reshape(nb, QK),
|
| 730 |
+
}
|
| 731 |
+
|
| 732 |
+
diff = None
|
| 733 |
+
sq = None
|
| 734 |
+
if need_diff:
|
| 735 |
+
diff = s2[:, :, None] * q1
|
| 736 |
+
np.subtract(xb, diff, out=diff)
|
| 737 |
+
diff += o2[:, :, None]
|
| 738 |
+
diff = diff.reshape(nb, QK)
|
| 739 |
+
sq = diff * diff
|
| 740 |
+
|
| 741 |
+
extras = {
|
| 742 |
+
"err_dec": err_dec,
|
| 743 |
+
"xerr_dec": xerr_dec,
|
| 744 |
+
"ref2": Sxx,
|
| 745 |
+
"err_rtn": err_rtn,
|
| 746 |
+
"Sw": Sw,
|
| 747 |
+
"diff": diff,
|
| 748 |
+
"sq": sq,
|
| 749 |
+
}
|
| 750 |
+
return params, rtn_params, extras, err_rtn
|
| 751 |
+
|
| 752 |
+
|
| 753 |
+
def _sse_from_moments(Sxx, Sx, Sxq, Sq, Sqq, s, o, Sw=None) -> np.ndarray:
|
| 754 |
+
"""Exact sum_i w_i (x_i - s q_i + o)^2 from second moments (per sub-block)."""
|
| 755 |
+
x2 = np.asarray(Sxx, np.float64)
|
| 756 |
+
x1 = np.asarray(Sx, np.float64)
|
| 757 |
+
xq = np.asarray(Sxq, np.float64)
|
| 758 |
+
q1 = np.asarray(Sq, np.float64)
|
| 759 |
+
q2 = np.asarray(Sqq, np.float64)
|
| 760 |
+
s = np.asarray(s, np.float64)
|
| 761 |
+
o = np.asarray(o, np.float64)
|
| 762 |
+
sw = np.float64(16.0) if Sw is None else np.asarray(Sw, np.float64)
|
| 763 |
+
err = (
|
| 764 |
+
x2
|
| 765 |
+
- 2.0 * s * xq
|
| 766 |
+
+ 2.0 * o * x1
|
| 767 |
+
+ s * s * q2
|
| 768 |
+
- 2.0 * s * o * q1
|
| 769 |
+
+ sw * o * o
|
| 770 |
+
)
|
| 771 |
+
return np.maximum(err, 0.0)
|
| 772 |
+
|
| 773 |
+
|
| 774 |
+
def factorize_q2k(
|
| 775 |
+
x: np.ndarray,
|
| 776 |
+
chunk_blocks: int = 32768,
|
| 777 |
+
save_path: Path | None = None,
|
| 778 |
+
save_shape: tuple[int, ...] | None = None,
|
| 779 |
+
progress=None,
|
| 780 |
+
rtn_baseline: bool = True,
|
| 781 |
+
metrics: bool = True,
|
| 782 |
+
weights: Any = None,
|
| 783 |
+
) -> tuple[np.ndarray, dict[str, float] | None, dict[str, float] | None, float]:
|
| 784 |
+
"""Single-pass factorization of (n_blocks, 256) weights to packed Q2_K.
|
| 785 |
+
|
| 786 |
+
Packing, residual metrics and optional .npy reconstruction streaming all
|
| 787 |
+
happen in the same loop. With ``rtn_baseline`` the round-to-nearest
|
| 788 |
+
baseline error is measured from moments in the same pass (and the per-block
|
| 789 |
+
guard keeps the better of the two); without it the pass is pure quadratic
|
| 790 |
+
decomposition. ``weights`` is an optional per-element importance array
|
| 791 |
+
(same shape as ``x``) that weights the projection, the guard and the
|
| 792 |
+
reported metrics. Returns (packed_blocks, smq_metrics, rtn_metrics_or_None,
|
| 793 |
+
elapsed_seconds); the RTN metrics carry rmse/rel_rmse only and both metric
|
| 794 |
+
dicts are None when ``metrics`` is off.
|
| 795 |
+
"""
|
| 796 |
+
n = int(x.shape[0])
|
| 797 |
+
chunk_blocks = max(1, int(chunk_blocks))
|
| 798 |
+
packed = np.empty((n, Q2K_BYTES), dtype=np.uint8)
|
| 799 |
+
full_weights = isinstance(weights, np.ndarray)
|
| 800 |
+
if full_weights and weights.shape != x.shape:
|
| 801 |
+
raise ValueError(f"weights shape {weights.shape} does not match x shape {x.shape}")
|
| 802 |
+
acc = _new_metric_accumulator() if metrics else None
|
| 803 |
+
rtn_acc = None if not (rtn_baseline and metrics) else {"n": 0.0, "err2": 0.0, "ref2": 0.0}
|
| 804 |
+
want_recon = save_path is not None and save_shape is not None
|
| 805 |
+
need_diff = bool(metrics or want_recon)
|
| 806 |
+
mmap = None
|
| 807 |
+
recon_flat = None
|
| 808 |
+
if want_recon:
|
| 809 |
+
save_path.parent.mkdir(parents=True, exist_ok=True)
|
| 810 |
+
mmap = np.lib.format.open_memmap(save_path, mode="w+", dtype=np.float32, shape=save_shape)
|
| 811 |
+
recon_flat = mmap.reshape(n, QK)
|
| 812 |
+
t0 = time.perf_counter()
|
| 813 |
+
for c0 in range(0, n, chunk_blocks):
|
| 814 |
+
c1 = min(c0 + chunk_blocks, n)
|
| 815 |
+
xb3 = x[c0:c1].reshape(c1 - c0, NSUB, 16)
|
| 816 |
+
if weights is None:
|
| 817 |
+
w3 = None
|
| 818 |
+
elif full_weights:
|
| 819 |
+
w3 = np.array(weights[c0:c1].reshape(c1 - c0, NSUB, 16), dtype=np.float32, copy=True)
|
| 820 |
+
else:
|
| 821 |
+
w3 = np.ascontiguousarray(weights.chunk(c0, c1).reshape(c1 - c0, NSUB, 16), dtype=np.float32)
|
| 822 |
+
params, _rtn_params, extras, _err_rtn = _factorize_chunk(
|
| 823 |
+
xb3, rtn_baseline=rtn_baseline, need_diff=need_diff, w=w3
|
| 824 |
+
)
|
| 825 |
+
packed[c0:c1] = pack_q2k_blocks(params)
|
| 826 |
+
if acc is not None:
|
| 827 |
+
diff = extras["diff"]
|
| 828 |
+
abs_err = max_abs = None
|
| 829 |
+
if diff is not None:
|
| 830 |
+
abs_err, max_abs = _abs_sum_max(diff, w3)
|
| 831 |
+
n_eff = float((c1 - c0) * QK) if w3 is None else float(
|
| 832 |
+
np.asarray(extras["Sw"], np.float64).sum()
|
| 833 |
+
)
|
| 834 |
+
_accumulate_moments(
|
| 835 |
+
acc, n_eff, extras["err_dec"], extras["xerr_dec"],
|
| 836 |
+
extras["ref2"], abs_err, max_abs,
|
| 837 |
+
)
|
| 838 |
+
if rtn_acc is not None:
|
| 839 |
+
rtn_acc["n"] += n_eff
|
| 840 |
+
rtn_acc["ref2"] += float(np.asarray(extras["ref2"], np.float64).sum())
|
| 841 |
+
rtn_acc["err2"] += float(np.asarray(extras["err_rtn"], np.float64).sum())
|
| 842 |
+
if recon_flat is not None:
|
| 843 |
+
recon_flat[c0:c1] = x[c0:c1] - extras["diff"]
|
| 844 |
+
if progress is not None:
|
| 845 |
+
progress(c1, n, f"quadratic factorization {c1:,}/{n:,} blocks")
|
| 846 |
+
if mmap is not None:
|
| 847 |
+
mmap.flush()
|
| 848 |
+
del recon_flat, mmap
|
| 849 |
+
rtn_stats = None
|
| 850 |
+
if rtn_acc is not None:
|
| 851 |
+
nn = max(rtn_acc["n"], 1.0)
|
| 852 |
+
rtn_rmse = math.sqrt(max(rtn_acc["err2"] / nn, 0.0))
|
| 853 |
+
rtn_stats = {
|
| 854 |
+
"rmse": rtn_rmse,
|
| 855 |
+
"rel_rmse": rtn_rmse / max(math.sqrt(rtn_acc["ref2"] / nn), 1.0e-30),
|
| 856 |
+
}
|
| 857 |
+
return packed, (_finish_metrics(acc) if acc is not None else None), rtn_stats, time.perf_counter() - t0
|
| 858 |
+
|
| 859 |
+
|
| 860 |
+
def pack_q2k_blocks(result: dict[str, Any]) -> np.ndarray:
|
| 861 |
+
"""Pack SMQ block parameters into GGML's native 84-byte Q2_K layout.
|
| 862 |
+
|
| 863 |
+
GGML Q2_K block layout:
|
| 864 |
+
scales[16] = (lm[j] << 4) | ls[j]
|
| 865 |
+
qs[64] = packed 2-bit codes in groups of 32 and planes 0..3
|
| 866 |
+
d[2] = FP16 superblock scale
|
| 867 |
+
dmin[2] = FP16 superblock minimum scale
|
| 868 |
+
|
| 869 |
+
The code-plane ordering mirrors gguf-py's Q2_K dequantizer.
|
| 870 |
+
"""
|
| 871 |
+
d = np.asarray(result["d"], dtype=np.float32).astype(np.float16)
|
| 872 |
+
dmin = np.asarray(result["dmin"], dtype=np.float32).astype(np.float16)
|
| 873 |
+
ls = np.asarray(result["ls"], dtype=np.uint8)
|
| 874 |
+
lm = np.asarray(result["lm"], dtype=np.uint8)
|
| 875 |
+
codes = np.asarray(result["codes"], dtype=np.uint8)
|
| 876 |
+
|
| 877 |
+
if ls.ndim != 2 or ls.shape[1] != NSUB or lm.shape != ls.shape:
|
| 878 |
+
raise ValueError(f"invalid Q2_K scale arrays: ls={ls.shape}, lm={lm.shape}")
|
| 879 |
+
n = ls.shape[0]
|
| 880 |
+
if d.shape != (n,) or dmin.shape != (n,) or codes.shape != (n, QK):
|
| 881 |
+
raise ValueError("invalid SMQ result dimensions for Q2_K packing")
|
| 882 |
+
if np.any(ls > 15) or np.any(lm > 15) or np.any(codes > 3):
|
| 883 |
+
raise ValueError("SMQ output contains values outside Q2_K code/scale ranges")
|
| 884 |
+
|
| 885 |
+
scales = ((lm << 4) | ls).astype(np.uint8)
|
| 886 |
+
qs = np.zeros((n, QK // 4), dtype=np.uint8)
|
| 887 |
+
# Q2_K's dequantized subblock order is [group, plane, half-of-32].
|
| 888 |
+
for sub in range(NSUB):
|
| 889 |
+
group = sub // 8
|
| 890 |
+
plane = (sub % 8) // 2
|
| 891 |
+
half = sub % 2
|
| 892 |
+
dst0 = group * 32 + half * 16
|
| 893 |
+
dst1 = dst0 + 16
|
| 894 |
+
src0 = sub * 16
|
| 895 |
+
src1 = src0 + 16
|
| 896 |
+
qs[:, dst0:dst1] |= (codes[:, src0:src1] << (2 * plane)).astype(np.uint8)
|
| 897 |
+
|
| 898 |
+
d_bytes = np.ascontiguousarray(d.astype("<f2", copy=False)).view(np.uint8).reshape(n, 2)
|
| 899 |
+
dm_bytes = np.ascontiguousarray(dmin.astype("<f2", copy=False)).view(np.uint8).reshape(n, 2)
|
| 900 |
+
packed = np.concatenate((scales, qs, d_bytes, dm_bytes), axis=1)
|
| 901 |
+
if packed.shape != (n, Q2K_BYTES):
|
| 902 |
+
raise AssertionError(f"Q2_K block must be {Q2K_BYTES} bytes, got {packed.shape}")
|
| 903 |
+
return np.ascontiguousarray(packed)
|
| 904 |
+
|
| 905 |
+
|
| 906 |
+
def unpack_q2k_blocks_for_test(packed: np.ndarray) -> np.ndarray:
|
| 907 |
+
"""Small independent decoder used by --self-test; output shape (n,256)."""
|
| 908 |
+
p = np.asarray(packed, dtype=np.uint8)
|
| 909 |
+
if p.ndim != 2 or p.shape[1] != Q2K_BYTES:
|
| 910 |
+
raise ValueError("expected packed Q2_K array with shape (n,84)")
|
| 911 |
+
n = p.shape[0]
|
| 912 |
+
scales = p[:, :16]
|
| 913 |
+
qs = p[:, 16:80]
|
| 914 |
+
d = p[:, 80:82].copy().reshape(-1).view("<f2").astype(np.float32)
|
| 915 |
+
dmin = p[:, 82:84].copy().reshape(-1).view("<f2").astype(np.float32)
|
| 916 |
+
# This is the gguf-py dequantization ordering: 2 groups x 4 bit planes x 32.
|
| 917 |
+
shifts = np.array([0, 2, 4, 6], dtype=np.uint8).reshape(1, 1, 4, 1)
|
| 918 |
+
q = ((qs.reshape(n, 2, 1, 32) >> shifts) & np.uint8(3)).reshape(n, 16, 16)
|
| 919 |
+
dl = (d[:, None] * (scales & 0x0F).astype(np.float32)).reshape(n, 16, 1)
|
| 920 |
+
ml = (dmin[:, None] * (scales >> 4).astype(np.float32)).reshape(n, 16, 1)
|
| 921 |
+
return (dl * q.astype(np.float32) - ml).reshape(n, QK)
|
| 922 |
+
|
| 923 |
+
|
| 924 |
+
def safe_filename(name: str) -> str:
|
| 925 |
+
cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "_", name).strip("._")
|
| 926 |
+
return cleaned[:180] or "tensor"
|
| 927 |
+
|
| 928 |
+
|
| 929 |
+
def copy_gguf_metadata(reader: Any, writer: Any, GGUFValueType: Any) -> None:
|
| 930 |
+
"""Copy source metadata without copying GGUF structural counters."""
|
| 931 |
+
# The writer constructor emits general.architecture; these two fields are
|
| 932 |
+
# deliberately regenerated below. Copying and then re-adding file_type /
|
| 933 |
+
# quantization_version would create duplicate GGUF keys.
|
| 934 |
+
structural = {
|
| 935 |
+
"GGUF.version", "GGUF.tensor_count", "GGUF.kv_count",
|
| 936 |
+
"general.architecture", "general.file_type", "general.quantization_version",
|
| 937 |
+
}
|
| 938 |
+
for key, field in reader.fields.items():
|
| 939 |
+
if key in structural:
|
| 940 |
+
continue
|
| 941 |
+
if not field.types:
|
| 942 |
+
continue
|
| 943 |
+
value_type = field.types[0]
|
| 944 |
+
value = field.contents()
|
| 945 |
+
if value_type == GGUFValueType.ARRAY:
|
| 946 |
+
writer.add_key_value(key, value, value_type, sub_type=field.types[-1])
|
| 947 |
+
else:
|
| 948 |
+
writer.add_key_value(key, value, value_type)
|
| 949 |
+
|
| 950 |
+
# Output is a mostly-Q2_K model. These fields are recognized by llama.cpp.
|
| 951 |
+
writer.add_uint32("general.file_type", Q2K_FILE_TYPE)
|
| 952 |
+
writer.add_uint32("general.quantization_version", GGML_QUANT_VERSION)
|
| 953 |
+
if "general.type" not in reader.fields:
|
| 954 |
+
writer.add_string("general.type", "model")
|
| 955 |
+
|
| 956 |
+
|
| 957 |
+
def add_tensor_passthrough(writer: Any, tensor: Any, reader: Any) -> None:
|
| 958 |
+
"""Copy one tensor in its original GGML type and raw byte representation."""
|
| 959 |
+
data = np.asarray(tensor.data)
|
| 960 |
+
if data.dtype == np.uint8:
|
| 961 |
+
writer.add_tensor(
|
| 962 |
+
tensor.name,
|
| 963 |
+
np.ascontiguousarray(data),
|
| 964 |
+
raw_dtype=tensor.tensor_type,
|
| 965 |
+
tensor_endianess=reader.endianess,
|
| 966 |
+
)
|
| 967 |
+
else:
|
| 968 |
+
writer.add_tensor(
|
| 969 |
+
tensor.name,
|
| 970 |
+
np.ascontiguousarray(data),
|
| 971 |
+
tensor_endianess=reader.endianess,
|
| 972 |
+
)
|
| 973 |
+
|
| 974 |
+
|
| 975 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 976 |
+
p = argparse.ArgumentParser(
|
| 977 |
+
description="Factor eligible GGUF tensors directly into Q2_K by closed-form quadratic decomposition.",
|
| 978 |
+
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
| 979 |
+
)
|
| 980 |
+
p.add_argument("gguf", type=Path, nargs="?", help="Input model GGUF")
|
| 981 |
+
p.add_argument("--output-gguf", "--out-gguf", type=Path,
|
| 982 |
+
help="Write a complete llama.cpp-compatible GGUF with eligible tensors quantized to Q2_K")
|
| 983 |
+
p.add_argument("--list-tensors", action="store_true", help="List Q2_K-row-eligible tensors and exit")
|
| 984 |
+
p.add_argument("--tensor", action="append", default=[],
|
| 985 |
+
help="Select tensor by exact name or substring (repeatable); others are copied unchanged")
|
| 986 |
+
p.add_argument("--tensor-pattern", help="Select tensors using a regular expression; others copied unchanged")
|
| 987 |
+
p.add_argument("--imatrix", type=Path,
|
| 988 |
+
help="Optional llama.cpp importance matrix (.dat); weights the quadratic fit, "
|
| 989 |
+
"per-block guard and reported metrics. Tensors absent from it use uniform weights.")
|
| 990 |
+
p.add_argument("--max-tensors", type=int, default=0,
|
| 991 |
+
help="Stop after N eligible tensors (0 = all); remaining tensors are copied unchanged")
|
| 992 |
+
p.add_argument("--chunk-blocks", type=int, default=32768, help="Blocks per factorization chunk")
|
| 993 |
+
p.add_argument("--progress-interval", type=float, default=5.0,
|
| 994 |
+
help="Seconds between long-stage heartbeat messages (0.5 minimum)")
|
| 995 |
+
p.add_argument("--save-recon", type=Path,
|
| 996 |
+
help="Optional directory to save each reconstruction as float32 .npy")
|
| 997 |
+
p.add_argument("--out", type=Path, default=Path("smq_gguf_bench"),
|
| 998 |
+
help="Output path prefix for .csv and .json reports")
|
| 999 |
+
p.add_argument("--quiet", action="store_true", help="Reduce per-chunk progress output")
|
| 1000 |
+
p.add_argument("--no-rtn-metrics", action="store_true",
|
| 1001 |
+
help="Skip the RTN baseline reconstruction, per-block guard and metrics "
|
| 1002 |
+
"(fastest pure quadratic decomposition; the output is then unguarded)")
|
| 1003 |
+
p.add_argument("--no-metrics", action="store_true",
|
| 1004 |
+
help="Skip residual metrics entirely (no per-element abs/max pass)")
|
| 1005 |
+
p.add_argument("--self-test", action="store_true", help="Run packing and factorization self-tests and exit")
|
| 1006 |
+
return p
|
| 1007 |
+
|
| 1008 |
+
|
| 1009 |
+
def run_self_test() -> int:
|
| 1010 |
+
rng = np.random.default_rng(20260115)
|
| 1011 |
+
|
| 1012 |
+
# 1. Packing roundtrip on adversarial parameter ranges.
|
| 1013 |
+
n = 37
|
| 1014 |
+
result = {
|
| 1015 |
+
"d": rng.uniform(1e-4, 0.15, n).astype(np.float32),
|
| 1016 |
+
"dmin": rng.uniform(0.0, 0.05, n).astype(np.float32),
|
| 1017 |
+
"ls": rng.integers(0, 16, size=(n, 16), dtype=np.uint8),
|
| 1018 |
+
"lm": rng.integers(0, 16, size=(n, 16), dtype=np.uint8),
|
| 1019 |
+
"codes": rng.integers(0, 4, size=(n, 256), dtype=np.uint8),
|
| 1020 |
+
}
|
| 1021 |
+
packed = pack_q2k_blocks(result)
|
| 1022 |
+
decoded = unpack_q2k_blocks_for_test(packed)
|
| 1023 |
+
emitted = dict(result)
|
| 1024 |
+
emitted["d"] = np.asarray(result["d"], dtype=np.float32).astype(np.float16).astype(np.float32)
|
| 1025 |
+
emitted["dmin"] = np.asarray(result["dmin"], dtype=np.float32).astype(np.float16).astype(np.float32)
|
| 1026 |
+
expected = _reconstruct(emitted["d"], emitted["dmin"], emitted["ls"], emitted["lm"], emitted["codes"])
|
| 1027 |
+
if not np.array_equal(decoded, expected):
|
| 1028 |
+
raise AssertionError(f"Q2_K pack/decode mismatch; max error={np.max(np.abs(decoded - expected))}")
|
| 1029 |
+
print(f"Q2_K packing self-test: PASS ({n} blocks, {n * Q2K_BYTES} bytes)")
|
| 1030 |
+
|
| 1031 |
+
# 2. Factorization: roundtrip exactness, guards, and never-worse-than-RTN
|
| 1032 |
+
# across Gaussian, heavy-tailed, constant, zero and outlier blocks.
|
| 1033 |
+
nb = 4096
|
| 1034 |
+
gauss = rng.standard_normal((nb // 4, QK)) * 0.02
|
| 1035 |
+
heavy = rng.standard_t(3, (nb // 4, QK)) * 0.02
|
| 1036 |
+
const = np.repeat((rng.standard_normal((nb // 8, 1)) * 0.02), QK, axis=1)
|
| 1037 |
+
sparse = rng.standard_normal((nb // 8, QK)) * 0.02 * (rng.random((nb // 8, QK)) < 0.05)
|
| 1038 |
+
outs = gauss[: nb - nb // 4 - nb // 8 - nb // 8].copy()
|
| 1039 |
+
outs[:, ::97] += 0.5 # rare large outliers
|
| 1040 |
+
x = np.concatenate((gauss, heavy, const, sparse, outs), axis=0).astype(np.float32)
|
| 1041 |
+
params, rtn, _, _ = _factorize_chunk(x.reshape(-1, NSUB, 16))
|
| 1042 |
+
|
| 1043 |
+
dec = _reconstruct(params["d"], params["dmin"], params["ls"], params["lm"], params["codes"])
|
| 1044 |
+
rec_rtn = _reconstruct(rtn["d"], rtn["dmin"], rtn["ls"], rtn["lm"], rtn["codes"])
|
| 1045 |
+
if not np.isfinite(dec).all():
|
| 1046 |
+
raise AssertionError("factorization produced non-finite reconstruction")
|
| 1047 |
+
if params["codes"].max() > 3 or params["ls"].max() > 15 or params["lm"].max() > 15:
|
| 1048 |
+
raise AssertionError("factorization produced out-of-range Q2_K parameters")
|
| 1049 |
+
if (np.asarray(params["d"]) <= 0).any() or (np.asarray(params["dmin"]) < 0).any():
|
| 1050 |
+
raise AssertionError("factorization produced invalid d/dmin")
|
| 1051 |
+
|
| 1052 |
+
err_dec = ((dec - x) ** 2).reshape(nb, QK).sum(axis=1)
|
| 1053 |
+
err_rtn = ((rec_rtn - x) ** 2).reshape(nb, QK).sum(axis=1)
|
| 1054 |
+
worse = int(np.sum(err_dec > err_rtn * (1.0 + 1e-6)))
|
| 1055 |
+
if worse:
|
| 1056 |
+
raise AssertionError(f"{worse} blocks worse than RTN (guard failed)")
|
| 1057 |
+
|
| 1058 |
+
rel_dec = math.sqrt(float((err_dec.sum() / x.size))) / math.sqrt(float((x * x).mean()))
|
| 1059 |
+
rel_rtn = math.sqrt(float((err_rtn.sum() / x.size))) / math.sqrt(float((x * x).mean()))
|
| 1060 |
+
gain = 100.0 * (1.0 - rel_dec / rel_rtn)
|
| 1061 |
+
if rel_dec > rel_rtn:
|
| 1062 |
+
raise AssertionError(f"factorization worse than RTN overall: {rel_dec} vs {rel_rtn}")
|
| 1063 |
+
print(f"factorization self-test: PASS (blocks never worse than RTN; relRMSE {rel_rtn:.5f} -> {rel_dec:.5f}, {gain:.1f}% better)")
|
| 1064 |
+
return 0
|
| 1065 |
+
|
| 1066 |
+
|
| 1067 |
+
def write_reports(args: argparse.Namespace, rows: list[dict[str, Any]], skipped: list[dict[str, Any]],
|
| 1068 |
+
eligible_count: int, selected_count: int, quantized_count: int,
|
| 1069 |
+
passthrough_count: int) -> tuple[Path, Path]:
|
| 1070 |
+
out_prefix = args.out
|
| 1071 |
+
if out_prefix.suffix.lower() in (".csv", ".json"):
|
| 1072 |
+
out_prefix = out_prefix.with_suffix("")
|
| 1073 |
+
out_prefix.parent.mkdir(parents=True, exist_ok=True)
|
| 1074 |
+
csv_path = out_prefix.with_suffix(".csv")
|
| 1075 |
+
json_path = out_prefix.with_suffix(".json")
|
| 1076 |
+
|
| 1077 |
+
fieldnames = sorted({key for row in rows for key in row.keys() if not isinstance(row[key], (list, dict))})
|
| 1078 |
+
with csv_path.open("w", newline="", encoding="utf-8") as handle:
|
| 1079 |
+
if fieldnames:
|
| 1080 |
+
writer = csv.DictWriter(handle, fieldnames=fieldnames)
|
| 1081 |
+
writer.writeheader()
|
| 1082 |
+
for row in rows:
|
| 1083 |
+
writer.writerow({key: row.get(key, "") for key in fieldnames})
|
| 1084 |
+
|
| 1085 |
+
report = {
|
| 1086 |
+
"input_gguf": str(args.gguf),
|
| 1087 |
+
"output_gguf": str(args.output_gguf) if args.output_gguf else None,
|
| 1088 |
+
"selected_count": selected_count,
|
| 1089 |
+
"quantized_count": quantized_count,
|
| 1090 |
+
"eligible_count": eligible_count,
|
| 1091 |
+
"passthrough_count": passthrough_count,
|
| 1092 |
+
"skipped_count": len(skipped),
|
| 1093 |
+
"results": rows,
|
| 1094 |
+
"skipped_tensors": skipped,
|
| 1095 |
+
}
|
| 1096 |
+
json_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
|
| 1097 |
+
return csv_path, json_path
|
| 1098 |
+
|
| 1099 |
+
|
| 1100 |
+
def main() -> int:
|
| 1101 |
+
args = build_parser().parse_args()
|
| 1102 |
+
if args.self_test:
|
| 1103 |
+
return run_self_test()
|
| 1104 |
+
if args.gguf is None:
|
| 1105 |
+
raise SystemExit("Input model GGUF is required unless --self-test is used")
|
| 1106 |
+
if not args.gguf.is_file():
|
| 1107 |
+
raise SystemExit(f"Input GGUF file does not exist: {args.gguf}")
|
| 1108 |
+
if args.max_tensors < 0 or args.chunk_blocks < 1 or args.progress_interval <= 0:
|
| 1109 |
+
raise SystemExit("max-tensors must be >=0; chunk-blocks positive; progress-interval positive")
|
| 1110 |
+
if args.output_gguf and args.output_gguf.resolve() == args.gguf.resolve():
|
| 1111 |
+
raise SystemExit("Output GGUF must be different from input GGUF")
|
| 1112 |
+
|
| 1113 |
+
gguf, GGUFReader, dequantize_fn, GGUFWriter, GGMLQuantizationType, GGUFValueType, GGUFEndian = load_gguf()
|
| 1114 |
+
reader = GGUFReader(str(args.gguf))
|
| 1115 |
+
imatrix = None
|
| 1116 |
+
if args.imatrix is not None:
|
| 1117 |
+
if not args.imatrix.is_file():
|
| 1118 |
+
raise SystemExit(f"Importance matrix does not exist: {args.imatrix}")
|
| 1119 |
+
imatrix = ImatrixFile(args.imatrix)
|
| 1120 |
+
print(
|
| 1121 |
+
f"Importance matrix: {args.imatrix} "
|
| 1122 |
+
f"({imatrix.entry_count} tensor entries, {imatrix.kind} format)"
|
| 1123 |
+
)
|
| 1124 |
+
imatrix_hits = 0
|
| 1125 |
+
imatrix_misses = 0
|
| 1126 |
+
if args.output_gguf and reader.endianess != GGUFEndian.LITTLE:
|
| 1127 |
+
raise SystemExit(
|
| 1128 |
+
"Writing currently supports little-endian input GGUF files only. "
|
| 1129 |
+
"Convert this model to little-endian GGUF before conversion; "
|
| 1130 |
+
"raw pass-through tensors are not byte-swapped."
|
| 1131 |
+
)
|
| 1132 |
+
eligible: list[Any] = []
|
| 1133 |
+
skipped_shape: list[dict[str, Any]] = []
|
| 1134 |
+
for tensor in reader.tensors:
|
| 1135 |
+
ok, reason = tensor_is_eligible(tensor)
|
| 1136 |
+
if ok:
|
| 1137 |
+
eligible.append(tensor)
|
| 1138 |
+
else:
|
| 1139 |
+
skipped_shape.append({
|
| 1140 |
+
"tensor": tensor.name,
|
| 1141 |
+
"shape_ggml": list(tensor_ggml_shape(tensor)),
|
| 1142 |
+
"reason": reason,
|
| 1143 |
+
"gguf_type": tensor.tensor_type.name,
|
| 1144 |
+
})
|
| 1145 |
+
|
| 1146 |
+
if args.list_tensors:
|
| 1147 |
+
print(f"GGUF: {args.gguf}")
|
| 1148 |
+
print(f"Q2_K row-eligible tensors: {len(eligible)} / {len(reader.tensors)}")
|
| 1149 |
+
print("NAME\tGGML_SHAPE(ne0,...)\tSOURCE_TYPE\tELEMENTS")
|
| 1150 |
+
for tensor in eligible:
|
| 1151 |
+
print(f"{tensor.name}\t{tensor_ggml_shape(tensor)}\t{tensor.tensor_type.name}\t{tensor.n_elements}")
|
| 1152 |
+
if skipped_shape:
|
| 1153 |
+
print(
|
| 1154 |
+
f"\nNot eligible (copied unchanged): {len(skipped_shape)} "
|
| 1155 |
+
f"(ne[0] must be divisible by {QK}; 1-D elementwise parameters excluded)",
|
| 1156 |
+
file=sys.stderr,
|
| 1157 |
+
)
|
| 1158 |
+
for entry in skipped_shape:
|
| 1159 |
+
print(
|
| 1160 |
+
f"{entry['tensor']}\t{entry['shape_ggml']}\t{entry['gguf_type']}\t{entry['reason']}",
|
| 1161 |
+
file=sys.stderr,
|
| 1162 |
+
)
|
| 1163 |
+
return 0
|
| 1164 |
+
|
| 1165 |
+
selected = []
|
| 1166 |
+
for tensor in eligible:
|
| 1167 |
+
if args.tensor and not any((s == tensor.name or s in tensor.name) for s in args.tensor):
|
| 1168 |
+
continue
|
| 1169 |
+
if args.tensor_pattern and re.search(args.tensor_pattern, tensor.name) is None:
|
| 1170 |
+
continue
|
| 1171 |
+
selected.append(tensor)
|
| 1172 |
+
if args.max_tensors:
|
| 1173 |
+
selected = selected[:args.max_tensors]
|
| 1174 |
+
selected_names = {tensor.name for tensor in selected}
|
| 1175 |
+
if not selected:
|
| 1176 |
+
raise SystemExit("No eligible tensors selected. Use --list-tensors to inspect the model.")
|
| 1177 |
+
|
| 1178 |
+
q2k_type = GGMLQuantizationType.Q2_K
|
| 1179 |
+
|
| 1180 |
+
writer = None
|
| 1181 |
+
prev_tempdir: str | None = None
|
| 1182 |
+
spool_dir: str | None = None
|
| 1183 |
+
tempdir_overridden = False
|
| 1184 |
+
tmp_output = None
|
| 1185 |
+
arch = None
|
| 1186 |
+
if args.output_gguf:
|
| 1187 |
+
args.output_gguf.parent.mkdir(parents=True, exist_ok=True)
|
| 1188 |
+
tmp_output = args.output_gguf.with_name(args.output_gguf.name + ".partial")
|
| 1189 |
+
if tmp_output.exists():
|
| 1190 |
+
tmp_output.unlink()
|
| 1191 |
+
arch_field = reader.fields.get("general.architecture")
|
| 1192 |
+
if arch_field is None:
|
| 1193 |
+
raise SystemExit("Input GGUF has no general.architecture metadata; cannot write model GGUF")
|
| 1194 |
+
arch = str(arch_field.contents())
|
| 1195 |
+
spool_dir = _spool_dir_for(args.output_gguf)
|
| 1196 |
+
if spool_dir is not None:
|
| 1197 |
+
prev_tempdir = tempfile.tempdir
|
| 1198 |
+
tempdir_overridden = tempfile.tempdir != spool_dir
|
| 1199 |
+
tempfile.tempdir = spool_dir
|
| 1200 |
+
if not args.quiet:
|
| 1201 |
+
print(f"GGUF tensor spool directory: {spool_dir}", flush=True)
|
| 1202 |
+
writer = GGUFWriter(
|
| 1203 |
+
str(tmp_output),
|
| 1204 |
+
arch=arch,
|
| 1205 |
+
use_temp_file=True,
|
| 1206 |
+
endianess=GGUFEndian.LITTLE,
|
| 1207 |
+
)
|
| 1208 |
+
writer.data_alignment = int(reader.alignment)
|
| 1209 |
+
copy_gguf_metadata(reader, writer, GGUFValueType)
|
| 1210 |
+
|
| 1211 |
+
rows: list[dict[str, Any]] = []
|
| 1212 |
+
skipped: list[dict[str, Any]] = list(skipped_shape)
|
| 1213 |
+
print(
|
| 1214 |
+
f"Selected tensors: {len(selected)} / {len(eligible)} eligible | total={len(reader.tensors)} "
|
| 1215 |
+
f"| output={'Q2_K GGUF' if writer is not None else 'reports only'}"
|
| 1216 |
+
)
|
| 1217 |
+
print("tensor | shape | blocks | Q2K relRMSE | RTN relRMSE | gain | time(s)")
|
| 1218 |
+
print("-" * 112)
|
| 1219 |
+
|
| 1220 |
+
selected_done = 0
|
| 1221 |
+
quantized_count = 0
|
| 1222 |
+
passthrough_count = 0
|
| 1223 |
+
start_total = time.perf_counter()
|
| 1224 |
+
try:
|
| 1225 |
+
for tensor_index, tensor in enumerate(reader.tensors, 1):
|
| 1226 |
+
eligible_tensor, _eligibility_reason = tensor_is_eligible(tensor)
|
| 1227 |
+
should_quantize = eligible_tensor and tensor.name in selected_names
|
| 1228 |
+
|
| 1229 |
+
if not should_quantize:
|
| 1230 |
+
if writer is not None:
|
| 1231 |
+
add_tensor_passthrough(writer, tensor, reader)
|
| 1232 |
+
passthrough_count += 1
|
| 1233 |
+
if not args.quiet and (tensor_index == 1 or tensor_index % 25 == 0 or tensor_index == len(reader.tensors)):
|
| 1234 |
+
pct = 100.0 * tensor_index / max(len(reader.tensors), 1)
|
| 1235 |
+
print(
|
| 1236 |
+
f"[GGUF copy] input tensors {tensor_index}/{len(reader.tensors)} ({pct:.1f}%) "
|
| 1237 |
+
f"| factorized {selected_done}/{len(selected)} | copied unchanged {passthrough_count}",
|
| 1238 |
+
flush=True,
|
| 1239 |
+
)
|
| 1240 |
+
continue
|
| 1241 |
+
|
| 1242 |
+
selected_done += 1
|
| 1243 |
+
tensor_prefix = (
|
| 1244 |
+
f"[{selected_done}/{len(selected)} selected | input {tensor_index}/{len(reader.tensors)}] "
|
| 1245 |
+
f"{tensor.name} shape={tensor_numpy_shape(tensor)}"
|
| 1246 |
+
)
|
| 1247 |
+
n_elements = int(tensor.n_elements)
|
| 1248 |
+
blocks_this = n_elements // QK
|
| 1249 |
+
if not args.quiet:
|
| 1250 |
+
done_pct = 100.0 * (selected_done - 1) / max(len(selected), 1)
|
| 1251 |
+
print(
|
| 1252 |
+
f"\n{tensor_prefix} | {n_elements:,} weights | {blocks_this:,} blocks "
|
| 1253 |
+
f"| overall selected progress {done_pct:.1f}% | BEGIN",
|
| 1254 |
+
flush=True,
|
| 1255 |
+
)
|
| 1256 |
+
|
| 1257 |
+
t_load = time.perf_counter()
|
| 1258 |
+
try:
|
| 1259 |
+
with progress_stage("dequantize source tensor", tensor_prefix, args.progress_interval, args.quiet):
|
| 1260 |
+
original = load_tensor_f32(tensor, dequantize_fn)
|
| 1261 |
+
except Exception as exc:
|
| 1262 |
+
skipped.append({
|
| 1263 |
+
"tensor": tensor.name,
|
| 1264 |
+
"shape_ggml": list(tensor_ggml_shape(tensor)),
|
| 1265 |
+
"reason": f"dequantization failed; copied unchanged: {exc}",
|
| 1266 |
+
"gguf_type": tensor.tensor_type.name,
|
| 1267 |
+
})
|
| 1268 |
+
if writer is not None:
|
| 1269 |
+
with progress_stage("copy original tensor unchanged", tensor_prefix, args.progress_interval, args.quiet):
|
| 1270 |
+
add_tensor_passthrough(writer, tensor, reader)
|
| 1271 |
+
passthrough_count += 1
|
| 1272 |
+
print(f"WARNING: {tensor.name}: dequantization failed; copied unchanged: {exc}", file=sys.stderr, flush=True)
|
| 1273 |
+
continue
|
| 1274 |
+
|
| 1275 |
+
np_shape = original.shape
|
| 1276 |
+
x = original.reshape(-1, QK)
|
| 1277 |
+
|
| 1278 |
+
tensor_weights = None
|
| 1279 |
+
if imatrix is not None:
|
| 1280 |
+
try:
|
| 1281 |
+
tensor_weights = imatrix.weights(tensor.name, tensor_ggml_shape(tensor))
|
| 1282 |
+
except ValueError as exc:
|
| 1283 |
+
imatrix_misses += 1
|
| 1284 |
+
print(
|
| 1285 |
+
f"WARNING: {tensor.name}: {exc}; using uniform weights",
|
| 1286 |
+
file=sys.stderr, flush=True,
|
| 1287 |
+
)
|
| 1288 |
+
else:
|
| 1289 |
+
if tensor_weights is None:
|
| 1290 |
+
imatrix_misses += 1
|
| 1291 |
+
if not args.quiet:
|
| 1292 |
+
print(f" note: {tensor.name} not present in imatrix; using uniform weights", flush=True)
|
| 1293 |
+
else:
|
| 1294 |
+
imatrix_hits += 1
|
| 1295 |
+
|
| 1296 |
+
def progress(done: int, total: int, message: str, _name=tensor.name):
|
| 1297 |
+
if not args.quiet:
|
| 1298 |
+
pct = 100.0 * done / max(total, 1)
|
| 1299 |
+
print(f" [{selected_done}/{len(selected)}] {_name}: {message} ({pct:.1f}% of tensor)", flush=True)
|
| 1300 |
+
|
| 1301 |
+
recon_path = None
|
| 1302 |
+
if args.save_recon:
|
| 1303 |
+
recon_path = args.save_recon / f"{safe_filename(tensor.name)}.npy"
|
| 1304 |
+
|
| 1305 |
+
with progress_stage("quadratic factorization to Q2_K", tensor_prefix, args.progress_interval, args.quiet):
|
| 1306 |
+
packed_blocks, dec_stats, rtn_stats, elapsed = factorize_q2k(
|
| 1307 |
+
x,
|
| 1308 |
+
chunk_blocks=args.chunk_blocks,
|
| 1309 |
+
save_path=recon_path,
|
| 1310 |
+
save_shape=np_shape,
|
| 1311 |
+
progress=None if args.quiet else progress,
|
| 1312 |
+
rtn_baseline=not args.no_rtn_metrics,
|
| 1313 |
+
metrics=not args.no_metrics,
|
| 1314 |
+
weights=tensor_weights,
|
| 1315 |
+
)
|
| 1316 |
+
|
| 1317 |
+
if rtn_stats is not None and dec_stats is not None:
|
| 1318 |
+
gain_pct = 100.0 * (1.0 - dec_stats["rel_rmse"] / max(rtn_stats["rel_rmse"], 1.0e-30))
|
| 1319 |
+
else:
|
| 1320 |
+
gain_pct = None
|
| 1321 |
+
row = {
|
| 1322 |
+
"tensor": tensor.name,
|
| 1323 |
+
"shape_numpy": list(np_shape),
|
| 1324 |
+
"shape_ggml": list(tensor_ggml_shape(tensor)),
|
| 1325 |
+
"source_gguf_type": tensor.tensor_type.name,
|
| 1326 |
+
"output_gguf_type": "Q2_K",
|
| 1327 |
+
"importance_weighted": tensor_weights is not None,
|
| 1328 |
+
"elements": n_elements,
|
| 1329 |
+
"blocks_256": blocks_this,
|
| 1330 |
+
"smq_rmse": dec_stats["rmse"] if dec_stats else None,
|
| 1331 |
+
"smq_rel_rmse": dec_stats["rel_rmse"] if dec_stats else None,
|
| 1332 |
+
"smq_mae": dec_stats["mae"] if dec_stats else None,
|
| 1333 |
+
"smq_max_abs": dec_stats["max_abs"] if dec_stats else None,
|
| 1334 |
+
"smq_cosine": dec_stats["cosine"] if dec_stats else None,
|
| 1335 |
+
"rms_reference": dec_stats["rms_reference"] if dec_stats else None,
|
| 1336 |
+
"improvement_pct_vs_rtn": gain_pct,
|
| 1337 |
+
"factorization_elapsed_s": elapsed,
|
| 1338 |
+
"load_seconds": time.perf_counter() - t_load - elapsed,
|
| 1339 |
+
"million_elements_per_s": n_elements / 1.0e6 / max(elapsed, 1.0e-12),
|
| 1340 |
+
}
|
| 1341 |
+
rows.append(row)
|
| 1342 |
+
|
| 1343 |
+
if recon_path is not None:
|
| 1344 |
+
row["reconstruction_file"] = str(recon_path)
|
| 1345 |
+
|
| 1346 |
+
if writer is not None:
|
| 1347 |
+
ne0 = tensor_ggml_shape(tensor)[0]
|
| 1348 |
+
blocks_per_row = ne0 // QK
|
| 1349 |
+
rows_per_tensor = n_elements // ne0
|
| 1350 |
+
bytes_per_row = blocks_per_row * Q2K_BYTES
|
| 1351 |
+
if packed_blocks.shape[0] != rows_per_tensor * blocks_per_row:
|
| 1352 |
+
raise AssertionError(
|
| 1353 |
+
f"{tensor.name}: expected {rows_per_tensor * blocks_per_row} Q2_K blocks, "
|
| 1354 |
+
f"got {packed_blocks.shape[0]}"
|
| 1355 |
+
)
|
| 1356 |
+
packed = packed_blocks.reshape(rows_per_tensor, bytes_per_row)
|
| 1357 |
+
if len(np_shape) == 1:
|
| 1358 |
+
packed = packed.reshape(bytes_per_row)
|
| 1359 |
+
else:
|
| 1360 |
+
packed = packed.reshape(np_shape[:-1] + (bytes_per_row,))
|
| 1361 |
+
writer.add_tensor(
|
| 1362 |
+
tensor.name,
|
| 1363 |
+
np.ascontiguousarray(packed),
|
| 1364 |
+
raw_dtype=q2k_type,
|
| 1365 |
+
tensor_endianess=GGUFEndian.LITTLE,
|
| 1366 |
+
)
|
| 1367 |
+
quantized_count += 1
|
| 1368 |
+
|
| 1369 |
+
print(
|
| 1370 |
+
f"{tensor.name} | {np_shape} | {blocks_this:,} | "
|
| 1371 |
+
+ (f"{dec_stats['rel_rmse']:.6g} | " if dec_stats is not None else "- | ")
|
| 1372 |
+
+ (f"{rtn_stats['rel_rmse']:.6g} | {gain_pct:+.2f}% | " if rtn_stats is not None and dec_stats is not None else "- | - | ")
|
| 1373 |
+
+ f"{elapsed:.2f}",
|
| 1374 |
+
flush=True,
|
| 1375 |
+
)
|
| 1376 |
+
if not args.quiet:
|
| 1377 |
+
done_pct = 100.0 * selected_done / max(len(selected), 1)
|
| 1378 |
+
print(
|
| 1379 |
+
f"{tensor_prefix} | COMPLETE | {done_pct:.1f}% selected tensors "
|
| 1380 |
+
f"| total wall {time.perf_counter() - start_total:.1f}s",
|
| 1381 |
+
flush=True,
|
| 1382 |
+
)
|
| 1383 |
+
del original, x, packed_blocks
|
| 1384 |
+
|
| 1385 |
+
if writer is not None:
|
| 1386 |
+
print(f"\nWriting GGUF metadata and tensor data to {tmp_output} ...", flush=True)
|
| 1387 |
+
with progress_stage("write GGUF header", "[GGUF output]", args.progress_interval, args.quiet):
|
| 1388 |
+
writer.write_header_to_file()
|
| 1389 |
+
with progress_stage("write GGUF metadata", "[GGUF output]", args.progress_interval, args.quiet):
|
| 1390 |
+
writer.write_kv_data_to_file()
|
| 1391 |
+
with progress_stage("write all GGUF tensor payloads", "[GGUF output]", args.progress_interval, args.quiet):
|
| 1392 |
+
writer.write_tensors_to_file(progress=False)
|
| 1393 |
+
with progress_stage("close GGUF writer", "[GGUF output]", args.progress_interval, args.quiet):
|
| 1394 |
+
writer.close()
|
| 1395 |
+
writer = None
|
| 1396 |
+
if tempdir_overridden:
|
| 1397 |
+
tempfile.tempdir = prev_tempdir
|
| 1398 |
+
tempdir_overridden = False
|
| 1399 |
+
|
| 1400 |
+
# Validate the emitted GGUF and tensor table before atomically publishing it.
|
| 1401 |
+
check_reader = GGUFReader(str(tmp_output))
|
| 1402 |
+
if len(check_reader.tensors) != len(reader.tensors):
|
| 1403 |
+
raise RuntimeError(
|
| 1404 |
+
f"GGUF validation failed: output has {len(check_reader.tensors)} tensors, "
|
| 1405 |
+
f"input had {len(reader.tensors)}"
|
| 1406 |
+
)
|
| 1407 |
+
out_names = [t.name for t in check_reader.tensors]
|
| 1408 |
+
in_names = [t.name for t in reader.tensors]
|
| 1409 |
+
if out_names != in_names:
|
| 1410 |
+
raise RuntimeError("GGUF validation failed: tensor names/order changed")
|
| 1411 |
+
out_by_name = {t.name: t for t in check_reader.tensors}
|
| 1412 |
+
quantized_names = {row["tensor"] for row in rows}
|
| 1413 |
+
for name, in_tensor in zip(in_names, reader.tensors):
|
| 1414 |
+
out_tensor = out_by_name[name]
|
| 1415 |
+
if tensor_numpy_shape(out_tensor) != tensor_numpy_shape(in_tensor):
|
| 1416 |
+
raise RuntimeError(f"GGUF validation failed: shape changed for {name}")
|
| 1417 |
+
if name in quantized_names:
|
| 1418 |
+
if out_tensor.tensor_type != q2k_type:
|
| 1419 |
+
raise RuntimeError(
|
| 1420 |
+
f"GGUF validation failed: {name} has output type "
|
| 1421 |
+
f"{out_tensor.tensor_type.name}, expected Q2_K"
|
| 1422 |
+
)
|
| 1423 |
+
elif out_tensor.tensor_type != in_tensor.tensor_type:
|
| 1424 |
+
raise RuntimeError(f"GGUF validation failed: pass-through tensor {name} changed type")
|
| 1425 |
+
out_arch = check_reader.fields.get("general.architecture")
|
| 1426 |
+
if out_arch is None or out_arch.contents() != arch:
|
| 1427 |
+
raise RuntimeError("GGUF validation failed: general.architecture was not preserved")
|
| 1428 |
+
out_file_type = check_reader.fields.get("general.file_type")
|
| 1429 |
+
if out_file_type is None or int(out_file_type.contents()) != Q2K_FILE_TYPE:
|
| 1430 |
+
raise RuntimeError("GGUF validation failed: general.file_type is not MOSTLY_Q2_K")
|
| 1431 |
+
os.replace(tmp_output, args.output_gguf)
|
| 1432 |
+
print(f"Wrote llama.cpp-compatible Q2_K GGUF: {args.output_gguf}")
|
| 1433 |
+
except Exception:
|
| 1434 |
+
if writer is not None:
|
| 1435 |
+
try:
|
| 1436 |
+
writer.close()
|
| 1437 |
+
except Exception:
|
| 1438 |
+
pass
|
| 1439 |
+
if tempdir_overridden:
|
| 1440 |
+
tempfile.tempdir = prev_tempdir
|
| 1441 |
+
tempdir_overridden = False
|
| 1442 |
+
if tmp_output is not None and tmp_output.exists():
|
| 1443 |
+
try:
|
| 1444 |
+
tmp_output.unlink()
|
| 1445 |
+
except Exception:
|
| 1446 |
+
pass
|
| 1447 |
+
if isinstance(sys.exc_info()[1], OSError) and sys.exc_info()[1].errno == 28:
|
| 1448 |
+
raise SystemExit(
|
| 1449 |
+
"Out of disk space while writing the output GGUF. gguf-py spools all tensor "
|
| 1450 |
+
f"payloads to a temp file on {spool_dir or tempfile.gettempdir()}; point the output at a "
|
| 1451 |
+
"filesystem with room for roughly 2x the output size, or free space there."
|
| 1452 |
+
) from None
|
| 1453 |
+
raise
|
| 1454 |
+
|
| 1455 |
+
elapsed_total = time.perf_counter() - start_total
|
| 1456 |
+
csv_path, json_path = write_reports(
|
| 1457 |
+
args, rows, skipped, len(eligible), len(selected), quantized_count, passthrough_count,
|
| 1458 |
+
)
|
| 1459 |
+
print("\nWrote reports:")
|
| 1460 |
+
print(f" CSV: {csv_path}")
|
| 1461 |
+
print(f" JSON: {json_path}")
|
| 1462 |
+
if args.output_gguf:
|
| 1463 |
+
print(f" GGUF: {args.output_gguf}")
|
| 1464 |
+
print(f" Factorized tensors: {quantized_count}")
|
| 1465 |
+
print(f" Copied unchanged: {passthrough_count}")
|
| 1466 |
+
if args.save_recon:
|
| 1467 |
+
print(f" Reconstructions: {args.save_recon}")
|
| 1468 |
+
if imatrix is not None:
|
| 1469 |
+
print(f" Importance-weighted tensors: {imatrix_hits} (uniform fallback: {imatrix_misses})")
|
| 1470 |
+
if skipped:
|
| 1471 |
+
print(f" Skipped/unchanged tensors: {len(skipped)}")
|
| 1472 |
+
print(f" Total elapsed: {elapsed_total:.2f}s")
|
| 1473 |
+
return 0
|
| 1474 |
+
|
| 1475 |
+
|
| 1476 |
+
if __name__ == "__main__":
|
| 1477 |
+
try:
|
| 1478 |
+
raise SystemExit(main())
|
| 1479 |
+
except KeyboardInterrupt:
|
| 1480 |
+
raise SystemExit("Interrupted")
|
smq_qat.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
train_smq_qat.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Block-wise quantization-aware training for SMQ (CLI).
|
| 3 |
+
|
| 4 |
+
Trains a GGUF model against SMQ's *real* ``Q2_K`` quantizer, block by block,
|
| 5 |
+
then writes a full-precision (F16) GGUF that ``smq_gguf_convert_live.py``
|
| 6 |
+
quantizes as usual.
|
| 7 |
+
|
| 8 |
+
All eligible weight matrices (``tensor_is_eligible``: >= 2 GGML dims, no 1-D
|
| 9 |
+
elementwise parameters, ne[0] % 256 == 0) become Q2_K and are trained; the
|
| 10 |
+
plan is fixed by the converter's own rule, so QAT always optimizes for
|
| 11 |
+
exactly what SMQ will emit.
|
| 12 |
+
|
| 13 |
+
Usage::
|
| 14 |
+
|
| 15 |
+
python train_smq_qat.py --gguf model.gguf --corpus corpus.txt \\
|
| 16 |
+
--output model-qat.gguf \\
|
| 17 |
+
--imatrix model.imatrix.gguf --steps 120 --blocks 0-8
|
| 18 |
+
|
| 19 |
+
# then quantize with the converter
|
| 20 |
+
python smq_gguf_convert_live.py model-qat.gguf --output-gguf model-qat-Q2K.gguf
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import argparse
|
| 26 |
+
import sys
|
| 27 |
+
from pathlib import Path
|
| 28 |
+
|
| 29 |
+
_HERE = str(Path(__file__).resolve().parent)
|
| 30 |
+
if _HERE and _HERE not in sys.path:
|
| 31 |
+
sys.path.insert(0, _HERE)
|
| 32 |
+
|
| 33 |
+
import smq_gguf_convert_live as smqc # noqa: E402
|
| 34 |
+
from smq_qat import ( # noqa: E402
|
| 35 |
+
block_targets,
|
| 36 |
+
real_blocks,
|
| 37 |
+
tensor_type_key,
|
| 38 |
+
total_blocks,
|
| 39 |
+
train,
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _parse_blocks(text: str | None) -> tuple[int, int] | None:
|
| 44 |
+
if not text:
|
| 45 |
+
return None
|
| 46 |
+
parts = text.replace(",", "-").split("-")
|
| 47 |
+
if len(parts) != 2:
|
| 48 |
+
raise SystemExit(f"--blocks expects LO-HI, got {text!r}")
|
| 49 |
+
try:
|
| 50 |
+
lo, hi = int(parts[0]), int(parts[1])
|
| 51 |
+
except ValueError:
|
| 52 |
+
raise SystemExit(f"invalid --blocks range {text!r}")
|
| 53 |
+
if lo < 0 or hi <= lo:
|
| 54 |
+
raise SystemExit(f"invalid --blocks range {text!r}")
|
| 55 |
+
return lo, hi
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _parse_train_types(text: str | None) -> set[str] | None:
|
| 59 |
+
if not text:
|
| 60 |
+
return None
|
| 61 |
+
return {t.strip() for t in text.split(",") if t.strip()}
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _parse_tensor_types(items: list[str]) -> list[tuple[str, str]]:
|
| 65 |
+
out = []
|
| 66 |
+
for item in items:
|
| 67 |
+
name, _, tt = item.partition("=")
|
| 68 |
+
if not name or not tt:
|
| 69 |
+
raise SystemExit(f"--tensor-type expects TYPE=Copy, got {item!r}")
|
| 70 |
+
out.append((name.strip(), tt.strip()))
|
| 71 |
+
return out
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def _build_parser() -> argparse.ArgumentParser:
|
| 75 |
+
p = argparse.ArgumentParser(
|
| 76 |
+
description="Block-wise QAT for SMQ (train weights to survive the real Q2_K quantizer)",
|
| 77 |
+
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
| 78 |
+
)
|
| 79 |
+
p.add_argument("--gguf", type=Path, required=True, help="Input model GGUF")
|
| 80 |
+
p.add_argument("--corpus", type=Path, required=True,
|
| 81 |
+
help="Calibration corpus (.txt/.md/.jsonl, or a directory of them)")
|
| 82 |
+
p.add_argument("--output", type=Path, required=True, help="Output trained GGUF (F16)")
|
| 83 |
+
g = p.add_argument_group("quantization plan")
|
| 84 |
+
g.add_argument("--blocks", default=None, metavar="LO-HI",
|
| 85 |
+
help="only train blocks in [LO, HI)")
|
| 86 |
+
g.add_argument("--tensor-type", action="append", default=[], metavar="TYPE=Copy",
|
| 87 |
+
help="mark a tensor type as Copy (never quantized/trained); repeatable")
|
| 88 |
+
g.add_argument("--imatrix", type=Path, default=None,
|
| 89 |
+
help="importance matrix (.dat or GGUF); weights the fake-quant and the converter")
|
| 90 |
+
g.add_argument("--tokenizer", type=Path, default=None,
|
| 91 |
+
help="HF tokenizer directory override (default: rebuild from the GGUF)")
|
| 92 |
+
g.add_argument("--verbose", action="store_true", help="list every trainable tensor per block")
|
| 93 |
+
t = p.add_argument_group("training")
|
| 94 |
+
t.add_argument("--num-seqs", type=int, default=32, help="calibration sequences")
|
| 95 |
+
t.add_argument("--seq-len", type=int, default=512, help="tokens per sequence")
|
| 96 |
+
t.add_argument("--max-tokens", type=int, default=0, help="cap corpus tokens (0 = all)")
|
| 97 |
+
t.add_argument("--lr", type=float, default=2.0e-5)
|
| 98 |
+
t.add_argument("--weight-decay", type=float, default=0.0)
|
| 99 |
+
t.add_argument("--weight-rmse-lambda", type=float, default=0.0,
|
| 100 |
+
help="weight ||W-Q(W)||^2 penalty coefficient")
|
| 101 |
+
t.add_argument("--cosine-lambda", type=float, default=0.5,
|
| 102 |
+
help="cosine output-similarity loss coefficient")
|
| 103 |
+
t.add_argument("--steps", type=int, default=120, help="optimizer steps per block")
|
| 104 |
+
t.add_argument("--micro-batch", type=int, default=4)
|
| 105 |
+
t.add_argument("--requant-every", type=int, default=0,
|
| 106 |
+
help="re-run the real quantizer every N steps (0 = once per block)")
|
| 107 |
+
t.add_argument("--train-types", default=None,
|
| 108 |
+
help="comma-separated tensor types to train (default: all quantized)")
|
| 109 |
+
t.add_argument("--no-grad-checkpointing", action="store_true")
|
| 110 |
+
t.add_argument("--propagate", choices=("qat", "full"), default="qat",
|
| 111 |
+
help="feed the next block fake-quant (qat) or full-precision activations")
|
| 112 |
+
h = p.add_argument_group("hardware / bookkeeping")
|
| 113 |
+
h.add_argument("--device", default="cuda", help="cuda, cuda:N or cpu")
|
| 114 |
+
h.add_argument("--work-dir", type=Path, default=None,
|
| 115 |
+
help="work directory (default: <output>.qat_work)")
|
| 116 |
+
h.add_argument("--keep-block-dirs", action="store_true",
|
| 117 |
+
help="keep per-block HF partial directories")
|
| 118 |
+
h.add_argument("--seed", type=int, default=1234)
|
| 119 |
+
h.add_argument("--dry-run", action="store_true", help="print the plan and exit")
|
| 120 |
+
return p
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def main() -> None:
|
| 124 |
+
args = _build_parser().parse_args()
|
| 125 |
+
blocks = _parse_blocks(args.blocks)
|
| 126 |
+
train_types = _parse_train_types(args.train_types)
|
| 127 |
+
tensor_type = _parse_tensor_types(args.tensor_type)
|
| 128 |
+
device = args.device or "cuda"
|
| 129 |
+
|
| 130 |
+
_gguf_py, reader_cls, _deq, *_rest = smqc.load_gguf()
|
| 131 |
+
reader = reader_cls(str(args.gguf))
|
| 132 |
+
n_blocks = real_blocks(reader)
|
| 133 |
+
n_nextn = total_blocks(reader) - n_blocks
|
| 134 |
+
lo, hi = (0, n_blocks) if blocks is None else (blocks[0], min(blocks[1], n_blocks))
|
| 135 |
+
excluded = {name for name, tt in tensor_type if tt.lower() == "copy"}
|
| 136 |
+
|
| 137 |
+
if args.dry_run:
|
| 138 |
+
print(f"{args.gguf}: {len(reader.tensors)} tensors, {n_blocks} trainable blocks")
|
| 139 |
+
if n_nextn:
|
| 140 |
+
print(f"{n_nextn} NextN/MTP draft block(s) excluded from training")
|
| 141 |
+
print(f"train blocks {lo}..{hi - 1}, device={device}")
|
| 142 |
+
counts: dict[str, int] = {}
|
| 143 |
+
for b in range(lo, hi):
|
| 144 |
+
for gguf_name, _hf in block_targets(reader, b):
|
| 145 |
+
key = tensor_type_key(gguf_name)
|
| 146 |
+
if train_types is not None and key not in train_types:
|
| 147 |
+
continue
|
| 148 |
+
if key in excluded:
|
| 149 |
+
continue
|
| 150 |
+
counts["Q2_K"] = counts.get("Q2_K", 0) + 1
|
| 151 |
+
if args.verbose:
|
| 152 |
+
print(f" blk.{b}: {gguf_name}")
|
| 153 |
+
print("plan: " + ", ".join(f"{k}={v}" for k, v in sorted(counts.items())))
|
| 154 |
+
if train_types:
|
| 155 |
+
print(f"train_types: {sorted(train_types)}")
|
| 156 |
+
if excluded:
|
| 157 |
+
print(f"excluded (Copy): {sorted(excluded)}")
|
| 158 |
+
return
|
| 159 |
+
|
| 160 |
+
train(
|
| 161 |
+
args.gguf,
|
| 162 |
+
args.corpus,
|
| 163 |
+
args.output,
|
| 164 |
+
blocks=blocks,
|
| 165 |
+
num_seqs=args.num_seqs,
|
| 166 |
+
seq_len=args.seq_len,
|
| 167 |
+
max_tokens=args.max_tokens,
|
| 168 |
+
lr=args.lr,
|
| 169 |
+
weight_decay=args.weight_decay,
|
| 170 |
+
weight_rmse_lambda=args.weight_rmse_lambda,
|
| 171 |
+
cosine_lambda=args.cosine_lambda,
|
| 172 |
+
steps=args.steps,
|
| 173 |
+
micro_batch=args.micro_batch,
|
| 174 |
+
requant_every=args.requant_every,
|
| 175 |
+
train_types=train_types,
|
| 176 |
+
tensor_type=tensor_type,
|
| 177 |
+
grad_checkpointing=not args.no_grad_checkpointing,
|
| 178 |
+
propagate=args.propagate,
|
| 179 |
+
work_dir=args.work_dir,
|
| 180 |
+
keep_block_dirs=args.keep_block_dirs,
|
| 181 |
+
seed=args.seed,
|
| 182 |
+
imatrix=args.imatrix,
|
| 183 |
+
tokenizer=args.tokenizer,
|
| 184 |
+
device=device,
|
| 185 |
+
)
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
if __name__ == "__main__":
|
| 189 |
+
main()
|