CompressedGemma commited on
Commit
013363e
·
verified ·
1 Parent(s): b0cfadb
Files changed (4) hide show
  1. smq.py +282 -0
  2. smq_gguf_convert_live.py +1480 -0
  3. smq_qat.py +0 -0
  4. train_smq_qat.py +189 -0
smq.py ADDED
@@ -0,0 +1,282 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """SMQ solver pieces under the matched metric: code projection + closed-form (d, m).
2
+
3
+ E = sum_i w_i e_i^2 + sum_{k=1..8} lambda_k * sum_j G_kj^2, e = x - xhat
4
+
5
+ The whole matched metric is a SINGLE quadratic form. Each fold level k is a
6
+ strided group-sum operator A_k (element i belongs to group i mod (256/2^k)),
7
+ and sum_j G_kj^2 = ||A_k e||^2 = e' A_k' A_k e, so
8
+
9
+ E = e' M e, M = diag(w) + L, L = sum_k lambda_k A_k' A_k (fixed 256x256)
10
+
11
+ L is precomputed once per (lam, dtype, device) by _fold_gram and every
12
+ objective evaluation, gradient tile, and Hessian/Gram entry is then just a
13
+ bilinear form with L (one matmul), with no per-level fold loops.
14
+
15
+ Per pass: project every code at once (Newton form, e_i = x_i - s_i q_i +
16
+ m lm_i, s_i = d ls_i, Lam = sum_k lambda_k). Since (L e)_i equals
17
+ sum_k lambda_k G_k(i), the per-element coupled gradient tile is e @ L:
18
+
19
+ q_i <- clip(round(q_i + (w_i e_i + (e @ L)_i) / ((w_i + Lam) s_i)), 0, 3)
20
+
21
+ After the pass, e is rebuilt from the new codes and the next pass re-evaluates.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ QK = 256
27
+ NSUB = 16
28
+ _TINY = 1.0e-30
29
+
30
+
31
+ def _fold_gram(lam):
32
+ """Single-matrix form of the spectral penalty: L = sum_k lambda_k A_k' A_k.
33
+
34
+ A_k is the strided fold operator with (A_k e)_r = sum over elements i with
35
+ i mod (256/2^k) == r, so (A_k' A_k)[i, j] = [i mod (256/2^k) == j mod
36
+ (256/2^k)]. The full matched metric of the module docstring is then
37
+ E = e' (diag(w) + L) e — one quadratic form, no fold loops.
38
+ """
39
+ import torch
40
+
41
+ ids = torch.arange(QK, device=lam.device)
42
+ L = torch.zeros(QK, QK, dtype=lam.dtype, device=lam.device)
43
+ for k in range(1, 9):
44
+ r = ids.remainder(QK >> k)
45
+ L = L + lam[k - 1] * (r[:, None] == r[None, :]).to(L.dtype)
46
+ return L
47
+
48
+
49
+ def _objective_quad(e, w, L):
50
+ """Matched metric E per block as one quadratic form: (w e^2) + e' L e -> (n,)."""
51
+ return (w * e * e).sum(dim=1) + (e @ L * e).sum(dim=1)
52
+
53
+
54
+ def _codes_pass(x, w, e, d, m, ls_e, lm_e, q, L, lam_sum):
55
+ """One code-projection pass over all 256 elements. Returns (q, e)."""
56
+ import torch
57
+
58
+ S = e @ L # == sum_k lambda_k G_k(i) tiled per element
59
+ step = d[:, None] * ls_e
60
+ curv = w + lam_sum
61
+ good = (step > _TINY) & (curv > _TINY)
62
+ safe = torch.where(good, curv * step, torch.ones_like(step))
63
+ ratio = q + (w * e + S) / safe
64
+ q_new = torch.where(good, torch.clamp(torch.round(ratio), 0.0, 3.0), q)
65
+ e_new = x - d[:, None] * ls_e * q_new + m[:, None] * lm_e
66
+ return q_new, e_new
67
+
68
+
69
+ def _dm_system(x, w, a, b, L):
70
+ """2x2 matched-metric WLS system for (d, m), one quadratic form, no folds.
71
+
72
+ a = ls_e * q (grid slope), b = lm_e (grid offset, r_i = 1); returns
73
+ H11, H12, H22, g1, g2 for E = th' H th - 2 th' g + c, th = (d, m).
74
+ Every fold-level term is a bilinear form u' L v of the single matrix L:
75
+ sum_k lam_k (A_k u) . (A_k v) = u' L v.
76
+ """
77
+ H11 = (w * a * a).sum(1) + (a @ L * a).sum(1)
78
+ H12 = -((w * a * b).sum(1) + (a @ L * b).sum(1))
79
+ H22 = (w * b * b).sum(1) + (b @ L * b).sum(1)
80
+ g1 = (w * x * a).sum(1) + (x @ L * a).sum(1)
81
+ g2 = -((w * x * b).sum(1) + (x @ L * b).sum(1))
82
+ return H11, H12, H22, g1, g2
83
+
84
+
85
+ def _solve2(H11, H12, H22, g1, g2):
86
+ """Solve a batch of 2x2 systems H th = g with a validity mask."""
87
+ import torch
88
+
89
+ det = H11 * H22 - H12 * H12
90
+ ok = (det > _TINY) & torch.isfinite(det) & torch.isfinite(g1) & torch.isfinite(g2)
91
+ det_s = torch.where(ok, det, torch.ones_like(det))
92
+ v1 = (H22 * g1 - H12 * g2) / det_s
93
+ v2 = (H11 * g2 - H12 * g1) / det_s
94
+ ok = ok & torch.isfinite(v1) & torch.isfinite(v2)
95
+ return ok, v1, v2
96
+
97
+
98
+ def _quad2(H11, H12, H22, g1, g2, d, m):
99
+ """Constant-free value of the 2-D quadratic represented by _dm_system."""
100
+ return (
101
+ H11 * d * d + 2.0 * H12 * d * m + H22 * m * m
102
+ - 2.0 * g1 * d - 2.0 * g2 * m
103
+ )
104
+
105
+
106
+ def _solve2_constrained(
107
+ H11, H12, H22, g1, g2, d_current, m_current, d_floor=_TINY
108
+ ):
109
+ """Minimize the convex 2-D quadratic subject to d >= d_floor and m >= 0.
110
+
111
+ The candidate set consists of the feasible unconstrained optimum, both
112
+ one-dimensional boundary optima, the corner (included by clamping the
113
+ boundary solutions), and the current feasible point as a numerical
114
+ fallback. Comparing the original quadratic makes this valid for singular
115
+ positive-semidefinite systems as well as nonsingular systems.
116
+ """
117
+ import torch
118
+
119
+ d0 = torch.clamp(torch.nan_to_num(d_current, nan=d_floor, posinf=65504.0, neginf=d_floor), min=d_floor)
120
+ m0 = torch.clamp(torch.nan_to_num(m_current, nan=0.0, posinf=65504.0, neginf=0.0), min=0.0)
121
+ best_d, best_m = d0, m0
122
+ best_v = _quad2(H11, H12, H22, g1, g2, best_d, best_m)
123
+ best_v = torch.where(torch.isfinite(best_v), best_v, torch.full_like(best_v, float("inf")))
124
+
125
+ # Interior stationary point.
126
+ ok_i, d_i, m_i = _solve2(H11, H12, H22, g1, g2)
127
+ ok_i = ok_i & (d_i >= d_floor) & (m_i >= 0.0)
128
+ v_i = _quad2(H11, H12, H22, g1, g2, d_i, m_i)
129
+ ok_i = ok_i & torch.isfinite(v_i)
130
+ take = ok_i & (v_i < best_v)
131
+ best_d = torch.where(take, d_i, best_d)
132
+ best_m = torch.where(take, m_i, best_m)
133
+ best_v = torch.where(take, v_i, best_v)
134
+
135
+ # Boundary m = 0: optimize d on its feasible half-line.
136
+ h11_ok = torch.isfinite(H11) & (H11 > _TINY)
137
+ d_m0 = torch.where(h11_ok, torch.clamp(g1 / torch.where(h11_ok, H11, torch.ones_like(H11)), min=d_floor), d0)
138
+ m_m0 = torch.zeros_like(m0)
139
+ v_m0 = _quad2(H11, H12, H22, g1, g2, d_m0, m_m0)
140
+ ok_m0 = torch.isfinite(v_m0) & torch.isfinite(d_m0)
141
+ take = ok_m0 & (v_m0 < best_v)
142
+ best_d = torch.where(take, d_m0, best_d)
143
+ best_m = torch.where(take, m_m0, best_m)
144
+ best_v = torch.where(take, v_m0, best_v)
145
+
146
+ # Boundary d = d_floor: optimize the nonnegative offset.
147
+ h22_ok = torch.isfinite(H22) & (H22 > _TINY)
148
+ h22_safe = torch.where(h22_ok, H22, torch.ones_like(H22))
149
+ m_df = torch.where(
150
+ h22_ok,
151
+ torch.clamp((g2 - H12 * d_floor) / h22_safe, min=0.0),
152
+ m0,
153
+ )
154
+ d_df = torch.full_like(d0, float(d_floor))
155
+ v_df = _quad2(H11, H12, H22, g1, g2, d_df, m_df)
156
+ ok_df = torch.isfinite(v_df) & torch.isfinite(m_df)
157
+ take = ok_df & (v_df < best_v)
158
+ best_d = torch.where(take, d_df, best_d)
159
+ best_m = torch.where(take, m_df, best_m)
160
+
161
+ return best_d, best_m
162
+
163
+
164
+ def _dm_resolve(x, w, q, ls_e, lm_e, d, m, L):
165
+ """Closed-form matched-metric (d, m) for ALL blocks at once, no iteration.
166
+
167
+ With codes and (ls, lm) fixed, e_i = x_i - d a_i + m b_i is affine in
168
+ (d, m), a_i = ls_i q_i, b_i = lm_i, so the single quadratic E = e' M e
169
+ (M = diag(w) + L, all eight fold levels included) is an exact 2x2
170
+ quadratic in th = (d, m): E = th' H th - 2 th' g + c. The solution is
171
+ th = H^-1 g (Cramer's rule), constrained to d > 0, m >= 0.
172
+ Returns continuous (d, m); the caller rounds to fp16 and guards on E.
173
+ """
174
+ a = ls_e * q
175
+ H11, H12, H22, g1, g2 = _dm_system(x, w, a, lm_e, L)
176
+ return _solve2_constrained(H11, H12, H22, g1, g2, d, m)
177
+
178
+
179
+ def _integer_lattice_min_2d(H11, H12, H22, g1, g2, current_s, current_t, upper=15):
180
+ """Exact minimum of a convex 2-D quadratic on {0..upper}^2.
181
+
182
+ Enumerate the 16 (or ``upper+1``) possible first coordinates. For each,
183
+ the conditional optimum in t is one-dimensional and convex, so its integer
184
+ minimum is floor(t*) or ceil(t*) (with clipping at the boundaries). This
185
+ gives at most 2*(upper+1) candidates instead of a full square lattice scan.
186
+ """
187
+ import torch
188
+
189
+ cur_s = torch.clamp(torch.round(current_s), 0.0, float(upper))
190
+ cur_t = torch.clamp(torch.round(current_t), 0.0, float(upper))
191
+ s_grid = torch.arange(upper + 1, device=H11.device, dtype=H11.dtype).view(1, -1)
192
+ s_grid = s_grid.expand(H11.shape[0], -1)
193
+ h22_ok = torch.isfinite(H22) & (H22 > _TINY)
194
+ h22_safe = torch.where(h22_ok, H22, torch.ones_like(H22))
195
+ t_star = torch.clamp(
196
+ (g2[:, None] - H12[:, None] * s_grid) / h22_safe[:, None],
197
+ 0.0, float(upper),
198
+ )
199
+ t_lo = torch.floor(t_star)
200
+ t_hi = torch.ceil(t_star)
201
+ # For a PSD quadratic H22=0 implies H12=0; t is then unidentifiable.
202
+ t_lo = torch.where(h22_ok[:, None], t_lo, cur_t[:, None])
203
+ t_hi = torch.where(h22_ok[:, None], t_hi, cur_t[:, None])
204
+ s_candidates = torch.cat((s_grid, s_grid), dim=1)
205
+ t_candidates = torch.cat((t_lo, t_hi), dim=1)
206
+ values = _quad2(
207
+ H11[:, None], H12[:, None], H22[:, None],
208
+ g1[:, None], g2[:, None], s_candidates, t_candidates,
209
+ )
210
+ values = torch.where(torch.isfinite(values), values, torch.full_like(values, float("inf")))
211
+ best_idx = torch.argmin(values, dim=1, keepdim=True)
212
+ best_val = values.gather(1, best_idx).squeeze(1)
213
+ best_s = s_candidates.gather(1, best_idx).squeeze(1)
214
+ best_t = t_candidates.gather(1, best_idx).squeeze(1)
215
+
216
+ # Retain the current lattice point if roundoff makes its evaluated value
217
+ # lower than the analytically enumerated conditional minima.
218
+ cur_val = _quad2(H11, H12, H22, g1, g2, cur_s, cur_t)
219
+ keep_cur = (~torch.isfinite(best_val)) | (cur_val < best_val)
220
+ return torch.where(keep_cur, cur_s, best_s), torch.where(keep_cur, cur_t, best_t), torch.minimum(cur_val, best_val)
221
+
222
+
223
+ def _ls_lm_resolve(x, w, e, d, m, q, ls, lm, lam):
224
+ """Closed-form matched-metric (ls, lm) for ALL 16 sub-blocks of ALL blocks at once.
225
+
226
+ For sub-block j, e_i = x_i - Ls alpha_i + Lm beta (alpha_i = d q_i, beta = m)
227
+ is affine in theta = (Ls, Lm). A level-k group g meets sub-block j in A_g =
228
+ sum alpha (and B_g = |g n j| m) and its group sum is T_g - Ls A_g + Lm B_g,
229
+ where T_g = G_g + Ls_cur A_g - Lm_cur B_g carries every other sub-block's
230
+ current contribution. So the matched E restricted to sub-block j is an exact
231
+ 2x2 quadratic th' H th - 2 th' g + c with
232
+ H11 = sum w alpha^2 + sum_k lam_k sum_g A_g^2 H12 = -(sum w alpha beta + sum_k lam_k sum_g A_g B_g)
233
+ H22 = sum w beta^2 + sum_k lam_k sum_g B_g^2
234
+ g1 = sum w x alpha + sum_k lam_k sum_g T_g A_g g2 = -(sum w x beta + sum_k lam_k sum_g T_g B_g)
235
+ Levels 1-4 (stride >= 16): each group holds one element of j. Levels 5-8
236
+ (stride 8, 4, 2, 1): each group holds 16/stride elements of j.
237
+ The integer minimum on {0..15}^2 is then enumerated exactly (no 256-point scan).
238
+ All sub-blocks are solved against the CURRENT state of the others.
239
+ The metric is still the single quadratic E = e' (diag(w) + L) e; this solver
240
+ keeps the explicit fold form because its per-sub-block "state of the others"
241
+ correction T_g does not reduce to cheap bilinear forms with L.
242
+ """
243
+ import torch
244
+
245
+ n = x.shape[0]
246
+ al = (d[:, None] * q).reshape(n, NSUB, 16)
247
+ be = m[:, None, None]
248
+ xs = x.reshape(n, NSUB, 16)
249
+ ws = w.reshape(n, NSUB, 16)
250
+ H11 = (ws * al * al).sum(2)
251
+ H12 = -(ws * al * be).sum(2)
252
+ H22 = (ws * be * be).sum(2)
253
+ g1 = (ws * xs * al).sum(2)
254
+ g2 = -(ws * xs * be).sum(2)
255
+
256
+ ids = torch.arange(QK, device=x.device).reshape(NSUB, 16)
257
+ ls3 = ls[:, :, None]
258
+ lm3 = lm[:, :, None]
259
+ for k in range(1, 9):
260
+ S = QK >> k
261
+ Gk = e.reshape(n, 1 << k, S).sum(1) # (n, S) group sums, indexed by residue
262
+ if S >= 16:
263
+ A = al
264
+ B = be + torch.zeros_like(al)
265
+ Gc = Gk[:, ids % S] # (n, 16, 16): group sum of each element's own group
266
+ else:
267
+ A = al.reshape(n, NSUB, 16 // S, S).sum(2) # (n, 16, S)
268
+ B = be * (16 // S) + torch.zeros_like(A)
269
+ Gc = Gk[:, None, :] + torch.zeros_like(A)
270
+ T = Gc + ls3 * A - lm3 * B
271
+ c = lam[k - 1]
272
+ H11 = H11 + c * (A * A).sum(2)
273
+ H12 = H12 - c * (A * B).sum(2)
274
+ H22 = H22 + c * (B * B).sum(2)
275
+ g1 = g1 + c * (T * A).sum(2)
276
+ g2 = g2 - c * (T * B).sum(2)
277
+
278
+ s_new, t_new, _ = _integer_lattice_min_2d(
279
+ H11.reshape(-1), H12.reshape(-1), H22.reshape(-1), g1.reshape(-1), g2.reshape(-1),
280
+ ls.reshape(-1), lm.reshape(-1),
281
+ )
282
+ return s_new.reshape(n, NSUB), t_new.reshape(n, NSUB)
smq_gguf_convert_live.py ADDED
@@ -0,0 +1,1480 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Factor GGUF weights directly into Q2_K by closed-form quadratic decomposition.
3
+
4
+ Every eligible tensor is processed in ONE vectorized NumPy pass, with no
5
+ iteration and no objective sweeps:
6
+
7
+ 1. RTN grid (algebraic, closed form): per 16-wide sub-block, step and offset
8
+ from min/max; per-block d = max step / 15, dmin = max offset / 15 (fp16);
9
+ codes q = clip(round((x + o) / s), 0, 3).
10
+ 2. Quadratic decomposition: with those codes fixed, each sub-block vector is
11
+ projected in closed form onto span{q, 1} by solving the 2x2 normal
12
+ equations of sum_i (x_i - s q_i + o)^2 (the exact least-squares
13
+ factorization x ~ s*q - o; degenerate all-equal-code sub-blocks fall back
14
+ to the RTN grid).
15
+ 3. The fitted (s, o) are snapped to the Q2_K integer/fp16 parameter lattice
16
+ (d = max s / 15, ls = round(s/d), dmin = max o / 15, lm = round(o/dmin))
17
+ and the codes are re-derived once against that final grid.
18
+ 4. Per block, the decomposition is kept only if its squared error does not
19
+ exceed the RTN reconstruction's, so the output is never worse than RTN.
20
+
21
+ An optional llama.cpp importance matrix (.dat) supplies a per-weight factor w:
22
+ the projection then minimizes sum_i w_i (x_i - s q_i + o)^2 (same closed
23
+ form, weighted 2x2 normal equations), the per-block guard compares weighted
24
+ errors, and the reported metrics become importance-weighted. Tensors absent
25
+ from the matrix are quantized with uniform weights.
26
+
27
+ Packing, reconstruction metrics (SMQ and RTN baselines) and optional .npy
28
+ reconstructions all happen inside the same pass; there are no extra stages.
29
+
30
+ Requirements:
31
+ pip install numpy
32
+ # Install gguf-py from the same llama.cpp checkout used by your runtime:
33
+ pip install -e /path/to/llama.cpp/gguf-py
34
+
35
+ Examples:
36
+ python smq_gguf_convert.py model-f16.gguf --output-gguf model-q2k.gguf
37
+ python smq_gguf_convert.py model-f16.gguf --imatrix imatrix.dat --output-gguf model-q2k.gguf
38
+ python smq_gguf_convert.py model.gguf --list-tensors
39
+ python smq_gguf_convert.py model.gguf --tensor blk.57.attn_q.weight --output-gguf test.gguf
40
+ python smq_gguf_convert.py --self-test
41
+
42
+ Notes:
43
+ * GGUF tensor shapes in ReaderTensor.shape are GGML dimensions (ne[0],...);
44
+ NumPy's physical row-major view is the reversed shape. Q2_K blocks must be
45
+ aligned within each ne[0] row, so eligibility checks shape[0], not shape[-1].
46
+ * Eligibility is conservative: only weight matrices (two or more GGML
47
+ dimensions, none of them 1) whose ne[0] is a multiple of 256 are quantized.
48
+ 1-D elementwise parameters (norms, biases, ...) are always copied
49
+ unchanged: llama.cpp applies them with ggml_mul/ggml_add, which require
50
+ floating-point operands ("binary_op: unsupported types: dst: f32,
51
+ src0: f32, src1: q2_K" if a norm weight is quantized).
52
+ * Q2_K uses 256 weights per block and 84 bytes per block: 16 packed
53
+ scale/min bytes, 64 code bytes, and two FP16 superblock parameters.
54
+ """
55
+
56
+ from __future__ import annotations
57
+
58
+ import argparse
59
+ import csv
60
+ import json
61
+ import math
62
+ import os
63
+ import re
64
+ import struct
65
+ import sys
66
+ import tempfile
67
+ import threading
68
+ import time
69
+ from contextlib import contextmanager
70
+ from pathlib import Path
71
+ from typing import Any
72
+
73
+ import numpy as np
74
+
75
+ QK = 256
76
+ NSUB = 16
77
+ _TINY = 1.0e-30
78
+ Q2K_BYTES = 84
79
+ Q2K_FILE_TYPE = 10 # LLAMA_FTYPE_MOSTLY_Q2_K
80
+ GGML_QUANT_VERSION = 2
81
+
82
+ # Sub-block reductions (16-wide inner axis) are scalar loops in numpy and
83
+ # dominate runtime, while torch's CPU kernels vectorize them ~10x faster. We
84
+ # only ever use torch here as a drop-in reducer on numpy buffers (zero-copy
85
+ # views, no torch tensors escape), so it stays an optional accelerator.
86
+ _TORCH = None
87
+ _TORCH_READY = False
88
+
89
+
90
+ def _torch_cpu():
91
+ global _TORCH, _TORCH_READY
92
+ if not _TORCH_READY:
93
+ _TORCH_READY = True
94
+ try:
95
+ import torch # noqa: PLC0415 - optional accelerator, imported lazily
96
+
97
+ _TORCH = torch
98
+ except Exception: # pragma: no cover - numpy fallback
99
+ _TORCH = None
100
+ return _TORCH
101
+
102
+
103
+ def _aminmax_last(x3: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
104
+ """Per-block (min, max) over each 16-wide sub-block, shape (n, 16)."""
105
+ t = _torch_cpu()
106
+ if t is not None:
107
+ lo, hi = t.aminmax(t.from_numpy(x3), dim=2)
108
+ return lo.numpy(), hi.numpy()
109
+ return x3.min(axis=2), x3.max(axis=2)
110
+
111
+
112
+ def _sum_last(x3: np.ndarray) -> np.ndarray:
113
+ t = _torch_cpu()
114
+ if t is not None:
115
+ return t.sum(t.from_numpy(x3), dim=2).numpy()
116
+ return x3.sum(axis=2)
117
+
118
+
119
+ def _sq_sum_last(x3: np.ndarray) -> np.ndarray:
120
+ t = _torch_cpu()
121
+ if t is not None:
122
+ tt = t.from_numpy(x3)
123
+ return t.einsum("bij,bij->bi", tt, tt).numpy()
124
+ return (x3 * x3).sum(axis=2)
125
+
126
+
127
+ def _dot_last(x3: np.ndarray, y3: np.ndarray) -> np.ndarray:
128
+ t = _torch_cpu()
129
+ if t is not None:
130
+ return t.einsum("bij,bij->bi", t.from_numpy(x3), t.from_numpy(y3)).numpy()
131
+ return (x3 * y3).sum(axis=2)
132
+
133
+
134
+ def _abs_sum_max(diff: np.ndarray, w: np.ndarray | None = None) -> tuple[float, float]:
135
+ t = _torch_cpu()
136
+ d3 = diff if w is None else diff.reshape(-1, NSUB, 16)
137
+ if t is not None:
138
+ td = t.from_numpy(d3)
139
+ ad = t.abs(td)
140
+ total = ad if w is None else ad * t.from_numpy(w)
141
+ return float(total.sum()), float(ad.max())
142
+ ad = np.abs(d3)
143
+ if w is None:
144
+ return float(ad.sum(dtype=np.float64)), float(ad.max())
145
+ return float((ad * w).sum(dtype=np.float64)), float(ad.max())
146
+
147
+
148
+ def _einsum_last(*arrs: np.ndarray) -> np.ndarray:
149
+ """Per-sub-block sum of the elementwise product of the given (n,16,16)s.
150
+
151
+ With one argument this is a plain sum over the 16-wide axis. The multi-
152
+ operand form is used for weighted moments (e.g. w*x*q) and stays fused in
153
+ torch so no full-size temporary is materialized.
154
+ """
155
+ t = _torch_cpu()
156
+ if t is not None:
157
+ tensors = [t.from_numpy(a) for a in arrs]
158
+ return t.einsum(",".join("bij" for _ in tensors) + "->bi", *tensors).numpy()
159
+ acc = np.array(arrs[0], dtype=np.float32, copy=True)
160
+ for a in arrs[1:]:
161
+ acc *= a
162
+ return acc.sum(axis=2)
163
+
164
+
165
+ IMATRIX_MAGIC = 0x696D6174 # legacy "imat" .dat
166
+ GGUF_MAGIC = 0x46554747 # "GGUF"
167
+
168
+
169
+ class FullWeights:
170
+ """Per-element importance with one value per tensor element."""
171
+
172
+ __slots__ = ("flat",)
173
+
174
+ def __init__(self, flat: np.ndarray):
175
+ self.flat = flat
176
+
177
+ def chunk(self, c0: int, c1: int, qk: int = QK) -> np.ndarray:
178
+ return np.array(
179
+ self.flat[c0 * qk:c1 * qk].reshape(c1 - c0, qk), dtype=np.float32, copy=True
180
+ )
181
+
182
+
183
+ class ChannelWeights:
184
+ """Per-input-channel importance, repeated across every output row.
185
+
186
+ Newer llama.cpp importance matrices store one value per input channel
187
+ (``<tensor>.in_sum2``) instead of one per element. Q2_K blocks are 256
188
+ consecutive channels along the contiguous (ne[0]) axis, so a block's
189
+ weight profile is the same slice in every row and can be tiled on demand
190
+ without materializing the full per-element array.
191
+ """
192
+
193
+ __slots__ = ("values", "ne0")
194
+
195
+ def __init__(self, values: np.ndarray, ne0: int):
196
+ self.values = np.ascontiguousarray(values, dtype=np.float32)
197
+ self.ne0 = int(ne0)
198
+
199
+ def chunk(self, c0: int, c1: int, qk: int = QK) -> np.ndarray:
200
+ count = (c1 - c0) * qk
201
+ off = (c0 * qk) % self.ne0
202
+ v = self.values if off == 0 else np.concatenate((self.values[off:], self.values[:off]))
203
+ reps = (count + self.ne0 - 1) // self.ne0
204
+ return np.tile(v, reps)[:count].reshape(c1 - c0, qk).astype(np.float32, copy=False)
205
+
206
+
207
+ class ImatrixFile:
208
+ """Importance matrix reader supporting both llama.cpp formats.
209
+
210
+ * Legacy ``.dat``: int32 magic "imat", int32 version, int32 n_entries, then
211
+ per entry name_len/name/ncall/nval/values. Memory-mapped so huge files
212
+ are not loaded wholesale.
213
+ * GGUF (``general.type = imatrix``): per-tensor ``<name>.in_sum2`` vectors
214
+ (one value per ne[0] input channel) and ``<name>.counts`` scalars. Values
215
+ are divided by their token count to give mean-squared activations.
216
+
217
+ ``weights(name, ggml_shape)`` returns None when the tensor is absent, or a
218
+ FullWeights/ChannelWeights object that yields per-block weights for a chunk.
219
+ """
220
+
221
+ def __init__(self, path: Path):
222
+ self.path = Path(path)
223
+ with open(self.path, "rb") as handle:
224
+ head = handle.read(4)
225
+ if len(head) < 4:
226
+ raise ValueError(f"{self.path}: file is too short to be an importance matrix")
227
+ self.kind = "gguf" if struct.unpack("<I", head)[0] == GGUF_MAGIC else "dat"
228
+ if self.kind == "gguf":
229
+ self._init_gguf()
230
+ else:
231
+ self._init_dat()
232
+
233
+ # -- legacy .dat -------------------------------------------------------
234
+ def _init_dat(self) -> None:
235
+ self._mm = np.memmap(self.path, dtype=np.uint8, mode="r")
236
+ self.entries: dict[str, tuple[int, int, int]] = {}
237
+ mm = self._mm
238
+ pos = 0
239
+
240
+ def take(n: int) -> bytes:
241
+ nonlocal pos
242
+ if pos + n > mm.size:
243
+ raise ValueError(f"truncated imatrix file {self.path}")
244
+ chunk = mm[pos:pos + n].tobytes()
245
+ pos += n
246
+ return chunk
247
+
248
+ magic, version, n_entries = struct.unpack("<iii", take(12))
249
+ if magic != IMATRIX_MAGIC:
250
+ raise ValueError(f"{self.path}: bad imatrix magic 0x{magic:08x}")
251
+ if version not in (1, 2):
252
+ raise ValueError(f"{self.path}: unsupported imatrix version {version}")
253
+ if n_entries < 0:
254
+ raise ValueError(f"{self.path}: invalid imatrix entry count {n_entries}")
255
+ for _ in range(n_entries):
256
+ (name_len,) = struct.unpack("<i", take(4))
257
+ name = take(name_len).decode("utf-8", "replace")
258
+ ncall, nval = struct.unpack("<ii", take(8))
259
+ if nval < 0:
260
+ raise ValueError(f"{self.path}: invalid value count {nval} for {name!r}")
261
+ offset = pos
262
+ pos += 4 * nval
263
+ self.entries[name] = (offset, nval, ncall)
264
+ self._entry_count = len(self.entries)
265
+
266
+ # -- GGUF --------------------------------------------------------------
267
+ def _init_gguf(self) -> None:
268
+ _gguf, GGUFReader, *_rest = load_gguf()
269
+ self._reader = GGUFReader(str(self.path))
270
+ type_field = self._reader.fields.get("general.type")
271
+ if type_field is None or str(type_field.contents()) != "imatrix":
272
+ raise ValueError(
273
+ f"{self.path}: GGUF is not an importance matrix (general.type != 'imatrix')"
274
+ )
275
+ self._in: dict[str, Any] = {}
276
+ self._counts: dict[str, Any] = {}
277
+ for tensor in self._reader.tensors:
278
+ if tensor.name.endswith(".in_sum2"):
279
+ self._in[tensor.name[: -len(".in_sum2")]] = tensor
280
+ elif tensor.name.endswith(".counts"):
281
+ self._counts[tensor.name[: -len(".counts")]] = tensor
282
+ if not self._in:
283
+ raise ValueError(f"{self.path}: imatrix GGUF contains no <tensor>.in_sum2 entries")
284
+ self._entry_count = len(self._in)
285
+
286
+ # -- public ------------------------------------------------------------
287
+ @property
288
+ def entry_count(self) -> int:
289
+ return self._entry_count
290
+
291
+ def names(self) -> list[str]:
292
+ if self.kind == "gguf":
293
+ return sorted(self._in)
294
+ return list(self.entries)
295
+
296
+ def weights(self, name: str, ggml_shape: tuple[int, ...]):
297
+ ne0 = int(ggml_shape[0]) if ggml_shape else 0
298
+ total = int(np.prod(ggml_shape, dtype=np.int64)) if ggml_shape else 0
299
+ if self.kind == "dat":
300
+ entry = self.entries.get(name)
301
+ if entry is None:
302
+ return None
303
+ offset, nval, _ncall = entry
304
+ values = np.frombuffer(self._mm, dtype="<f4", count=nval, offset=offset)
305
+ else:
306
+ tensor = self._in.get(name)
307
+ if tensor is None:
308
+ return None
309
+ values = np.asarray(tensor.data, dtype=np.float32).reshape(-1)
310
+ counts = self._counts.get(name)
311
+ if counts is not None:
312
+ ncall = float(np.asarray(counts.data, dtype=np.float64).reshape(-1)[0])
313
+ if ncall > 0:
314
+ values = values / ncall
315
+ if total and values.size == total:
316
+ return FullWeights(values)
317
+ if ne0 and values.size == ne0:
318
+ return ChannelWeights(values, ne0)
319
+ raise ValueError(
320
+ f"imatrix entry for {name!r} has {values.size} values; "
321
+ f"expected {total} (per element) or ne[0]={ne0} (per channel)"
322
+ )
323
+
324
+
325
+ @contextmanager
326
+ def progress_stage(label: str, prefix: str, interval: float = 5.0, quiet: bool = False):
327
+ """Print a stage start/end plus periodic heartbeats for long blocking work."""
328
+ started = time.perf_counter()
329
+ if quiet:
330
+ yield
331
+ return
332
+
333
+ stop_event = threading.Event()
334
+
335
+ def heartbeat() -> None:
336
+ while not stop_event.wait(max(0.5, float(interval))):
337
+ elapsed = time.perf_counter() - started
338
+ print(f"{prefix} {label}: still working ({elapsed:.0f}s elapsed)", flush=True)
339
+
340
+ print(f"{prefix} {label} ...", flush=True)
341
+ thread = threading.Thread(target=heartbeat, name="smq-progress-heartbeat", daemon=True)
342
+ thread.start()
343
+ try:
344
+ yield
345
+ finally:
346
+ stop_event.set()
347
+ thread.join(timeout=max(1.0, min(float(interval) + 1.0, 6.0)))
348
+ elapsed = time.perf_counter() - started
349
+ print(f"{prefix} {label}: done ({elapsed:.2f}s)", flush=True)
350
+
351
+
352
+ def load_gguf():
353
+ try:
354
+ import gguf
355
+ except ImportError as exc:
356
+ raise SystemExit(
357
+ "Missing llama.cpp gguf-py package. Install the gguf-py package from "
358
+ "your llama.cpp checkout, for example:\n"
359
+ " pip install -e /path/to/llama.cpp/gguf-py"
360
+ ) from exc
361
+
362
+ GGUFReader = getattr(gguf, "GGUFReader", None)
363
+ if GGUFReader is None:
364
+ from gguf.gguf_reader import GGUFReader
365
+
366
+ dequantize_fn = getattr(gguf, "dequantize", None)
367
+ if dequantize_fn is None:
368
+ from gguf.quants import dequantize as dequantize_fn
369
+
370
+ GGUFWriter = getattr(gguf, "GGUFWriter", None)
371
+ if GGUFWriter is None:
372
+ from gguf.gguf_writer import GGUFWriter
373
+
374
+ try:
375
+ from gguf.constants import GGMLQuantizationType, GGUFValueType, GGUFEndian
376
+ except ImportError:
377
+ GGMLQuantizationType = gguf.GGMLQuantizationType
378
+ GGUFValueType = gguf.GGUFValueType
379
+ GGUFEndian = gguf.GGUFEndian
380
+
381
+ return gguf, GGUFReader, dequantize_fn, GGUFWriter, GGMLQuantizationType, GGUFValueType, GGUFEndian
382
+
383
+
384
+ def tensor_ggml_shape(tensor: Any) -> tuple[int, ...]:
385
+ """GGML dimensions in stored order: (ne[0], ne[1], ...)."""
386
+ return tuple(int(v) for v in np.asarray(tensor.shape).tolist())
387
+
388
+
389
+ def tensor_numpy_shape(tensor: Any) -> tuple[int, ...]:
390
+ """Physical C-order NumPy shape: (..., ne[1], ne[0])."""
391
+ return tuple(reversed(tensor_ggml_shape(tensor)))
392
+
393
+
394
+ def _spool_dir_for(path: Path) -> str | None:
395
+ """Pick a spool directory for gguf-py's temp file on the output filesystem.
396
+
397
+ gguf-py spools tensor payloads to a SpooledTemporaryFile, which lives under
398
+ the process temp dir (often a small tmpfs) once it exceeds 256 MiB. A
399
+ multi-GB Q2_K output overflows that, so keep the spool next to the output.
400
+ """
401
+ try:
402
+ candidate = path.resolve().parent
403
+ probe = candidate / f".smq-spool-probe-{os.getpid()}"
404
+ probe.write_bytes(b"")
405
+ probe.unlink()
406
+ return str(candidate)
407
+ except OSError:
408
+ return None
409
+
410
+
411
+ def tensor_is_eligible(tensor: Any) -> tuple[bool, str]:
412
+ """Q2_K eligibility: a weight matrix with ne[0] % 256 == 0.
413
+
414
+ "Weight matrix" means at least two GGML dimensions and no 1-sized
415
+ dimension. Everything else is left byte-identical (pass-through). In
416
+ particular, 1-D tensors are elementwise parameters (norms, biases, ...):
417
+ llama.cpp multiplies/adds them directly, and those ops require
418
+ floating-point operands, so quantizing them to Q2_K breaks inference with
419
+ "binary_op: unsupported types: dst: f32, src0: f32, src1: q2_K".
420
+ """
421
+ shape = tensor_ggml_shape(tensor)
422
+ if not shape:
423
+ return False, "scalar"
424
+ ne0 = shape[0]
425
+ n = int(np.prod(shape, dtype=np.int64))
426
+ if n == 0:
427
+ return False, "empty tensor"
428
+ if ne0 % QK:
429
+ return False, f"GGML row length ne[0]={ne0} is not divisible by {QK}"
430
+ if n % QK:
431
+ return False, "element count is not divisible by 256"
432
+ if len(shape) < 2:
433
+ return False, "1-D elementwise parameter (norm/bias); must stay unquantized"
434
+ if any(int(d) == 1 for d in shape[1:]):
435
+ return False, "degenerate dimension (elementwise parameter); must stay unquantized"
436
+ return True, ""
437
+
438
+
439
+ def load_tensor_f32(tensor: Any, dequantize_fn) -> np.ndarray:
440
+ """Dequantize a GGUF tensor to its physical NumPy shape, preserving byte order."""
441
+ try:
442
+ arr = np.asarray(dequantize_fn(tensor.data, tensor.tensor_type), dtype=np.float32)
443
+ except Exception as exc:
444
+ raise RuntimeError(
445
+ f"could not dequantize tensor {tensor.name!r} ({tensor.tensor_type}): {exc}"
446
+ ) from exc
447
+ expected_shape = tensor_numpy_shape(tensor)
448
+ expected_size = int(np.prod(expected_shape, dtype=np.int64))
449
+ if arr.size != expected_size:
450
+ raise ValueError(
451
+ f"dequantized size mismatch for {tensor.name}: got {arr.size}, "
452
+ f"expected {expected_size} for physical shape {expected_shape}"
453
+ )
454
+ arr = np.ascontiguousarray(arr.reshape(expected_shape), dtype=np.float32)
455
+ if not arr.flags.writeable: # dequantizers may hand back read-only views
456
+ arr = arr.copy()
457
+ if not np.isfinite(arr).all():
458
+ raise ValueError(f"tensor {tensor.name!r} contains NaN or infinity")
459
+ return arr
460
+
461
+
462
+ def _new_metric_accumulator() -> dict[str, float]:
463
+ return {
464
+ "n": 0.0,
465
+ "err2": 0.0,
466
+ "ref2": 0.0,
467
+ "xerr": 0.0,
468
+ "abs_err": 0.0,
469
+ "max_abs": 0.0,
470
+ }
471
+
472
+
473
+ def _accumulate_moments(
474
+ acc: dict[str, float],
475
+ n_elements: int,
476
+ err2_blocks: np.ndarray,
477
+ xerr_blocks: np.ndarray,
478
+ ref2_blocks: np.ndarray,
479
+ abs_err: float | None = None,
480
+ max_abs: float | None = None,
481
+ ) -> None:
482
+ """Accumulate metrics from per-sub-block moment sums (no residual arrays).
483
+
484
+ The residual metrics are recovered algebraically from second moments:
485
+ with diff = rec - ref, sum(diff^2) is the SSE, sum(ref*diff) = xerr, and
486
+ sum(ref^2) = ref2, so rmse/rel_rmse/cosine need no reconstruction pass.
487
+ ``abs_err``/``max_abs`` are supplied only when a diff array exists.
488
+ """
489
+ acc["n"] += float(n_elements)
490
+ acc["err2"] += float(np.asarray(err2_blocks, np.float64).sum())
491
+ acc["xerr"] += float(np.asarray(xerr_blocks, np.float64).sum())
492
+ acc["ref2"] += float(np.asarray(ref2_blocks, np.float64).sum())
493
+ if abs_err is not None:
494
+ acc["abs_err"] += float(abs_err)
495
+ if max_abs is not None:
496
+ acc["max_abs"] = max(acc["max_abs"], float(max_abs))
497
+
498
+
499
+ def _finish_metrics(acc: dict[str, float]) -> dict[str, float]:
500
+ n = max(acc["n"], 1.0)
501
+ rmse = math.sqrt(max(acc["err2"] / n, 0.0))
502
+ rms_ref = math.sqrt(max(acc["ref2"] / n, 0.0))
503
+ # rec = ref - err => sum(ref*rec) = ref2 - xerr, sum(rec^2) = ref2 - 2 xerr + err2
504
+ dot = acc["ref2"] - acc["xerr"]
505
+ rec2 = max(acc["ref2"] - 2.0 * acc["xerr"] + acc["err2"], 0.0)
506
+ denom = math.sqrt(max(acc["ref2"], 0.0) * rec2)
507
+ cosine = dot / denom if denom > 0 else (1.0 if acc["ref2"] == 0 and rec2 == 0 else 0.0)
508
+ return {
509
+ "rmse": rmse,
510
+ "rel_rmse": rmse / max(rms_ref, 1.0e-30),
511
+ "mae": acc["abs_err"] / n,
512
+ "max_abs": acc["max_abs"],
513
+ "cosine": cosine,
514
+ "rms_reference": rms_ref,
515
+ }
516
+
517
+
518
+ def _snap_fp16(v: np.ndarray, floor: float = 0.0, d_floor: float = 2.0 ** -24) -> np.ndarray:
519
+ """Round to the fp16 storage lattice Q2_K uses for d / dmin."""
520
+ out = np.where(v > 0, v, floor).astype(np.float16).astype(np.float32)
521
+ return np.where(out <= 0, np.float32(d_floor), out)
522
+
523
+
524
+ def _reconstruct(d, dmin, ls, lm, q) -> np.ndarray:
525
+ """Q2_K decode of one chunk of parameters -> (n, 256) float32."""
526
+ q = np.asarray(q, np.float32).reshape(len(np.asarray(d, np.float32)), QK)
527
+ return (
528
+ np.asarray(d, np.float32)[:, None]
529
+ * np.repeat(np.asarray(ls, np.float32), 16, axis=1)
530
+ * q
531
+ - np.asarray(dmin, np.float32)[:, None]
532
+ * np.repeat(np.asarray(lm, np.float32), 16, axis=1)
533
+ )
534
+
535
+
536
+ def _ls_grid_fit(
537
+ xb: np.ndarray,
538
+ q: np.ndarray,
539
+ step: np.ndarray,
540
+ lmin: np.ndarray,
541
+ *,
542
+ Sx: np.ndarray | None = None,
543
+ Sq: np.ndarray | None = None,
544
+ Sqq: np.ndarray | None = None,
545
+ Sxq: np.ndarray | None = None,
546
+ Sw: np.ndarray | None = None,
547
+ ):
548
+ """Exact closed-form least-squares factorization x ~ s*q - o per sub-block.
549
+
550
+ For fixed integer codes q this is the quadratic projection of each 16-wide
551
+ sub-block onto span{q, 1}: solving the 2x2 normal equations of
552
+ sum_i (x_i - s q_i + o)^2 gives
553
+ s = (Sw Sxq - Sq Sx) / (Sw Sqq - Sq^2), o = (Sq Sxq - Sqq Sx) / det,
554
+ with Sw = 16 for uniform weights (importance weighting only changes the
555
+ moments, not the shape of the solution). Q2_K can only subtract a
556
+ nonnegative offset, so o < 0 is clamped to zero and s is refit along that
557
+ boundary (s = Sxq / Sqq). Degenerate sub-blocks (all-equal codes, or weight
558
+ concentrated on one element) are unidentifiable and fall back to the RTN
559
+ grid. Returns (s, o), both nonnegative.
560
+
561
+ The moment sums may be passed in when the caller already has them.
562
+ """
563
+ if Sq is None:
564
+ Sq = _sum_last(q)
565
+ if Sqq is None:
566
+ Sqq = _sq_sum_last(q)
567
+ if Sx is None:
568
+ Sx = _sum_last(xb)
569
+ if Sxq is None:
570
+ Sxq = _dot_last(xb, q)
571
+ Sqq = np.asarray(Sqq, np.float64)
572
+ Sq = np.asarray(Sq, np.float64)
573
+ Sx = np.asarray(Sx, np.float64)
574
+ Sxq = np.asarray(Sxq, np.float64)
575
+ if Sw is None:
576
+ Sw = np.float64(16.0)
577
+ dw = 16.0
578
+ else:
579
+ Sw = np.asarray(Sw, np.float64)
580
+ dw = np.maximum(Sw, 1.0e-30)
581
+ det = Sw * Sqq - Sq * Sq
582
+ good = det > 1.0e-3 * dw * dw
583
+ det_s = np.where(good, det, 1.0)
584
+ s = np.where(good, (Sw * Sxq - Sq * Sx) / det_s, step)
585
+ o = np.where(good, (Sq * Sxq - Sqq * Sx) / det_s, lmin)
586
+ neg = o < 0.0
587
+ s = np.where(neg & (Sqq > 1.0e-12), Sxq / np.where(Sqq > 1.0e-12, Sqq, 1.0), s)
588
+ o = np.where(neg, 0.0, o)
589
+ s = np.where(s > _TINY, s, step)
590
+ s = np.maximum(s, 0.0)
591
+ o = np.maximum(o, 0.0)
592
+ return s.astype(np.float32), o.astype(np.float32)
593
+
594
+
595
+ def _factorize_chunk(
596
+ xb: np.ndarray,
597
+ rtn_baseline: bool = True,
598
+ need_diff: bool = True,
599
+ w: np.ndarray | None = None,
600
+ ) -> tuple[dict[str, np.ndarray], dict[str, np.ndarray] | None, dict[str, np.ndarray], np.ndarray | None]:
601
+ """Closed-form quadratic factorization of one chunk of blocks.
602
+
603
+ xb: (nb, 16, 16) float32. Returns (params, rtn_params, extras, err_rtn).
604
+
605
+ The pipeline is two exact least-squares projections and one lattice snap:
606
+ 1. RTN grid (min/max -> d/dmin fp16, ls/lm) gives codes q0;
607
+ 2. q0 is projected onto span{q, 1} in closed form to get continuous
608
+ (s, o) = argmin sum_i w_i (x_i - s q0_i + o)^2 per sub-block;
609
+ 3. (s, o) are snapped onto the Q2_K lattice, then the codes are re-derived
610
+ once against that final grid (q1) and the better of decomposition vs
611
+ RTN is kept per block (in the same w-weighted norm).
612
+
613
+ ``w`` is an optional (nb, 16, 16) importance factor (uniform when None).
614
+ Residual SSEs and the cross term come from per-sub-block second moments
615
+ (Sw, Sxx, Sx, Sxq, Sq, Sqq), so neither residual array is materialized for
616
+ the guard or the metrics. ``extras`` carries those per-block SSE/xerr/ref2
617
+ moments, plus (diff, sq) only when ``need_diff`` is set.
618
+ """
619
+ nb = xb.shape[0]
620
+ w3 = None if w is None else np.asarray(w, np.float32)
621
+ lo, hi = _aminmax_last(xb)
622
+ lmin = np.maximum(-lo, 0.0)
623
+ step = np.maximum(hi + lmin, 0.0) / 3.0
624
+
625
+ max_step = step.max(axis=1)
626
+ max_off = lmin.max(axis=1)
627
+ d = _snap_fp16(np.divide(max_step, 15.0, out=np.zeros_like(max_step), where=max_step > 0))
628
+ dmin = _snap_fp16(np.divide(max_off, 15.0, out=np.zeros_like(max_off), where=max_off > 0))
629
+ ls0 = np.rint(step / d[:, None]).clip(0, 15).astype(np.uint8)
630
+ lm0 = np.rint(
631
+ np.divide(lmin, dmin[:, None], out=np.zeros_like(lmin), where=dmin[:, None] > 0)
632
+ ).clip(0, 15).astype(np.uint8)
633
+
634
+ s0 = d[:, None] * ls0 # (nb, 16) grid step per sub-block
635
+ o0 = dmin[:, None] * lm0
636
+ safe_s0 = np.where(s0 > _TINY, s0, 1.0e30)
637
+ q0 = xb + o0[:, :, None]
638
+ q0 /= safe_s0[:, :, None]
639
+ np.clip(q0, 0.0, 3.0, out=q0)
640
+ np.rint(q0, out=q0)
641
+
642
+ # Second moments of the reference; reused by every SSE and the metrics.
643
+ if w3 is None:
644
+ Sx = _sum_last(xb)
645
+ Sxx = _sq_sum_last(xb)
646
+ Sw = None
647
+ Sq0 = _sum_last(q0)
648
+ Sqq0 = _sq_sum_last(q0)
649
+ Sxq0 = _dot_last(xb, q0)
650
+ else:
651
+ Sx = _einsum_last(w3, xb)
652
+ Sxx = _einsum_last(w3, xb, xb)
653
+ Sw = _sum_last(w3)
654
+ Sq0 = _einsum_last(w3, q0)
655
+ Sqq0 = _einsum_last(w3, q0, q0)
656
+ Sxq0 = _einsum_last(w3, xb, q0)
657
+
658
+ if rtn_baseline:
659
+ err_rtn = _sse_from_moments(Sxx, Sx, Sxq0, Sq0, Sqq0, s0, o0, Sw=Sw)
660
+ else:
661
+ err_rtn = None
662
+
663
+ # Exact LS factorization x ~ s*q0 - o for the RTN codes, reusing moments.
664
+ s1, o1 = _ls_grid_fit(
665
+ xb, q0, step, lmin, Sx=Sx, Sq=Sq0, Sqq=Sqq0, Sxq=Sxq0, Sw=Sw
666
+ )
667
+
668
+ # Snap the fitted (s, o) onto the Q2_K integer/fp16 parameter lattice and
669
+ # re-derive the codes once against that final grid.
670
+ d2 = _snap_fp16(np.divide(s1.max(axis=1), 15.0, out=np.zeros(nb), where=s1.max(axis=1) > 0))
671
+ dmin2 = _snap_fp16(np.divide(o1.max(axis=1), 15.0, out=np.zeros(nb), where=o1.max(axis=1) > 0))
672
+ ls1 = np.rint(s1 / d2[:, None]).clip(0, 15).astype(np.uint8)
673
+ lm1 = np.rint(
674
+ np.divide(o1, dmin2[:, None], out=np.zeros_like(o1), where=dmin2[:, None] > 0)
675
+ ).clip(0, 15).astype(np.uint8)
676
+ s2 = d2[:, None] * ls1
677
+ o2 = dmin2[:, None] * lm1
678
+ safe_s2 = np.where(s2 > _TINY, s2, 1.0e30)
679
+ q1 = xb + o2[:, :, None]
680
+ q1 /= safe_s2[:, :, None]
681
+ np.clip(q1, 0.0, 3.0, out=q1)
682
+ np.rint(q1, out=q1)
683
+
684
+ if w3 is None:
685
+ Sq1 = _sum_last(q1)
686
+ Sqq1 = _sq_sum_last(q1)
687
+ Sxq1 = _dot_last(xb, q1)
688
+ else:
689
+ Sq1 = _einsum_last(w3, q1)
690
+ Sqq1 = _einsum_last(w3, q1, q1)
691
+ Sxq1 = _einsum_last(w3, xb, q1)
692
+ err_dec = _sse_from_moments(Sxx, Sx, Sxq1, Sq1, Sqq1, s2, o2, Sw=Sw)
693
+ # sum(w * x * (rec - x)) = s*Sxq - o*Sx - Sxx, used for residual metrics.
694
+ xerr_dec = (
695
+ np.asarray(s2, np.float64) * Sxq1
696
+ - np.asarray(o2, np.float64) * Sx
697
+ - Sxx.astype(np.float64)
698
+ )
699
+
700
+ if rtn_baseline:
701
+ # The guard is per 256-weight block: compare the block-total SSEs.
702
+ take = err_dec.sum(axis=1) <= err_rtn.sum(axis=1)
703
+ else:
704
+ take = None
705
+
706
+ if take is None or bool(take.all()):
707
+ d_out, dmin_out = d2, dmin2
708
+ ls_out, lm_out = ls1, lm1
709
+ codes_out = q1.astype(np.uint8).reshape(nb, QK)
710
+ else:
711
+ d_out = np.where(take, d2, d)
712
+ dmin_out = np.where(take, dmin2, dmin)
713
+ ls_out = np.where(take[:, None], ls1, ls0)
714
+ lm_out = np.where(take[:, None], lm1, lm0)
715
+ codes_out = np.where(take[:, None, None], q1, q0).astype(np.uint8).reshape(nb, QK)
716
+
717
+ params = {
718
+ "d": np.asarray(d_out, np.float64),
719
+ "dmin": np.asarray(dmin_out, np.float64),
720
+ "ls": ls_out.astype(np.uint8),
721
+ "lm": lm_out.astype(np.uint8),
722
+ "codes": codes_out,
723
+ }
724
+ rtn_params = None if not rtn_baseline else {
725
+ "d": d.astype(np.float64),
726
+ "dmin": dmin.astype(np.float64),
727
+ "ls": ls0,
728
+ "lm": lm0,
729
+ "codes": q0.astype(np.uint8).reshape(nb, QK),
730
+ }
731
+
732
+ diff = None
733
+ sq = None
734
+ if need_diff:
735
+ diff = s2[:, :, None] * q1
736
+ np.subtract(xb, diff, out=diff)
737
+ diff += o2[:, :, None]
738
+ diff = diff.reshape(nb, QK)
739
+ sq = diff * diff
740
+
741
+ extras = {
742
+ "err_dec": err_dec,
743
+ "xerr_dec": xerr_dec,
744
+ "ref2": Sxx,
745
+ "err_rtn": err_rtn,
746
+ "Sw": Sw,
747
+ "diff": diff,
748
+ "sq": sq,
749
+ }
750
+ return params, rtn_params, extras, err_rtn
751
+
752
+
753
+ def _sse_from_moments(Sxx, Sx, Sxq, Sq, Sqq, s, o, Sw=None) -> np.ndarray:
754
+ """Exact sum_i w_i (x_i - s q_i + o)^2 from second moments (per sub-block)."""
755
+ x2 = np.asarray(Sxx, np.float64)
756
+ x1 = np.asarray(Sx, np.float64)
757
+ xq = np.asarray(Sxq, np.float64)
758
+ q1 = np.asarray(Sq, np.float64)
759
+ q2 = np.asarray(Sqq, np.float64)
760
+ s = np.asarray(s, np.float64)
761
+ o = np.asarray(o, np.float64)
762
+ sw = np.float64(16.0) if Sw is None else np.asarray(Sw, np.float64)
763
+ err = (
764
+ x2
765
+ - 2.0 * s * xq
766
+ + 2.0 * o * x1
767
+ + s * s * q2
768
+ - 2.0 * s * o * q1
769
+ + sw * o * o
770
+ )
771
+ return np.maximum(err, 0.0)
772
+
773
+
774
+ def factorize_q2k(
775
+ x: np.ndarray,
776
+ chunk_blocks: int = 32768,
777
+ save_path: Path | None = None,
778
+ save_shape: tuple[int, ...] | None = None,
779
+ progress=None,
780
+ rtn_baseline: bool = True,
781
+ metrics: bool = True,
782
+ weights: Any = None,
783
+ ) -> tuple[np.ndarray, dict[str, float] | None, dict[str, float] | None, float]:
784
+ """Single-pass factorization of (n_blocks, 256) weights to packed Q2_K.
785
+
786
+ Packing, residual metrics and optional .npy reconstruction streaming all
787
+ happen in the same loop. With ``rtn_baseline`` the round-to-nearest
788
+ baseline error is measured from moments in the same pass (and the per-block
789
+ guard keeps the better of the two); without it the pass is pure quadratic
790
+ decomposition. ``weights`` is an optional per-element importance array
791
+ (same shape as ``x``) that weights the projection, the guard and the
792
+ reported metrics. Returns (packed_blocks, smq_metrics, rtn_metrics_or_None,
793
+ elapsed_seconds); the RTN metrics carry rmse/rel_rmse only and both metric
794
+ dicts are None when ``metrics`` is off.
795
+ """
796
+ n = int(x.shape[0])
797
+ chunk_blocks = max(1, int(chunk_blocks))
798
+ packed = np.empty((n, Q2K_BYTES), dtype=np.uint8)
799
+ full_weights = isinstance(weights, np.ndarray)
800
+ if full_weights and weights.shape != x.shape:
801
+ raise ValueError(f"weights shape {weights.shape} does not match x shape {x.shape}")
802
+ acc = _new_metric_accumulator() if metrics else None
803
+ rtn_acc = None if not (rtn_baseline and metrics) else {"n": 0.0, "err2": 0.0, "ref2": 0.0}
804
+ want_recon = save_path is not None and save_shape is not None
805
+ need_diff = bool(metrics or want_recon)
806
+ mmap = None
807
+ recon_flat = None
808
+ if want_recon:
809
+ save_path.parent.mkdir(parents=True, exist_ok=True)
810
+ mmap = np.lib.format.open_memmap(save_path, mode="w+", dtype=np.float32, shape=save_shape)
811
+ recon_flat = mmap.reshape(n, QK)
812
+ t0 = time.perf_counter()
813
+ for c0 in range(0, n, chunk_blocks):
814
+ c1 = min(c0 + chunk_blocks, n)
815
+ xb3 = x[c0:c1].reshape(c1 - c0, NSUB, 16)
816
+ if weights is None:
817
+ w3 = None
818
+ elif full_weights:
819
+ w3 = np.array(weights[c0:c1].reshape(c1 - c0, NSUB, 16), dtype=np.float32, copy=True)
820
+ else:
821
+ w3 = np.ascontiguousarray(weights.chunk(c0, c1).reshape(c1 - c0, NSUB, 16), dtype=np.float32)
822
+ params, _rtn_params, extras, _err_rtn = _factorize_chunk(
823
+ xb3, rtn_baseline=rtn_baseline, need_diff=need_diff, w=w3
824
+ )
825
+ packed[c0:c1] = pack_q2k_blocks(params)
826
+ if acc is not None:
827
+ diff = extras["diff"]
828
+ abs_err = max_abs = None
829
+ if diff is not None:
830
+ abs_err, max_abs = _abs_sum_max(diff, w3)
831
+ n_eff = float((c1 - c0) * QK) if w3 is None else float(
832
+ np.asarray(extras["Sw"], np.float64).sum()
833
+ )
834
+ _accumulate_moments(
835
+ acc, n_eff, extras["err_dec"], extras["xerr_dec"],
836
+ extras["ref2"], abs_err, max_abs,
837
+ )
838
+ if rtn_acc is not None:
839
+ rtn_acc["n"] += n_eff
840
+ rtn_acc["ref2"] += float(np.asarray(extras["ref2"], np.float64).sum())
841
+ rtn_acc["err2"] += float(np.asarray(extras["err_rtn"], np.float64).sum())
842
+ if recon_flat is not None:
843
+ recon_flat[c0:c1] = x[c0:c1] - extras["diff"]
844
+ if progress is not None:
845
+ progress(c1, n, f"quadratic factorization {c1:,}/{n:,} blocks")
846
+ if mmap is not None:
847
+ mmap.flush()
848
+ del recon_flat, mmap
849
+ rtn_stats = None
850
+ if rtn_acc is not None:
851
+ nn = max(rtn_acc["n"], 1.0)
852
+ rtn_rmse = math.sqrt(max(rtn_acc["err2"] / nn, 0.0))
853
+ rtn_stats = {
854
+ "rmse": rtn_rmse,
855
+ "rel_rmse": rtn_rmse / max(math.sqrt(rtn_acc["ref2"] / nn), 1.0e-30),
856
+ }
857
+ return packed, (_finish_metrics(acc) if acc is not None else None), rtn_stats, time.perf_counter() - t0
858
+
859
+
860
+ def pack_q2k_blocks(result: dict[str, Any]) -> np.ndarray:
861
+ """Pack SMQ block parameters into GGML's native 84-byte Q2_K layout.
862
+
863
+ GGML Q2_K block layout:
864
+ scales[16] = (lm[j] << 4) | ls[j]
865
+ qs[64] = packed 2-bit codes in groups of 32 and planes 0..3
866
+ d[2] = FP16 superblock scale
867
+ dmin[2] = FP16 superblock minimum scale
868
+
869
+ The code-plane ordering mirrors gguf-py's Q2_K dequantizer.
870
+ """
871
+ d = np.asarray(result["d"], dtype=np.float32).astype(np.float16)
872
+ dmin = np.asarray(result["dmin"], dtype=np.float32).astype(np.float16)
873
+ ls = np.asarray(result["ls"], dtype=np.uint8)
874
+ lm = np.asarray(result["lm"], dtype=np.uint8)
875
+ codes = np.asarray(result["codes"], dtype=np.uint8)
876
+
877
+ if ls.ndim != 2 or ls.shape[1] != NSUB or lm.shape != ls.shape:
878
+ raise ValueError(f"invalid Q2_K scale arrays: ls={ls.shape}, lm={lm.shape}")
879
+ n = ls.shape[0]
880
+ if d.shape != (n,) or dmin.shape != (n,) or codes.shape != (n, QK):
881
+ raise ValueError("invalid SMQ result dimensions for Q2_K packing")
882
+ if np.any(ls > 15) or np.any(lm > 15) or np.any(codes > 3):
883
+ raise ValueError("SMQ output contains values outside Q2_K code/scale ranges")
884
+
885
+ scales = ((lm << 4) | ls).astype(np.uint8)
886
+ qs = np.zeros((n, QK // 4), dtype=np.uint8)
887
+ # Q2_K's dequantized subblock order is [group, plane, half-of-32].
888
+ for sub in range(NSUB):
889
+ group = sub // 8
890
+ plane = (sub % 8) // 2
891
+ half = sub % 2
892
+ dst0 = group * 32 + half * 16
893
+ dst1 = dst0 + 16
894
+ src0 = sub * 16
895
+ src1 = src0 + 16
896
+ qs[:, dst0:dst1] |= (codes[:, src0:src1] << (2 * plane)).astype(np.uint8)
897
+
898
+ d_bytes = np.ascontiguousarray(d.astype("<f2", copy=False)).view(np.uint8).reshape(n, 2)
899
+ dm_bytes = np.ascontiguousarray(dmin.astype("<f2", copy=False)).view(np.uint8).reshape(n, 2)
900
+ packed = np.concatenate((scales, qs, d_bytes, dm_bytes), axis=1)
901
+ if packed.shape != (n, Q2K_BYTES):
902
+ raise AssertionError(f"Q2_K block must be {Q2K_BYTES} bytes, got {packed.shape}")
903
+ return np.ascontiguousarray(packed)
904
+
905
+
906
+ def unpack_q2k_blocks_for_test(packed: np.ndarray) -> np.ndarray:
907
+ """Small independent decoder used by --self-test; output shape (n,256)."""
908
+ p = np.asarray(packed, dtype=np.uint8)
909
+ if p.ndim != 2 or p.shape[1] != Q2K_BYTES:
910
+ raise ValueError("expected packed Q2_K array with shape (n,84)")
911
+ n = p.shape[0]
912
+ scales = p[:, :16]
913
+ qs = p[:, 16:80]
914
+ d = p[:, 80:82].copy().reshape(-1).view("<f2").astype(np.float32)
915
+ dmin = p[:, 82:84].copy().reshape(-1).view("<f2").astype(np.float32)
916
+ # This is the gguf-py dequantization ordering: 2 groups x 4 bit planes x 32.
917
+ shifts = np.array([0, 2, 4, 6], dtype=np.uint8).reshape(1, 1, 4, 1)
918
+ q = ((qs.reshape(n, 2, 1, 32) >> shifts) & np.uint8(3)).reshape(n, 16, 16)
919
+ dl = (d[:, None] * (scales & 0x0F).astype(np.float32)).reshape(n, 16, 1)
920
+ ml = (dmin[:, None] * (scales >> 4).astype(np.float32)).reshape(n, 16, 1)
921
+ return (dl * q.astype(np.float32) - ml).reshape(n, QK)
922
+
923
+
924
+ def safe_filename(name: str) -> str:
925
+ cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "_", name).strip("._")
926
+ return cleaned[:180] or "tensor"
927
+
928
+
929
+ def copy_gguf_metadata(reader: Any, writer: Any, GGUFValueType: Any) -> None:
930
+ """Copy source metadata without copying GGUF structural counters."""
931
+ # The writer constructor emits general.architecture; these two fields are
932
+ # deliberately regenerated below. Copying and then re-adding file_type /
933
+ # quantization_version would create duplicate GGUF keys.
934
+ structural = {
935
+ "GGUF.version", "GGUF.tensor_count", "GGUF.kv_count",
936
+ "general.architecture", "general.file_type", "general.quantization_version",
937
+ }
938
+ for key, field in reader.fields.items():
939
+ if key in structural:
940
+ continue
941
+ if not field.types:
942
+ continue
943
+ value_type = field.types[0]
944
+ value = field.contents()
945
+ if value_type == GGUFValueType.ARRAY:
946
+ writer.add_key_value(key, value, value_type, sub_type=field.types[-1])
947
+ else:
948
+ writer.add_key_value(key, value, value_type)
949
+
950
+ # Output is a mostly-Q2_K model. These fields are recognized by llama.cpp.
951
+ writer.add_uint32("general.file_type", Q2K_FILE_TYPE)
952
+ writer.add_uint32("general.quantization_version", GGML_QUANT_VERSION)
953
+ if "general.type" not in reader.fields:
954
+ writer.add_string("general.type", "model")
955
+
956
+
957
+ def add_tensor_passthrough(writer: Any, tensor: Any, reader: Any) -> None:
958
+ """Copy one tensor in its original GGML type and raw byte representation."""
959
+ data = np.asarray(tensor.data)
960
+ if data.dtype == np.uint8:
961
+ writer.add_tensor(
962
+ tensor.name,
963
+ np.ascontiguousarray(data),
964
+ raw_dtype=tensor.tensor_type,
965
+ tensor_endianess=reader.endianess,
966
+ )
967
+ else:
968
+ writer.add_tensor(
969
+ tensor.name,
970
+ np.ascontiguousarray(data),
971
+ tensor_endianess=reader.endianess,
972
+ )
973
+
974
+
975
+ def build_parser() -> argparse.ArgumentParser:
976
+ p = argparse.ArgumentParser(
977
+ description="Factor eligible GGUF tensors directly into Q2_K by closed-form quadratic decomposition.",
978
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
979
+ )
980
+ p.add_argument("gguf", type=Path, nargs="?", help="Input model GGUF")
981
+ p.add_argument("--output-gguf", "--out-gguf", type=Path,
982
+ help="Write a complete llama.cpp-compatible GGUF with eligible tensors quantized to Q2_K")
983
+ p.add_argument("--list-tensors", action="store_true", help="List Q2_K-row-eligible tensors and exit")
984
+ p.add_argument("--tensor", action="append", default=[],
985
+ help="Select tensor by exact name or substring (repeatable); others are copied unchanged")
986
+ p.add_argument("--tensor-pattern", help="Select tensors using a regular expression; others copied unchanged")
987
+ p.add_argument("--imatrix", type=Path,
988
+ help="Optional llama.cpp importance matrix (.dat); weights the quadratic fit, "
989
+ "per-block guard and reported metrics. Tensors absent from it use uniform weights.")
990
+ p.add_argument("--max-tensors", type=int, default=0,
991
+ help="Stop after N eligible tensors (0 = all); remaining tensors are copied unchanged")
992
+ p.add_argument("--chunk-blocks", type=int, default=32768, help="Blocks per factorization chunk")
993
+ p.add_argument("--progress-interval", type=float, default=5.0,
994
+ help="Seconds between long-stage heartbeat messages (0.5 minimum)")
995
+ p.add_argument("--save-recon", type=Path,
996
+ help="Optional directory to save each reconstruction as float32 .npy")
997
+ p.add_argument("--out", type=Path, default=Path("smq_gguf_bench"),
998
+ help="Output path prefix for .csv and .json reports")
999
+ p.add_argument("--quiet", action="store_true", help="Reduce per-chunk progress output")
1000
+ p.add_argument("--no-rtn-metrics", action="store_true",
1001
+ help="Skip the RTN baseline reconstruction, per-block guard and metrics "
1002
+ "(fastest pure quadratic decomposition; the output is then unguarded)")
1003
+ p.add_argument("--no-metrics", action="store_true",
1004
+ help="Skip residual metrics entirely (no per-element abs/max pass)")
1005
+ p.add_argument("--self-test", action="store_true", help="Run packing and factorization self-tests and exit")
1006
+ return p
1007
+
1008
+
1009
+ def run_self_test() -> int:
1010
+ rng = np.random.default_rng(20260115)
1011
+
1012
+ # 1. Packing roundtrip on adversarial parameter ranges.
1013
+ n = 37
1014
+ result = {
1015
+ "d": rng.uniform(1e-4, 0.15, n).astype(np.float32),
1016
+ "dmin": rng.uniform(0.0, 0.05, n).astype(np.float32),
1017
+ "ls": rng.integers(0, 16, size=(n, 16), dtype=np.uint8),
1018
+ "lm": rng.integers(0, 16, size=(n, 16), dtype=np.uint8),
1019
+ "codes": rng.integers(0, 4, size=(n, 256), dtype=np.uint8),
1020
+ }
1021
+ packed = pack_q2k_blocks(result)
1022
+ decoded = unpack_q2k_blocks_for_test(packed)
1023
+ emitted = dict(result)
1024
+ emitted["d"] = np.asarray(result["d"], dtype=np.float32).astype(np.float16).astype(np.float32)
1025
+ emitted["dmin"] = np.asarray(result["dmin"], dtype=np.float32).astype(np.float16).astype(np.float32)
1026
+ expected = _reconstruct(emitted["d"], emitted["dmin"], emitted["ls"], emitted["lm"], emitted["codes"])
1027
+ if not np.array_equal(decoded, expected):
1028
+ raise AssertionError(f"Q2_K pack/decode mismatch; max error={np.max(np.abs(decoded - expected))}")
1029
+ print(f"Q2_K packing self-test: PASS ({n} blocks, {n * Q2K_BYTES} bytes)")
1030
+
1031
+ # 2. Factorization: roundtrip exactness, guards, and never-worse-than-RTN
1032
+ # across Gaussian, heavy-tailed, constant, zero and outlier blocks.
1033
+ nb = 4096
1034
+ gauss = rng.standard_normal((nb // 4, QK)) * 0.02
1035
+ heavy = rng.standard_t(3, (nb // 4, QK)) * 0.02
1036
+ const = np.repeat((rng.standard_normal((nb // 8, 1)) * 0.02), QK, axis=1)
1037
+ sparse = rng.standard_normal((nb // 8, QK)) * 0.02 * (rng.random((nb // 8, QK)) < 0.05)
1038
+ outs = gauss[: nb - nb // 4 - nb // 8 - nb // 8].copy()
1039
+ outs[:, ::97] += 0.5 # rare large outliers
1040
+ x = np.concatenate((gauss, heavy, const, sparse, outs), axis=0).astype(np.float32)
1041
+ params, rtn, _, _ = _factorize_chunk(x.reshape(-1, NSUB, 16))
1042
+
1043
+ dec = _reconstruct(params["d"], params["dmin"], params["ls"], params["lm"], params["codes"])
1044
+ rec_rtn = _reconstruct(rtn["d"], rtn["dmin"], rtn["ls"], rtn["lm"], rtn["codes"])
1045
+ if not np.isfinite(dec).all():
1046
+ raise AssertionError("factorization produced non-finite reconstruction")
1047
+ if params["codes"].max() > 3 or params["ls"].max() > 15 or params["lm"].max() > 15:
1048
+ raise AssertionError("factorization produced out-of-range Q2_K parameters")
1049
+ if (np.asarray(params["d"]) <= 0).any() or (np.asarray(params["dmin"]) < 0).any():
1050
+ raise AssertionError("factorization produced invalid d/dmin")
1051
+
1052
+ err_dec = ((dec - x) ** 2).reshape(nb, QK).sum(axis=1)
1053
+ err_rtn = ((rec_rtn - x) ** 2).reshape(nb, QK).sum(axis=1)
1054
+ worse = int(np.sum(err_dec > err_rtn * (1.0 + 1e-6)))
1055
+ if worse:
1056
+ raise AssertionError(f"{worse} blocks worse than RTN (guard failed)")
1057
+
1058
+ rel_dec = math.sqrt(float((err_dec.sum() / x.size))) / math.sqrt(float((x * x).mean()))
1059
+ rel_rtn = math.sqrt(float((err_rtn.sum() / x.size))) / math.sqrt(float((x * x).mean()))
1060
+ gain = 100.0 * (1.0 - rel_dec / rel_rtn)
1061
+ if rel_dec > rel_rtn:
1062
+ raise AssertionError(f"factorization worse than RTN overall: {rel_dec} vs {rel_rtn}")
1063
+ print(f"factorization self-test: PASS (blocks never worse than RTN; relRMSE {rel_rtn:.5f} -> {rel_dec:.5f}, {gain:.1f}% better)")
1064
+ return 0
1065
+
1066
+
1067
+ def write_reports(args: argparse.Namespace, rows: list[dict[str, Any]], skipped: list[dict[str, Any]],
1068
+ eligible_count: int, selected_count: int, quantized_count: int,
1069
+ passthrough_count: int) -> tuple[Path, Path]:
1070
+ out_prefix = args.out
1071
+ if out_prefix.suffix.lower() in (".csv", ".json"):
1072
+ out_prefix = out_prefix.with_suffix("")
1073
+ out_prefix.parent.mkdir(parents=True, exist_ok=True)
1074
+ csv_path = out_prefix.with_suffix(".csv")
1075
+ json_path = out_prefix.with_suffix(".json")
1076
+
1077
+ fieldnames = sorted({key for row in rows for key in row.keys() if not isinstance(row[key], (list, dict))})
1078
+ with csv_path.open("w", newline="", encoding="utf-8") as handle:
1079
+ if fieldnames:
1080
+ writer = csv.DictWriter(handle, fieldnames=fieldnames)
1081
+ writer.writeheader()
1082
+ for row in rows:
1083
+ writer.writerow({key: row.get(key, "") for key in fieldnames})
1084
+
1085
+ report = {
1086
+ "input_gguf": str(args.gguf),
1087
+ "output_gguf": str(args.output_gguf) if args.output_gguf else None,
1088
+ "selected_count": selected_count,
1089
+ "quantized_count": quantized_count,
1090
+ "eligible_count": eligible_count,
1091
+ "passthrough_count": passthrough_count,
1092
+ "skipped_count": len(skipped),
1093
+ "results": rows,
1094
+ "skipped_tensors": skipped,
1095
+ }
1096
+ json_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
1097
+ return csv_path, json_path
1098
+
1099
+
1100
+ def main() -> int:
1101
+ args = build_parser().parse_args()
1102
+ if args.self_test:
1103
+ return run_self_test()
1104
+ if args.gguf is None:
1105
+ raise SystemExit("Input model GGUF is required unless --self-test is used")
1106
+ if not args.gguf.is_file():
1107
+ raise SystemExit(f"Input GGUF file does not exist: {args.gguf}")
1108
+ if args.max_tensors < 0 or args.chunk_blocks < 1 or args.progress_interval <= 0:
1109
+ raise SystemExit("max-tensors must be >=0; chunk-blocks positive; progress-interval positive")
1110
+ if args.output_gguf and args.output_gguf.resolve() == args.gguf.resolve():
1111
+ raise SystemExit("Output GGUF must be different from input GGUF")
1112
+
1113
+ gguf, GGUFReader, dequantize_fn, GGUFWriter, GGMLQuantizationType, GGUFValueType, GGUFEndian = load_gguf()
1114
+ reader = GGUFReader(str(args.gguf))
1115
+ imatrix = None
1116
+ if args.imatrix is not None:
1117
+ if not args.imatrix.is_file():
1118
+ raise SystemExit(f"Importance matrix does not exist: {args.imatrix}")
1119
+ imatrix = ImatrixFile(args.imatrix)
1120
+ print(
1121
+ f"Importance matrix: {args.imatrix} "
1122
+ f"({imatrix.entry_count} tensor entries, {imatrix.kind} format)"
1123
+ )
1124
+ imatrix_hits = 0
1125
+ imatrix_misses = 0
1126
+ if args.output_gguf and reader.endianess != GGUFEndian.LITTLE:
1127
+ raise SystemExit(
1128
+ "Writing currently supports little-endian input GGUF files only. "
1129
+ "Convert this model to little-endian GGUF before conversion; "
1130
+ "raw pass-through tensors are not byte-swapped."
1131
+ )
1132
+ eligible: list[Any] = []
1133
+ skipped_shape: list[dict[str, Any]] = []
1134
+ for tensor in reader.tensors:
1135
+ ok, reason = tensor_is_eligible(tensor)
1136
+ if ok:
1137
+ eligible.append(tensor)
1138
+ else:
1139
+ skipped_shape.append({
1140
+ "tensor": tensor.name,
1141
+ "shape_ggml": list(tensor_ggml_shape(tensor)),
1142
+ "reason": reason,
1143
+ "gguf_type": tensor.tensor_type.name,
1144
+ })
1145
+
1146
+ if args.list_tensors:
1147
+ print(f"GGUF: {args.gguf}")
1148
+ print(f"Q2_K row-eligible tensors: {len(eligible)} / {len(reader.tensors)}")
1149
+ print("NAME\tGGML_SHAPE(ne0,...)\tSOURCE_TYPE\tELEMENTS")
1150
+ for tensor in eligible:
1151
+ print(f"{tensor.name}\t{tensor_ggml_shape(tensor)}\t{tensor.tensor_type.name}\t{tensor.n_elements}")
1152
+ if skipped_shape:
1153
+ print(
1154
+ f"\nNot eligible (copied unchanged): {len(skipped_shape)} "
1155
+ f"(ne[0] must be divisible by {QK}; 1-D elementwise parameters excluded)",
1156
+ file=sys.stderr,
1157
+ )
1158
+ for entry in skipped_shape:
1159
+ print(
1160
+ f"{entry['tensor']}\t{entry['shape_ggml']}\t{entry['gguf_type']}\t{entry['reason']}",
1161
+ file=sys.stderr,
1162
+ )
1163
+ return 0
1164
+
1165
+ selected = []
1166
+ for tensor in eligible:
1167
+ if args.tensor and not any((s == tensor.name or s in tensor.name) for s in args.tensor):
1168
+ continue
1169
+ if args.tensor_pattern and re.search(args.tensor_pattern, tensor.name) is None:
1170
+ continue
1171
+ selected.append(tensor)
1172
+ if args.max_tensors:
1173
+ selected = selected[:args.max_tensors]
1174
+ selected_names = {tensor.name for tensor in selected}
1175
+ if not selected:
1176
+ raise SystemExit("No eligible tensors selected. Use --list-tensors to inspect the model.")
1177
+
1178
+ q2k_type = GGMLQuantizationType.Q2_K
1179
+
1180
+ writer = None
1181
+ prev_tempdir: str | None = None
1182
+ spool_dir: str | None = None
1183
+ tempdir_overridden = False
1184
+ tmp_output = None
1185
+ arch = None
1186
+ if args.output_gguf:
1187
+ args.output_gguf.parent.mkdir(parents=True, exist_ok=True)
1188
+ tmp_output = args.output_gguf.with_name(args.output_gguf.name + ".partial")
1189
+ if tmp_output.exists():
1190
+ tmp_output.unlink()
1191
+ arch_field = reader.fields.get("general.architecture")
1192
+ if arch_field is None:
1193
+ raise SystemExit("Input GGUF has no general.architecture metadata; cannot write model GGUF")
1194
+ arch = str(arch_field.contents())
1195
+ spool_dir = _spool_dir_for(args.output_gguf)
1196
+ if spool_dir is not None:
1197
+ prev_tempdir = tempfile.tempdir
1198
+ tempdir_overridden = tempfile.tempdir != spool_dir
1199
+ tempfile.tempdir = spool_dir
1200
+ if not args.quiet:
1201
+ print(f"GGUF tensor spool directory: {spool_dir}", flush=True)
1202
+ writer = GGUFWriter(
1203
+ str(tmp_output),
1204
+ arch=arch,
1205
+ use_temp_file=True,
1206
+ endianess=GGUFEndian.LITTLE,
1207
+ )
1208
+ writer.data_alignment = int(reader.alignment)
1209
+ copy_gguf_metadata(reader, writer, GGUFValueType)
1210
+
1211
+ rows: list[dict[str, Any]] = []
1212
+ skipped: list[dict[str, Any]] = list(skipped_shape)
1213
+ print(
1214
+ f"Selected tensors: {len(selected)} / {len(eligible)} eligible | total={len(reader.tensors)} "
1215
+ f"| output={'Q2_K GGUF' if writer is not None else 'reports only'}"
1216
+ )
1217
+ print("tensor | shape | blocks | Q2K relRMSE | RTN relRMSE | gain | time(s)")
1218
+ print("-" * 112)
1219
+
1220
+ selected_done = 0
1221
+ quantized_count = 0
1222
+ passthrough_count = 0
1223
+ start_total = time.perf_counter()
1224
+ try:
1225
+ for tensor_index, tensor in enumerate(reader.tensors, 1):
1226
+ eligible_tensor, _eligibility_reason = tensor_is_eligible(tensor)
1227
+ should_quantize = eligible_tensor and tensor.name in selected_names
1228
+
1229
+ if not should_quantize:
1230
+ if writer is not None:
1231
+ add_tensor_passthrough(writer, tensor, reader)
1232
+ passthrough_count += 1
1233
+ if not args.quiet and (tensor_index == 1 or tensor_index % 25 == 0 or tensor_index == len(reader.tensors)):
1234
+ pct = 100.0 * tensor_index / max(len(reader.tensors), 1)
1235
+ print(
1236
+ f"[GGUF copy] input tensors {tensor_index}/{len(reader.tensors)} ({pct:.1f}%) "
1237
+ f"| factorized {selected_done}/{len(selected)} | copied unchanged {passthrough_count}",
1238
+ flush=True,
1239
+ )
1240
+ continue
1241
+
1242
+ selected_done += 1
1243
+ tensor_prefix = (
1244
+ f"[{selected_done}/{len(selected)} selected | input {tensor_index}/{len(reader.tensors)}] "
1245
+ f"{tensor.name} shape={tensor_numpy_shape(tensor)}"
1246
+ )
1247
+ n_elements = int(tensor.n_elements)
1248
+ blocks_this = n_elements // QK
1249
+ if not args.quiet:
1250
+ done_pct = 100.0 * (selected_done - 1) / max(len(selected), 1)
1251
+ print(
1252
+ f"\n{tensor_prefix} | {n_elements:,} weights | {blocks_this:,} blocks "
1253
+ f"| overall selected progress {done_pct:.1f}% | BEGIN",
1254
+ flush=True,
1255
+ )
1256
+
1257
+ t_load = time.perf_counter()
1258
+ try:
1259
+ with progress_stage("dequantize source tensor", tensor_prefix, args.progress_interval, args.quiet):
1260
+ original = load_tensor_f32(tensor, dequantize_fn)
1261
+ except Exception as exc:
1262
+ skipped.append({
1263
+ "tensor": tensor.name,
1264
+ "shape_ggml": list(tensor_ggml_shape(tensor)),
1265
+ "reason": f"dequantization failed; copied unchanged: {exc}",
1266
+ "gguf_type": tensor.tensor_type.name,
1267
+ })
1268
+ if writer is not None:
1269
+ with progress_stage("copy original tensor unchanged", tensor_prefix, args.progress_interval, args.quiet):
1270
+ add_tensor_passthrough(writer, tensor, reader)
1271
+ passthrough_count += 1
1272
+ print(f"WARNING: {tensor.name}: dequantization failed; copied unchanged: {exc}", file=sys.stderr, flush=True)
1273
+ continue
1274
+
1275
+ np_shape = original.shape
1276
+ x = original.reshape(-1, QK)
1277
+
1278
+ tensor_weights = None
1279
+ if imatrix is not None:
1280
+ try:
1281
+ tensor_weights = imatrix.weights(tensor.name, tensor_ggml_shape(tensor))
1282
+ except ValueError as exc:
1283
+ imatrix_misses += 1
1284
+ print(
1285
+ f"WARNING: {tensor.name}: {exc}; using uniform weights",
1286
+ file=sys.stderr, flush=True,
1287
+ )
1288
+ else:
1289
+ if tensor_weights is None:
1290
+ imatrix_misses += 1
1291
+ if not args.quiet:
1292
+ print(f" note: {tensor.name} not present in imatrix; using uniform weights", flush=True)
1293
+ else:
1294
+ imatrix_hits += 1
1295
+
1296
+ def progress(done: int, total: int, message: str, _name=tensor.name):
1297
+ if not args.quiet:
1298
+ pct = 100.0 * done / max(total, 1)
1299
+ print(f" [{selected_done}/{len(selected)}] {_name}: {message} ({pct:.1f}% of tensor)", flush=True)
1300
+
1301
+ recon_path = None
1302
+ if args.save_recon:
1303
+ recon_path = args.save_recon / f"{safe_filename(tensor.name)}.npy"
1304
+
1305
+ with progress_stage("quadratic factorization to Q2_K", tensor_prefix, args.progress_interval, args.quiet):
1306
+ packed_blocks, dec_stats, rtn_stats, elapsed = factorize_q2k(
1307
+ x,
1308
+ chunk_blocks=args.chunk_blocks,
1309
+ save_path=recon_path,
1310
+ save_shape=np_shape,
1311
+ progress=None if args.quiet else progress,
1312
+ rtn_baseline=not args.no_rtn_metrics,
1313
+ metrics=not args.no_metrics,
1314
+ weights=tensor_weights,
1315
+ )
1316
+
1317
+ if rtn_stats is not None and dec_stats is not None:
1318
+ gain_pct = 100.0 * (1.0 - dec_stats["rel_rmse"] / max(rtn_stats["rel_rmse"], 1.0e-30))
1319
+ else:
1320
+ gain_pct = None
1321
+ row = {
1322
+ "tensor": tensor.name,
1323
+ "shape_numpy": list(np_shape),
1324
+ "shape_ggml": list(tensor_ggml_shape(tensor)),
1325
+ "source_gguf_type": tensor.tensor_type.name,
1326
+ "output_gguf_type": "Q2_K",
1327
+ "importance_weighted": tensor_weights is not None,
1328
+ "elements": n_elements,
1329
+ "blocks_256": blocks_this,
1330
+ "smq_rmse": dec_stats["rmse"] if dec_stats else None,
1331
+ "smq_rel_rmse": dec_stats["rel_rmse"] if dec_stats else None,
1332
+ "smq_mae": dec_stats["mae"] if dec_stats else None,
1333
+ "smq_max_abs": dec_stats["max_abs"] if dec_stats else None,
1334
+ "smq_cosine": dec_stats["cosine"] if dec_stats else None,
1335
+ "rms_reference": dec_stats["rms_reference"] if dec_stats else None,
1336
+ "improvement_pct_vs_rtn": gain_pct,
1337
+ "factorization_elapsed_s": elapsed,
1338
+ "load_seconds": time.perf_counter() - t_load - elapsed,
1339
+ "million_elements_per_s": n_elements / 1.0e6 / max(elapsed, 1.0e-12),
1340
+ }
1341
+ rows.append(row)
1342
+
1343
+ if recon_path is not None:
1344
+ row["reconstruction_file"] = str(recon_path)
1345
+
1346
+ if writer is not None:
1347
+ ne0 = tensor_ggml_shape(tensor)[0]
1348
+ blocks_per_row = ne0 // QK
1349
+ rows_per_tensor = n_elements // ne0
1350
+ bytes_per_row = blocks_per_row * Q2K_BYTES
1351
+ if packed_blocks.shape[0] != rows_per_tensor * blocks_per_row:
1352
+ raise AssertionError(
1353
+ f"{tensor.name}: expected {rows_per_tensor * blocks_per_row} Q2_K blocks, "
1354
+ f"got {packed_blocks.shape[0]}"
1355
+ )
1356
+ packed = packed_blocks.reshape(rows_per_tensor, bytes_per_row)
1357
+ if len(np_shape) == 1:
1358
+ packed = packed.reshape(bytes_per_row)
1359
+ else:
1360
+ packed = packed.reshape(np_shape[:-1] + (bytes_per_row,))
1361
+ writer.add_tensor(
1362
+ tensor.name,
1363
+ np.ascontiguousarray(packed),
1364
+ raw_dtype=q2k_type,
1365
+ tensor_endianess=GGUFEndian.LITTLE,
1366
+ )
1367
+ quantized_count += 1
1368
+
1369
+ print(
1370
+ f"{tensor.name} | {np_shape} | {blocks_this:,} | "
1371
+ + (f"{dec_stats['rel_rmse']:.6g} | " if dec_stats is not None else "- | ")
1372
+ + (f"{rtn_stats['rel_rmse']:.6g} | {gain_pct:+.2f}% | " if rtn_stats is not None and dec_stats is not None else "- | - | ")
1373
+ + f"{elapsed:.2f}",
1374
+ flush=True,
1375
+ )
1376
+ if not args.quiet:
1377
+ done_pct = 100.0 * selected_done / max(len(selected), 1)
1378
+ print(
1379
+ f"{tensor_prefix} | COMPLETE | {done_pct:.1f}% selected tensors "
1380
+ f"| total wall {time.perf_counter() - start_total:.1f}s",
1381
+ flush=True,
1382
+ )
1383
+ del original, x, packed_blocks
1384
+
1385
+ if writer is not None:
1386
+ print(f"\nWriting GGUF metadata and tensor data to {tmp_output} ...", flush=True)
1387
+ with progress_stage("write GGUF header", "[GGUF output]", args.progress_interval, args.quiet):
1388
+ writer.write_header_to_file()
1389
+ with progress_stage("write GGUF metadata", "[GGUF output]", args.progress_interval, args.quiet):
1390
+ writer.write_kv_data_to_file()
1391
+ with progress_stage("write all GGUF tensor payloads", "[GGUF output]", args.progress_interval, args.quiet):
1392
+ writer.write_tensors_to_file(progress=False)
1393
+ with progress_stage("close GGUF writer", "[GGUF output]", args.progress_interval, args.quiet):
1394
+ writer.close()
1395
+ writer = None
1396
+ if tempdir_overridden:
1397
+ tempfile.tempdir = prev_tempdir
1398
+ tempdir_overridden = False
1399
+
1400
+ # Validate the emitted GGUF and tensor table before atomically publishing it.
1401
+ check_reader = GGUFReader(str(tmp_output))
1402
+ if len(check_reader.tensors) != len(reader.tensors):
1403
+ raise RuntimeError(
1404
+ f"GGUF validation failed: output has {len(check_reader.tensors)} tensors, "
1405
+ f"input had {len(reader.tensors)}"
1406
+ )
1407
+ out_names = [t.name for t in check_reader.tensors]
1408
+ in_names = [t.name for t in reader.tensors]
1409
+ if out_names != in_names:
1410
+ raise RuntimeError("GGUF validation failed: tensor names/order changed")
1411
+ out_by_name = {t.name: t for t in check_reader.tensors}
1412
+ quantized_names = {row["tensor"] for row in rows}
1413
+ for name, in_tensor in zip(in_names, reader.tensors):
1414
+ out_tensor = out_by_name[name]
1415
+ if tensor_numpy_shape(out_tensor) != tensor_numpy_shape(in_tensor):
1416
+ raise RuntimeError(f"GGUF validation failed: shape changed for {name}")
1417
+ if name in quantized_names:
1418
+ if out_tensor.tensor_type != q2k_type:
1419
+ raise RuntimeError(
1420
+ f"GGUF validation failed: {name} has output type "
1421
+ f"{out_tensor.tensor_type.name}, expected Q2_K"
1422
+ )
1423
+ elif out_tensor.tensor_type != in_tensor.tensor_type:
1424
+ raise RuntimeError(f"GGUF validation failed: pass-through tensor {name} changed type")
1425
+ out_arch = check_reader.fields.get("general.architecture")
1426
+ if out_arch is None or out_arch.contents() != arch:
1427
+ raise RuntimeError("GGUF validation failed: general.architecture was not preserved")
1428
+ out_file_type = check_reader.fields.get("general.file_type")
1429
+ if out_file_type is None or int(out_file_type.contents()) != Q2K_FILE_TYPE:
1430
+ raise RuntimeError("GGUF validation failed: general.file_type is not MOSTLY_Q2_K")
1431
+ os.replace(tmp_output, args.output_gguf)
1432
+ print(f"Wrote llama.cpp-compatible Q2_K GGUF: {args.output_gguf}")
1433
+ except Exception:
1434
+ if writer is not None:
1435
+ try:
1436
+ writer.close()
1437
+ except Exception:
1438
+ pass
1439
+ if tempdir_overridden:
1440
+ tempfile.tempdir = prev_tempdir
1441
+ tempdir_overridden = False
1442
+ if tmp_output is not None and tmp_output.exists():
1443
+ try:
1444
+ tmp_output.unlink()
1445
+ except Exception:
1446
+ pass
1447
+ if isinstance(sys.exc_info()[1], OSError) and sys.exc_info()[1].errno == 28:
1448
+ raise SystemExit(
1449
+ "Out of disk space while writing the output GGUF. gguf-py spools all tensor "
1450
+ f"payloads to a temp file on {spool_dir or tempfile.gettempdir()}; point the output at a "
1451
+ "filesystem with room for roughly 2x the output size, or free space there."
1452
+ ) from None
1453
+ raise
1454
+
1455
+ elapsed_total = time.perf_counter() - start_total
1456
+ csv_path, json_path = write_reports(
1457
+ args, rows, skipped, len(eligible), len(selected), quantized_count, passthrough_count,
1458
+ )
1459
+ print("\nWrote reports:")
1460
+ print(f" CSV: {csv_path}")
1461
+ print(f" JSON: {json_path}")
1462
+ if args.output_gguf:
1463
+ print(f" GGUF: {args.output_gguf}")
1464
+ print(f" Factorized tensors: {quantized_count}")
1465
+ print(f" Copied unchanged: {passthrough_count}")
1466
+ if args.save_recon:
1467
+ print(f" Reconstructions: {args.save_recon}")
1468
+ if imatrix is not None:
1469
+ print(f" Importance-weighted tensors: {imatrix_hits} (uniform fallback: {imatrix_misses})")
1470
+ if skipped:
1471
+ print(f" Skipped/unchanged tensors: {len(skipped)}")
1472
+ print(f" Total elapsed: {elapsed_total:.2f}s")
1473
+ return 0
1474
+
1475
+
1476
+ if __name__ == "__main__":
1477
+ try:
1478
+ raise SystemExit(main())
1479
+ except KeyboardInterrupt:
1480
+ raise SystemExit("Interrupted")
smq_qat.py ADDED
The diff for this file is too large to render. See raw diff
 
train_smq_qat.py ADDED
@@ -0,0 +1,189 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Block-wise quantization-aware training for SMQ (CLI).
3
+
4
+ Trains a GGUF model against SMQ's *real* ``Q2_K`` quantizer, block by block,
5
+ then writes a full-precision (F16) GGUF that ``smq_gguf_convert_live.py``
6
+ quantizes as usual.
7
+
8
+ All eligible weight matrices (``tensor_is_eligible``: >= 2 GGML dims, no 1-D
9
+ elementwise parameters, ne[0] % 256 == 0) become Q2_K and are trained; the
10
+ plan is fixed by the converter's own rule, so QAT always optimizes for
11
+ exactly what SMQ will emit.
12
+
13
+ Usage::
14
+
15
+ python train_smq_qat.py --gguf model.gguf --corpus corpus.txt \\
16
+ --output model-qat.gguf \\
17
+ --imatrix model.imatrix.gguf --steps 120 --blocks 0-8
18
+
19
+ # then quantize with the converter
20
+ python smq_gguf_convert_live.py model-qat.gguf --output-gguf model-qat-Q2K.gguf
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import argparse
26
+ import sys
27
+ from pathlib import Path
28
+
29
+ _HERE = str(Path(__file__).resolve().parent)
30
+ if _HERE and _HERE not in sys.path:
31
+ sys.path.insert(0, _HERE)
32
+
33
+ import smq_gguf_convert_live as smqc # noqa: E402
34
+ from smq_qat import ( # noqa: E402
35
+ block_targets,
36
+ real_blocks,
37
+ tensor_type_key,
38
+ total_blocks,
39
+ train,
40
+ )
41
+
42
+
43
+ def _parse_blocks(text: str | None) -> tuple[int, int] | None:
44
+ if not text:
45
+ return None
46
+ parts = text.replace(",", "-").split("-")
47
+ if len(parts) != 2:
48
+ raise SystemExit(f"--blocks expects LO-HI, got {text!r}")
49
+ try:
50
+ lo, hi = int(parts[0]), int(parts[1])
51
+ except ValueError:
52
+ raise SystemExit(f"invalid --blocks range {text!r}")
53
+ if lo < 0 or hi <= lo:
54
+ raise SystemExit(f"invalid --blocks range {text!r}")
55
+ return lo, hi
56
+
57
+
58
+ def _parse_train_types(text: str | None) -> set[str] | None:
59
+ if not text:
60
+ return None
61
+ return {t.strip() for t in text.split(",") if t.strip()}
62
+
63
+
64
+ def _parse_tensor_types(items: list[str]) -> list[tuple[str, str]]:
65
+ out = []
66
+ for item in items:
67
+ name, _, tt = item.partition("=")
68
+ if not name or not tt:
69
+ raise SystemExit(f"--tensor-type expects TYPE=Copy, got {item!r}")
70
+ out.append((name.strip(), tt.strip()))
71
+ return out
72
+
73
+
74
+ def _build_parser() -> argparse.ArgumentParser:
75
+ p = argparse.ArgumentParser(
76
+ description="Block-wise QAT for SMQ (train weights to survive the real Q2_K quantizer)",
77
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
78
+ )
79
+ p.add_argument("--gguf", type=Path, required=True, help="Input model GGUF")
80
+ p.add_argument("--corpus", type=Path, required=True,
81
+ help="Calibration corpus (.txt/.md/.jsonl, or a directory of them)")
82
+ p.add_argument("--output", type=Path, required=True, help="Output trained GGUF (F16)")
83
+ g = p.add_argument_group("quantization plan")
84
+ g.add_argument("--blocks", default=None, metavar="LO-HI",
85
+ help="only train blocks in [LO, HI)")
86
+ g.add_argument("--tensor-type", action="append", default=[], metavar="TYPE=Copy",
87
+ help="mark a tensor type as Copy (never quantized/trained); repeatable")
88
+ g.add_argument("--imatrix", type=Path, default=None,
89
+ help="importance matrix (.dat or GGUF); weights the fake-quant and the converter")
90
+ g.add_argument("--tokenizer", type=Path, default=None,
91
+ help="HF tokenizer directory override (default: rebuild from the GGUF)")
92
+ g.add_argument("--verbose", action="store_true", help="list every trainable tensor per block")
93
+ t = p.add_argument_group("training")
94
+ t.add_argument("--num-seqs", type=int, default=32, help="calibration sequences")
95
+ t.add_argument("--seq-len", type=int, default=512, help="tokens per sequence")
96
+ t.add_argument("--max-tokens", type=int, default=0, help="cap corpus tokens (0 = all)")
97
+ t.add_argument("--lr", type=float, default=2.0e-5)
98
+ t.add_argument("--weight-decay", type=float, default=0.0)
99
+ t.add_argument("--weight-rmse-lambda", type=float, default=0.0,
100
+ help="weight ||W-Q(W)||^2 penalty coefficient")
101
+ t.add_argument("--cosine-lambda", type=float, default=0.5,
102
+ help="cosine output-similarity loss coefficient")
103
+ t.add_argument("--steps", type=int, default=120, help="optimizer steps per block")
104
+ t.add_argument("--micro-batch", type=int, default=4)
105
+ t.add_argument("--requant-every", type=int, default=0,
106
+ help="re-run the real quantizer every N steps (0 = once per block)")
107
+ t.add_argument("--train-types", default=None,
108
+ help="comma-separated tensor types to train (default: all quantized)")
109
+ t.add_argument("--no-grad-checkpointing", action="store_true")
110
+ t.add_argument("--propagate", choices=("qat", "full"), default="qat",
111
+ help="feed the next block fake-quant (qat) or full-precision activations")
112
+ h = p.add_argument_group("hardware / bookkeeping")
113
+ h.add_argument("--device", default="cuda", help="cuda, cuda:N or cpu")
114
+ h.add_argument("--work-dir", type=Path, default=None,
115
+ help="work directory (default: <output>.qat_work)")
116
+ h.add_argument("--keep-block-dirs", action="store_true",
117
+ help="keep per-block HF partial directories")
118
+ h.add_argument("--seed", type=int, default=1234)
119
+ h.add_argument("--dry-run", action="store_true", help="print the plan and exit")
120
+ return p
121
+
122
+
123
+ def main() -> None:
124
+ args = _build_parser().parse_args()
125
+ blocks = _parse_blocks(args.blocks)
126
+ train_types = _parse_train_types(args.train_types)
127
+ tensor_type = _parse_tensor_types(args.tensor_type)
128
+ device = args.device or "cuda"
129
+
130
+ _gguf_py, reader_cls, _deq, *_rest = smqc.load_gguf()
131
+ reader = reader_cls(str(args.gguf))
132
+ n_blocks = real_blocks(reader)
133
+ n_nextn = total_blocks(reader) - n_blocks
134
+ lo, hi = (0, n_blocks) if blocks is None else (blocks[0], min(blocks[1], n_blocks))
135
+ excluded = {name for name, tt in tensor_type if tt.lower() == "copy"}
136
+
137
+ if args.dry_run:
138
+ print(f"{args.gguf}: {len(reader.tensors)} tensors, {n_blocks} trainable blocks")
139
+ if n_nextn:
140
+ print(f"{n_nextn} NextN/MTP draft block(s) excluded from training")
141
+ print(f"train blocks {lo}..{hi - 1}, device={device}")
142
+ counts: dict[str, int] = {}
143
+ for b in range(lo, hi):
144
+ for gguf_name, _hf in block_targets(reader, b):
145
+ key = tensor_type_key(gguf_name)
146
+ if train_types is not None and key not in train_types:
147
+ continue
148
+ if key in excluded:
149
+ continue
150
+ counts["Q2_K"] = counts.get("Q2_K", 0) + 1
151
+ if args.verbose:
152
+ print(f" blk.{b}: {gguf_name}")
153
+ print("plan: " + ", ".join(f"{k}={v}" for k, v in sorted(counts.items())))
154
+ if train_types:
155
+ print(f"train_types: {sorted(train_types)}")
156
+ if excluded:
157
+ print(f"excluded (Copy): {sorted(excluded)}")
158
+ return
159
+
160
+ train(
161
+ args.gguf,
162
+ args.corpus,
163
+ args.output,
164
+ blocks=blocks,
165
+ num_seqs=args.num_seqs,
166
+ seq_len=args.seq_len,
167
+ max_tokens=args.max_tokens,
168
+ lr=args.lr,
169
+ weight_decay=args.weight_decay,
170
+ weight_rmse_lambda=args.weight_rmse_lambda,
171
+ cosine_lambda=args.cosine_lambda,
172
+ steps=args.steps,
173
+ micro_batch=args.micro_batch,
174
+ requant_every=args.requant_every,
175
+ train_types=train_types,
176
+ tensor_type=tensor_type,
177
+ grad_checkpointing=not args.no_grad_checkpointing,
178
+ propagate=args.propagate,
179
+ work_dir=args.work_dir,
180
+ keep_block_dirs=args.keep_block_dirs,
181
+ seed=args.seed,
182
+ imatrix=args.imatrix,
183
+ tokenizer=args.tokenizer,
184
+ device=device,
185
+ )
186
+
187
+
188
+ if __name__ == "__main__":
189
+ main()