| |
| """Build per-tensor feature matrix for Gnom training (deterministic). |
| |
| build_features.py --model M.gguf --imatrix I.gguf --out gnom/features/<m>.npz |
| [--ssim-table models/ssim_table.npz] [--n-layers N] |
| |
| Output .npz: X (n_tensors x 16, RAW, see gnom/README), names (tensor names), |
| stats (timp_mean/std, logN_median for per-model standardization at train). |
| 16 = 6 type one-hot + layer + logN + timp_mean + timp_max + frag + econc |
| + 4 weight stats (std/kurt/max-mean/outlier-fraction). The README's older |
| "15-dim" counts predate the outlier-fraction column (RL-PLAN step 2). |
| Weight stats are computed chunk-wise from the mmap (bounded RAM) and BF16 |
| tensors are byte-viewed through ml_dtypes (scar 2026-10-16: the old |
| np.array(t.data, float32) path read BF16 bytes as numbers — 4 garbage |
| columns on every BF16 model, and the 18 GB Ornith build OOMed on a |
| float64 copy). |
| |
| --ssim-table is DEAD (2026-10-16): 4/4 structural metrics rank together and |
| orthogonal to KLD/PPL — kept only to reproduce archival features byte-for-byte. |
| """ |
| import argparse |
| import json |
| import os |
| import sys |
|
|
| import numpy as np |
|
|
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
| import ml_dtypes |
| from gguf import GGUFReader |
| from llama_inspect import read_model |
| from imatrix_reader import read_imatrix, build_importance_table |
| from tinynet import featurize |
|
|
|
|
| def load_ssim(path): |
| if not path or not os.path.exists(path): |
| return {} |
| try: |
| z = np.load(path, allow_pickle=False) |
| return {n: float(np.ravel(z[n])[0]) for n in z.files} |
| except Exception: |
| return {} |
|
|
|
|
| def _w_view(t): |
| """Reader tensor -> typed array view, or None if not decodable. |
| gguf yields F16/F32/F64 natively; BF16 arrives as raw uint8 bytes |
| (rows x bytes) and must be byte-viewed, never cast.""" |
| arr = t.data |
| if arr.dtype in (np.float16, np.float32, np.float64): |
| return arr |
| if arr.dtype == np.uint8 and arr.ndim >= 1 and arr.shape[-1] % 2 == 0: |
| try: |
| return arr.view(ml_dtypes.bfloat16) |
| except Exception: |
| return None |
| return None |
|
|
|
|
| def w_stats_stream(arr, chunk_rows=1024): |
| """The 4 weight-stat columns of featurize, computed in bounded RAM. |
| Two chunked passes; accumulation in float64 scalars over small slabs. |
| Formulas are exactly featurize's: log10 std, clipped kurt/10, |
| log10(max|w|/mean|w|+1), fraction(|w| > 6*std). |
| (skewness was trialed and REMOVED 2026-10-18: zero effect on v4-tz |
| 0.575, directionally negative on v2-raw, and it switched the best |
| variant under noise — destabilizing, not neutral.)""" |
| two_d = arr.ndim > 1 |
| rows = arr.shape[0] if two_d else 1 |
| step = max(1, int(chunk_rows)) |
| m1 = m2 = m3 = m4 = amax = asum = 0.0 |
| nn = 0 |
| for i in range(0, rows, step): |
| blk = np.asarray(arr[i:i + step] if two_d else arr, dtype=np.float64) |
| blk = blk.ravel() |
| blk2 = blk * blk |
| m1 += float(blk.sum()) |
| m2 += float(blk2.sum()) |
| m3 += float((blk2 * blk).sum()) |
| m4 += float((blk2 * blk2).sum()) |
| a = np.abs(blk) |
| am = float(a.max()) |
| if am > amax: |
| amax = am |
| asum += float(a.sum()) |
| nn += blk.size |
| if nn == 0: |
| return None |
| mean = m1 / nn |
| var = max(m2 / nn - mean * mean, 0.0) |
| std = var ** 0.5 |
| m4c = m4 / nn - 4.0 * mean * (m3 / nn) + 6.0 * mean * mean * (m2 / nn) \ |
| - 3.0 * mean ** 4 |
| kurt = m4c / (var * var + 1e-12) - 3.0 |
| mxm = amax / (asum / nn + 1e-12) |
| thr = 6.0 * (std + 1e-12) |
| cnt = 0 |
| for i in range(0, rows, step): |
| blk = np.asarray(arr[i:i + step] if two_d else arr, dtype=np.float32) |
| cnt += int(np.count_nonzero(np.abs(blk) > thr)) |
| outl = cnt / nn |
| return (float(np.log10(std + 1e-12)), |
| float(np.clip(kurt, -2, 50)) / 10.0, |
| float(np.log10(mxm + 1)), float(outl)) |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--model", required=True) |
| ap.add_argument("--imatrix", required=True) |
| ap.add_argument("--out", required=True) |
| ap.add_argument("--ssim-table", default=None) |
| ap.add_argument("--n-layers", type=int, default=0, |
| help="0 = estimate from tensor names") |
| ap.add_argument("--labels", default=None, |
| help="jsonl damage file: unit tensors the imatrix does " |
| "not cover (MoE/SSM/MLA) still get feature rows") |
| args = ap.parse_args() |
|
|
| model = read_model(args.model) |
| im = read_imatrix(args.imatrix) |
| table = build_importance_table(im, model) |
| ssim = load_ssim(args.ssim_table) |
|
|
| |
| |
| |
| |
| |
| if args.labels: |
| lab_t = set() |
| with open(args.labels) as f: |
| for line in f: |
| d = json.loads(line) |
| if d.get("unit") == "BASELINE": |
| continue |
| lab_t.update(d.get("tensors", [])) |
| missing = sorted(lab_t - set(table.keys())) |
| for tname in missing: |
| tinfo = model.get("tensors", {}).get(tname, {}) |
| table[tname] = { |
| "importance_mean": 0.0, "importance_max": 0.0, |
| "importance_min": 0.0, "importance_sum": 0.0, |
| "n_elements": int(tinfo.get("n_elements", 0)), |
| "type": tname, |
| } |
| print(f" +{len(missing)} label tensors absent from the imatrix " |
| f"→ zero-importance rows (table now {len(table)})", flush=True) |
|
|
| reader = GGUFReader(args.model) |
| wmap = {t.name: t for t in reader.tensors} |
|
|
| names = sorted(table.keys()) |
| if args.n_layers <= 0: |
| layers = [int(n.split(".")[1]) for n in names |
| if n.startswith("blk.") and n.split(".")[1].isdigit()] |
| n_layers = max(layers) + 1 if layers else 28 |
| else: |
| n_layers = args.n_layers |
|
|
| X, kept = [], [] |
| for idx, n in enumerate(names): |
| info = table[n] |
| frag = 1.0 - ssim.get(n, 1.0) |
| tm0, tx0 = info.get("importance_mean", 0.0), info.get("importance_max", 0.0) |
| tmn = info.get("importance_min", 0.0) |
| tsm = info.get("importance_sum", 0.0) |
| econc = tx0 / (tm0 + 1e-12) |
| ws = None |
| if n in wmap: |
| try: |
| v = _w_view(wmap[n]) |
| if v is not None and v.size: |
| ws = w_stats_stream(v) |
| except Exception: |
| ws = None |
| row = featurize(n, info.get("n_elements", 0), tm0, tx0, |
| frag=frag, econc=econc, n_layers=n_layers, wstats=ws, |
| imp_min=tmn, imp_sum=tsm) |
| X.append(row) |
| kept.append(n) |
| if (idx + 1) % 32 == 0 or idx + 1 == len(names): |
| print(f" [{idx + 1}/{len(names)}] {n}", flush=True) |
| X = np.array(X, dtype=np.float64) |
| tm = np.array([table[n].get("importance_mean", 0.0) for n in kept]) |
| logn = np.log10(np.array([table[n].get("n_elements", 1) |
| for n in kept]) + 1) |
| os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) |
| np.savez(args.out, X=X, names=np.array(kept), |
| timp_mean=np.array([tm.mean()]), timp_std=np.array([tm.std()]), |
| logN_med=np.array([np.median(logn)])) |
| print(f"{args.out}: {X.shape[0]} tensors x {X.shape[1]}, " |
| f"n_layers={n_layers}", flush=True) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
