v3_scale: banc v4.1 complet old vs broad14 — McNemar patch + bootstrap blocs, verdict
7a69401 verified | #!/usr/bin/env python3 | |
| """Checker — rapport banc v4 (registry_v4.tsv : tag corpus étiquette). Lit hits_v4/<tag>__<w>.npz + results_v4.jsonl (dernière ligne non-smoke). | |
| fused_acc_invocab ; moyenne fresh = 6 fenêtres (fc2, fresh_en, fresh_fr, test20k, queries, test_v3) ; écart = seen − fresh ; neither_code rapporté à part | |
| (neither_code exclu : contaminé 100 % broad / 90 % old). McNemar exact par paires sur fused_hit, toutes fenêtres fixes (seen inclus même si corpus différents : octets différents, à lire comme comparaison de mémorisation). | |
| v4.1 (18 h) : McNemar apparié par octet (tgt_start/tgt_len dans les npz) ; paires sans offsets = NON APPARIÉ. | |
| Sorties scale_bench_v4.md / .json. Env HITS, OUT, SUFFIX (smoke).""" | |
| import os, json, itertools, numpy as np | |
| from math import comb | |
| D = os.path.dirname(os.path.abspath(__file__)); HITS = os.environ.get("HITS", f"{D}/hits_v4"); OUT = os.environ.get("OUT", f"{D}/results_v4.jsonl"); SUF = os.environ.get("SUFFIX", "") | |
| WIN = ["seen", "fc2", "fresh_en", "fresh_fr", "neither_code", "test20k", "queries", "test_v3"]; FRESH = ["fc2", "fresh_en", "fresh_fr", "neither_code", "test20k"] # titre = définition Relay 18:22 (inclut neither_code, contaminé) | |
| FRESH6 = ["fc2", "fresh_en", "fresh_fr", "test20k", "test_v3"] # secondaire : 5 fenêtres propres (test_v3 au lieu de neither_code) ; queries et test_v3 aussi rapportés à part | |
| def mcn(b, c): | |
| n = b + c; k = min(b, c) | |
| return 1.0 if n == 0 else min(1.0, 2 * sum(comb(n, i) for i in range(k + 1)) / 2 ** n) | |
| def boot(x, y, blk=512, nb=2000, seed=0): | |
| """Bootstrap par blocs (DÉCISIF, Relay 18:14) : blocs contigus de 512 octets, 2 000 rééchantillonnages, seed 0 ; Δ = acc_B − acc_A par octet.""" | |
| nbk = len(x) // blk; X = x[:nbk * blk].reshape(nbk, blk).mean(1); Y = y[:nbk * blk].reshape(nbk, blk).mean(1); d = Y - X | |
| r = np.random.default_rng(seed); bs = d[r.integers(0, nbk, (nb, nbk))].mean(1); lo, hi = np.percentile(bs, [2.5, 97.5]) | |
| return float(d.mean()), float(lo), float(hi), float(min(1.0, 2 * min((bs <= 0).mean(), (bs >= 0).mean()))) | |
| def unit_mcn(za, zb, lo, hi): | |
| """McNemar exact niveau PATCH (DÉCISIF) : unités = segments entre frontières de patch COMMUNES aux deux tokenizers, dans [lo, hi). | |
| Une unité est réussie pour un modèle si toutes ses cibles (patches) dans l'unité sont des hits. Unités appariées → McNemar exact.""" | |
| def bd(z): | |
| st, ln = z["tgt_start"], z["tgt_len"]; return st, st + ln, z["fused_hit"] | |
| sa, ea, ha_ = bd(za); sb, eb, hb_ = bd(zb) | |
| common = np.intersect1d(np.concatenate([sa, ea[-1:]]), np.concatenate([sb, eb[-1:]])); common = common[(common >= lo) & (common <= hi)] | |
| def uh(s, e, h): | |
| k = np.searchsorted(common, s, side="right") - 1; ok = (s >= common[0]) & (e <= common[-1]) | |
| u = np.ones(len(common) - 1, bool); np.logical_and.at(u, k[ok], h[ok]); return u | |
| x, y = uh(sa, ea, ha_), uh(sb, eb, hb_); b, c = int((x & ~y).sum()), int((~x & y).sum()); return len(x), b, c, mcn(b, c) | |
| def bytehits(z): | |
| """v4.1 : chaque octet de la cible t+1 hérite de fused_hit[t] (OOV = 0) → tableau par octet + plage [début, fin) couverte.""" | |
| st, ln, h = z["tgt_start"], z["tgt_len"], z["fused_hit"]; L = int(z["window_bytes"]); a = np.zeros(L, bool) | |
| for s0, l0, hh in zip(st, ln, h): | |
| if hh: a[s0:s0 + l0] = True | |
| return a, int(st[0]), int(st[-1] + ln[-1]) | |
| R = {} | |
| if os.path.exists(OUT): | |
| for l in open(OUT): | |
| r = json.loads(l); R.setdefault(r["tag"], {})[r["window"]] = r | |
| E = [] | |
| for l in open(f"{D}/registry_v4.tsv"): | |
| if not l.strip() or l.startswith("#"): continue | |
| tag, c, lab = l.rstrip("\n").split("\t"); P = {w: f"{HITS}/{tag}__{w}.npz" for w in WIN} | |
| E.append((lab, tag, c, {w: dict(np.load(p)) for w, p in P.items()} if all(map(os.path.exists, P.values())) else None)) | |
| J = {"models": {}, "pairs": []} | |
| rows = ["| modèle | corpus | " + " | ".join(WIN) + " | fresh moy. titre (fc2,en,fr,neither,test20k) | fresh moy. propre (fc2,en,fr,test20k,test_v3) | écart seen−titre | wall s | pic RAM Mo | scale cal |", "|---|---|" + "---|" * (len(WIN) + 5)] | |
| for lab, tag, c, h in E: | |
| if h is None: rows.append(f"| {lab} | {c} | " + " | ".join("…" for _ in WIN) + " | | | en attente | | | |"); continue | |
| a = {w: 100 * float(h[w]["fused_hit"][h[w]["invocab"]].mean()) for w in WIN}; fm = float(np.mean([a[w] for w in FRESH])); fm6 = float(np.mean([a[w] for w in FRESH6])); rr = R.get(tag, {}) | |
| ws = sum(x["eval_s"] for x in rr.values()) + (next(iter(rr.values()))["load_s"] if rr else 0); pk = max([x["peak_rss_mb"] for x in rr.values()] or [0]) | |
| sc = next(iter(rr.values()))["fusion_cal"]["scale"] if rr else float("nan") | |
| J["models"][lab] = {"tag": tag, "acc_invocab": a, "fresh_mean_titre_1822": fm, "fresh_mean_propre5": fm6, "gap_propre5": a["seen"] - fm6, "gap": a["seen"] - fm, "wall_s": ws, "peak_rss_mb": pk, "scale": sc, "scale_ok": 0.03 <= sc <= 0.11} | |
| rows.append(f"| {lab} | {c} | " + " | ".join(f"{a[w]:.2f}" for w in WIN) + f" | **{fm:.2f}** | {fm6:.2f} | **{a['seen'] - fm:.2f}** | {ws:.0f} | {pk:.0f} | {sc:.3f}{'' if 0.03 <= sc <= 0.11 else ' HORS'} |") | |
| pr = ["| A | B | fenêtre | A seul (octets) | B seul (octets) | p (indicatif) | signe | A hits/n patches | B hits/n patches | Δ B−A % octets, IC95 bootstrap blocs 512 o |McNemar PATCH (unités à frontières communes) A seul / B seul (DÉCISIF) |", "|---|---|---|---|---|---|---|---|---|---|---|"] | |
| def pc(z): return f"{int(z['fused_hit'].sum())}/{len(z['fused_hit'])}" | |
| for (la, ta, ca, ha), (lb, tb, cb, hb) in itertools.combinations([e for e in E if e[3] is not None], 2): | |
| for w in WIN: | |
| if not all("tgt_start" in z for z in (ha[w], hb[w])): | |
| J["pairs"].append({"A": la, "B": lb, "window": w, "status": "non_apparie"}); pr.append(f"| {la} | {lb} | {w} | – | – | – | NON APPARIÉ (positions de tokenizers différents, à relancer en v4.1) |"); continue | |
| if str(ha[w]["window_sha256"]) != str(hb[w]["window_sha256"]): | |
| J["pairs"].append({"A": la, "B": lb, "window": w, "status": "non_comparable_octets_differents", "A_patch": pc(ha[w]), "B_patch": pc(hb[w])}); pr.append(f"| {la} | {lb} | {w} | – | – | – | NON COMPARABLE (octets différents : seen du corpus propre) | {pc(ha[w])} | {pc(hb[w])} |"); continue | |
| x, y = bytehits(ha[w]), bytehits(hb[w]); lo = max(x[1], y[1]); hi = min(x[2], y[2]); x, y = x[0][lo:hi], y[0][lo:hi] | |
| b, cc = int((x & ~y).sum()), int((~x & y).sum()); p = mcn(b, cc); dm, dlo, dhi, pb = boot(x, y); sig = dlo > 0 or dhi < 0; nu, ub, uc, up = unit_mcn(ha[w], hb[w], lo, hi) | |
| J["pairs"].append({"A": la, "B": lb, "window": w, "unit": "byte", "boot_delta_pct": 100 * dm, "boot_ci95_pct": [100 * dlo, 100 * dhi], "boot_p": pb, "boot_sig": bool(sig), "patch_units": nu, "patch_A_only": ub, "patch_B_only": uc, "patch_p": up, "patch_sig": bool(up < 0.05), "byte_range": [int(lo), int(hi)], "A_only": b, "B_only": cc, "p": p, "A_patch": pc(ha[w]), "B_patch": pc(hb[w])}) | |
| pr.append(f"| {la} | {lb} | {w} | {b} | {cc} | {p:.2g} | {'B>A' if cc > b else ('A>B' if b > cc else '=')}{' *' if p < 0.05 else ''} | {pc(ha[w])} | {pc(hb[w])} | {100*dm:+.2f} [{100*dlo:+.2f} ; {100*dhi:+.2f}] p={pb:.3g}{' **' if sig else ''} | {ub} / {uc} sur {nu} unités, p={up:.2g}{' **' if up < 0.05 else ''} |") | |
| md = "# Banc scale v4 — fused_acc_invocab (%) — fenêtres fixes identiques (seen = corpus propre)\n\n" + "\n".join(rows) + "\n\n## McNemar exact par paires — v4.1 apparié par OCTET (même flux d'octets, plage commune ; tokenizers différents ⇒ positions non comparables). p McNemar octet = INDICATIF (octets d'un patch corrélés) ; tests DÉCISIFS = McNemar exact niveau patch (unités entre frontières communes) et bootstrap par blocs de 512 o (IC95 excluant 0 = **). Toutes les paires doivent être en v4.1 avant tout verdict. Moyenne titre = Relay 18:22 (fc2, fresh_en, fresh_fr, neither_code, test20k) ; neither_code contaminé (100 % broad / 90 % old) → variante propre (test_v3 au lieu de neither_code) en secondaire ; queries et test_v3 à part.\n" + "\n".join(pr) + "\n" | |
| open(f"{D}/scale_bench_v4{SUF}.md", "w").write(md); json.dump(J, open(f"{D}/scale_bench_v4{SUF}.json", "w"), indent=1); print(md) | |