v3_scale: banc v4.1 — fenêtre queries appariée par octet (old/broad14), rapports + harnais
8044bdc verified | #!/usr/bin/env python3 | |
| """Checker — banc scale v4. Usage : eval_bench_v4.py CKPT CORPUS_NAME TAG env MAXPOS (cap/fenêtre, smoke) WINDOWS HITS OUT | |
| Fenêtres : seen (corpus propre) + fc2, fresh_en, fresh_fr, neither_code, test20k, queries, test_v3 (octets fixes, identiques pour tous). | |
| Boucle par position = eval_hits() de checker/eval_ref_v3.py (copie ligne à ligne de SPIRA.eval_stream), importée telle quelle. | |
| Enregistre wall-time et pic RSS (ru_maxrss) par fenêtre.""" | |
| import os, sys, json, time, resource, hashlib | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| import numpy as np | |
| from bench_common import * | |
| ckpt, cname, tag = sys.argv[1:4]; D = os.path.dirname(os.path.abspath(__file__)) | |
| HITS = os.environ.get("HITS", f"{D}/hits_v4"); OUT = os.environ.get("OUT", f"{D}/results_v4.jsonl"); MAXPOS = int(os.environ.get("MAXPOS", "0")); SEL = os.environ.get("WINDOWS") | |
| src = open(f"{S}/checker/eval_ref_v3.py").read(); g = {"np": np, "fnv1a64": fnv1a64} | |
| exec(src[src.index("def eval_hits"):src.index("m = SPIRA.load(ckpt)")], g); eval_hits = g["eval_hits"] | |
| t0 = time.perf_counter(); train = rd(f"{CORPORA[cname]}/train.bin") | |
| m, kind, saved, cal = load_model(ckpt, train); tload = time.perf_counter() - t0 | |
| W = {"seen": (train[2_600_000:2_800_000], 8000), **fixed_windows()}; del train | |
| os.makedirs(HITS, exist_ok=True) | |
| for name, (data, mp) in W.items(): | |
| if SEL and name not in SEL.split(","): continue | |
| mp = min(mp, MAXPOS) if MAXPOS else mp; t1 = time.perf_counter(); r, H = eval_hits(m, data, max_positions=mp) | |
| pl = np.fromiter((len(p) for p in m.ept.segment(data)), dtype=np.int64); off = np.concatenate([[0], np.cumsum(pl)]) | |
| n = len(H["fused_hit"]); bb = hashlib.sha256(data).hexdigest() # v4.1 : offsets octets de la cible t+1 → appariement McNemar par octet entre tokenizers différents | |
| np.savez_compressed(f"{HITS}/{tag}__{name}.npz", **H, tgt_start=off[1:n + 1], tgt_len=pl[1:n + 1], window_sha256=np.array(bb), window_bytes=np.array(len(data)), ept_budget_bits=np.array(float(m.ept.budget_bits))) | |
| rec = {"tag": tag, "corpus": cname, "ckpt": ckpt, "kind": kind, "window": name, "n_positions": r["n_positions"], | |
| "fused_invocab_pct": round(100 * r["fused_acc_invocab"], 3), "snn_invocab_pct": round(100 * r["snn_acc_invocab"], 3), | |
| "vocab_coverage": r["vocab_coverage"], "fusion_saved": saved, "fusion_cal": cal, "load_s": round(tload, 1), | |
| "eval_s": round(time.perf_counter() - t1, 1), "peak_rss_mb": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1), | |
| "maxpos": MAXPOS, "window_sha256": bb, "window_bytes": len(data), "ept_budget_bits": float(m.ept.budget_bits), "script": "eval_bench_v4.1"} | |
| open(OUT, "a").write(json.dumps(rec) + "\n"); print(name, rec["fused_invocab_pct"], rec["eval_s"], rec["peak_rss_mb"], flush=True) | |