Blame for v3_scale/code/scale/eval_bench_v4.py

#!/usr/bin/env python3
"""Checker — banc scale v4. Usage : eval_bench_v4.py CKPT CORPUS_NAME TAG   env MAXPOS (cap/fenêtre, smoke) WINDOWS HITS OUT
Fenêtres : seen (corpus propre) + fc2, fresh_en, fresh_fr, neither_code, test20k, queries, test_v3 (octets fixes, identiques pour tous).
Boucle par position = eval_hits() de checker/eval_ref_v3.py (copie ligne à ligne de SPIRA.eval_stream), importée telle quelle.
Enregistre wall-time et pic RSS (ru_maxrss) par fenêtre."""
import os, sys, json, time, resource, hashlib
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import numpy as np
from bench_common import *
ckpt, cname, tag = sys.argv[1:4]; D = os.path.dirname(os.path.abspath(__file__))
HITS = os.environ.get("HITS", f"{D}/hits_v4"); OUT = os.environ.get("OUT", f"{D}/results_v4.jsonl"); MAXPOS = int(os.environ.get("MAXPOS", "0")); SEL = os.environ.get("WINDOWS")
src = open(f"{S}/checker/eval_ref_v3.py").read(); g = {"np": np, "fnv1a64": fnv1a64}
exec(src[src.index("def eval_hits"):src.index("m = SPIRA.load(ckpt)")], g); eval_hits = g["eval_hits"]
t0 = time.perf_counter(); train = rd(f"{CORPORA[cname]}/train.bin")
m, kind, saved, cal = load_model(ckpt, train); tload = time.perf_counter() - t0
W = {"seen": (train[2_600_000:2_800_000], 8000), **fixed_windows()}; del train
os.makedirs(HITS, exist_ok=True)
for name, (data, mp) in W.items():
    if SEL and name not in SEL.split(","): continue
    mp = min(mp, MAXPOS) if MAXPOS else mp; t1 = time.perf_counter(); r, H = eval_hits(m, data, max_positions=mp)
    pl = np.fromiter((len(p) for p in m.ept.segment(data)), dtype=np.int64); off = np.concatenate([[0], np.cumsum(pl)])
    n = len(H["fused_hit"]); bb = hashlib.sha256(data).hexdigest()  # v4.1 : offsets octets de la cible t+1 → appariement McNemar par octet entre tokenizers différents
    np.savez_compressed(f"{HITS}/{tag}__{name}.npz", **H, tgt_start=off[1:n + 1], tgt_len=pl[1:n + 1], window_sha256=np.array(bb), window_bytes=np.array(len(data)), ept_budget_bits=np.array(float(m.ept.budget_bits)))
    rec = {"tag": tag, "corpus": cname, "ckpt": ckpt, "kind": kind, "window": name, "n_positions": r["n_positions"],
           "fused_invocab_pct": round(100 * r["fused_acc_invocab"], 3), "snn_invocab_pct": round(100 * r["snn_acc_invocab"], 3),
           "vocab_coverage": r["vocab_coverage"], "fusion_saved": saved, "fusion_cal": cal, "load_s": round(tload, 1),
           "eval_s": round(time.perf_counter() - t1, 1), "peak_rss_mb": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1),
           "maxpos": MAXPOS, "window_sha256": bb, "window_bytes": len(data), "ept_budget_bits": float(m.ept.budget_bits), "script": "eval_bench_v4.1"}
    open(OUT, "a").write(json.dumps(rec) + "\n"); print(name, rec["fused_invocab_pct"], rec["eval_s"], rec["peak_rss_mb"], flush=True)