v3_scale: code banc v4/gen/v3, rapports validés, corpus_v3 (sha256 vérifié), README
519e2ef verified | """Checker — commun banc v4 + génération : chargement modèle (auto cm/plain), fenêtres FIXES identiques pour tous les modèles. | |
| seen = train[2_600_000:2_800_000] du corpus propre du modèle (seule fenêtre dépendante du corpus) ; toutes les autres sont des octets fixes.""" | |
| import os, sys, glob, json | |
| sys.path.insert(0, "/workspace/SPIRA/python"); sys.path.insert(0, "/workspace/spira_shared/semhash"); sys.path.insert(0, "/workspace/spira_shared/checker") | |
| import numpy as np | |
| from spira.model import SPIRA | |
| from spira.hashing import fnv1a64 | |
| S = "/workspace/spira_shared"; T = "/workspace/repos/toroidal-fractal-intelligence/data"; B = f"{S}/broad/broad_local_v1" | |
| CORPORA = {"old_local_v1": f"{S}/broad/old_local_v1", "broad_local_v1": B, "corpus_v3": f"{S}/corpus_v3"} | |
| rd = lambda p: open(p, "rb").read() | |
| def fixed_windows(): | |
| from windows_v2 import freshcode_v2 | |
| from build_corpus_local import strip_gutenberg_header | |
| return {"fc2": (freshcode_v2(), 8000), "fresh_en": (rd(f"{B}/oneshot.bin"), 8000), | |
| "fresh_fr": (strip_gutenberg_header(rd(f"{T}/contes_perrault.txt")), 8000), | |
| "neither_code": (b"\n\n".join(rd(p) for p in sorted(glob.glob("/workspace/kahnn/data/code/*"))), 8000), | |
| "test20k": (rd(f"{B}/test.bin"), 20000), "queries": (rd(f"{T}/chat_seed_fr_en.txt"), 8000), | |
| "test_v3": (rd(f"{S}/corpus_v3/test.bin"), 20000)} | |
| def kind_of(ckpt): | |
| return "cm" if "cm_params" in np.load(ckpt, allow_pickle=False).files else "plain" | |
| def load_model(ckpt, train): | |
| k = kind_of(ckpt) | |
| if k == "cm": | |
| import countmin | |
| m = SPIRA.load.__func__(countmin.CMSPIRA, ckpt); z = np.load(ckpt, allow_pickle=False) | |
| m.cm = json.loads(str(z["cm_params"])); m.cm_sem = z["cm_sem"].astype(np.int32); m._row_cache = {} | |
| else: m = SPIRA.load(ckpt) | |
| ps = m.ept.segment(train); m._ids_train = np.fromiter((fnv1a64(p) for p in ps), dtype=np.uint64, count=len(ps)) | |
| saved = dict(m.fusion); cal = m.calibrate_fusion(train[:60_000]) | |
| return m, k, saved, cal | |