File size: 2,033 Bytes
519e2ef | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | """Checker — commun banc v4 + génération : chargement modèle (auto cm/plain), fenêtres FIXES identiques pour tous les modèles.
seen = train[2_600_000:2_800_000] du corpus propre du modèle (seule fenêtre dépendante du corpus) ; toutes les autres sont des octets fixes."""
import os, sys, glob, json
sys.path.insert(0, "/workspace/SPIRA/python"); sys.path.insert(0, "/workspace/spira_shared/semhash"); sys.path.insert(0, "/workspace/spira_shared/checker")
import numpy as np
from spira.model import SPIRA
from spira.hashing import fnv1a64
S = "/workspace/spira_shared"; T = "/workspace/repos/toroidal-fractal-intelligence/data"; B = f"{S}/broad/broad_local_v1"
CORPORA = {"old_local_v1": f"{S}/broad/old_local_v1", "broad_local_v1": B, "corpus_v3": f"{S}/corpus_v3"}
rd = lambda p: open(p, "rb").read()
def fixed_windows():
from windows_v2 import freshcode_v2
from build_corpus_local import strip_gutenberg_header
return {"fc2": (freshcode_v2(), 8000), "fresh_en": (rd(f"{B}/oneshot.bin"), 8000),
"fresh_fr": (strip_gutenberg_header(rd(f"{T}/contes_perrault.txt")), 8000),
"neither_code": (b"\n\n".join(rd(p) for p in sorted(glob.glob("/workspace/kahnn/data/code/*"))), 8000),
"test20k": (rd(f"{B}/test.bin"), 20000), "queries": (rd(f"{T}/chat_seed_fr_en.txt"), 8000),
"test_v3": (rd(f"{S}/corpus_v3/test.bin"), 20000)}
def kind_of(ckpt):
return "cm" if "cm_params" in np.load(ckpt, allow_pickle=False).files else "plain"
def load_model(ckpt, train):
k = kind_of(ckpt)
if k == "cm":
import countmin
m = SPIRA.load.__func__(countmin.CMSPIRA, ckpt); z = np.load(ckpt, allow_pickle=False)
m.cm = json.loads(str(z["cm_params"])); m.cm_sem = z["cm_sem"].astype(np.int32); m._row_cache = {}
else: m = SPIRA.load(ckpt)
ps = m.ept.segment(train); m._ids_train = np.fromiter((fnv1a64(p) for p in ps), dtype=np.uint64, count=len(ps))
saved = dict(m.fusion); cal = m.calibrate_fusion(train[:60_000])
return m, k, saved, cal
|