File size: 1,767 Bytes
032a961 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | #!/usr/bin/env python3
"""Rebuild the corpus hippocampal AM with the FIXED save format.
The pre-fix checkpoints packed/rebuilt AM entries at wrong addresses
(load wrote at k[lane]%size instead of the derivable primary/LOO slots),
making the loaded corpus AM invisible to queries. This script resets the AM,
re-runs the corpus write pass in-memory, and saves with the corrected
packing (dedup at derivable slots).
"""
import sys
import time
sys.path.insert(0, "/home/z/my-project/download/spira")
import numpy as np
from spira import SPIRA
from spira.hippocampus import Hippocampus
BASE = "/home/z/my-project"
CKPT = f"{BASE}/download/spira/artifacts/spira_model.npz"
m = SPIRA.load(CKPT)
print(f"loaded (old AM: {m.am.stats()})")
m.am = Hippocampus(m=m.cfg.am_m, size=m.cfg.am_size, min_lanes=m.cfg.am_min_lanes)
data = open(f"{BASE}/data/train.bin", "rb").read()
patches = m.ept.segment(data)
ids = np.fromiter((__import__("spira.hashing", fromlist=["fnv1a64"]).fnv1a64(p)
for p in patches), dtype=np.uint64, count=len(patches))
t0 = time.perf_counter()
taught = m.teach_ids(ids[:1_200_000], episodic=False)
print(f"corpus AM write: {taught['associations']} associations in "
f"{time.perf_counter()-t0:.0f}s -> {m.am.stats()}")
t0 = time.perf_counter()
m.save(CKPT)
print(f"saved (fixed format) in {time.perf_counter()-t0:.0f}s")
# roundtrip validation
m2 = SPIRA.load(CKPT)
st = m2.am.stats()
print(f"reloaded AM: {st}")
v, w = m2.am.query(m2.am.lane_keys(ids[100:104]))
print(f"query ctx[100:104] -> match={v == ids[104]} (w={w:.2f})")
hits = 0
for i in range(500, 1000):
v, w = m2.am.query(m2.am.lane_keys(ids[i : i + 4]))
hits += int(v == ids[i + 4])
print(f"roundtrip retrieval on 500 corpus contexts: {hits/5:.1f}%")
|