File size: 2,359 Bytes
987349a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | """gary-neuron-chat benchmark: (1) fact-recall with the plastic memory ON vs OFF,
(2) sleep consolidation learns new material while base English stays intact."""
import os, numpy as np
import gpt_numpy as G
from brain import load_brain, sleep, CFG, tok, EOT
import hippo
D=os.path.dirname(os.path.abspath(__file__))
P,Hm,buf,age=load_brain()
# ---- 1. fact recall (held-out episodes, plastic memory ON vs OFF) ----
va=dict(np.load(f"{D}/episodes_val.npz")); on,off=hippo.evaluate(Hm,va)
print(f"[fact recall, {len(va['ids'])} held-out episodes]")
print(f" plastic memory ON : {on*100:.1f}% OFF (frozen cortex alone): {off*100:.1f}%")
# ---- 2. sleep consolidation: learn from use without forgetting ----
T=CFG["BLK"]; val=np.memmap(f"{D}/val.bin",dtype=np.uint16,mode="r"); rng=np.random.default_rng(0)
def base_ppl(P,n=30):
ls=[]
for _ in range(n):
j=rng.integers(0,len(val)-T-1); s=np.asarray(val[j:j+T+1],dtype=np.int64)
ls.append(G.forward(P,s[None,:T],CFG,s[None,1:T+1])[0])
return float(np.exp(np.mean(ls)))
def text_loss(P,texts):
ls=[]
for t in texts:
ids=tok.encode(t).ids
if len(ids)<3: continue
x=np.array([ids[:-1]]); y=np.array([ids[1:]]); ls.append(G.forward(P,x,CFG,y)[0])
return float(np.mean(ls))
# simulated 'usage': new personal facts/topics the base model never saw much
session=["U: my name is Garrett and i hunt zero day exploits\nG: that is fascinating work Garrett",
"U: my favorite cipher is chacha20\nG: chacha20 is fast and secure",
"U: i found a heap overflow in the parser today\nG: nice catch, heap overflows are tricky",
"U: my dog is named Buddy and he guards my servers\nG: a good security dog"]*6
import copy; P0={k:v.copy() for k,v in P.items()}
ppl0=base_ppl(P0); sl0=text_loss(P0,session[:4])
P1,_=sleep(P0, session, steps=50, lr=2e-4, mix=0.5)
ppl1=base_ppl(P1); sl1=text_loss(P1,session[:4])
print(f"\n[sleep consolidation: replay {len(set(session))} convos x6, 50 steps]")
print(f" loss on the session material : {sl0:.3f} -> {sl1:.3f} ({'LEARNED' if sl1<sl0-0.05 else 'flat'})")
print(f" base English perplexity : {ppl0:.1f} -> {ppl1:.1f} ({'intact' if ppl1<ppl0*1.15 else 'DEGRADED'})")
print(" (learns new material while base English stays intact = use-driven learning without catastrophic forgetting)")
|