Alpha / alpha_train.py
SofiTesfay2010's picture
Alpha: System-One-style typed decision model
af6e981 verified
Raw History Blame Contribute Delete
41.4 kB
"""
Alpha training / benchmark / publish pipeline (Kaggle, 2x T4).
Framework rules from the spec are used as *design priors and instruments*, not as proven facts:
* Accessibility A(theta,F): squared-ReLU + zero-init Slot Reasoner, pretrained encoder, schema-compositional heads
(the general function is the easiest one to reach), many paraphrased schemas (high information rate).
* Regularisation shapes the manifold: very different weight decay for pretrained backbone vs new modules, adapted online
by the Discovery Controller from rho = d(val)/d(train) (proxy for P(B_G)/P(B_M)).
* Functional memory M_F: function-space signatures of past EMA checkpoints on diagnostic set E_F, used as a KL anchor
(self-distillation to earlier functions); L_FRR measured mid-training scales the anchor weight.
* FRR: perturb the discovered solution (basin test), retrain, time re-discovery -> FRR = T_first / T_re.
* Meta-grokking: held-out task families never trained on; adaptation speed measured at checkpoints (MGR).
* DMR: T_discover / T_memorize from train/val trajectories ( ~1 => no delayed generalisation, i.e. 'groks from the start').
"""
import os, re, json, math, time, random, shutil
import numpy as np
import torch
import torch.nn.functional as F
from alpha_s1 import Alpha, AlphaConfig, build_batch, to_device
dev = "cuda:0" if torch.cuda.is_available() else "cpu"
ac = lambda: torch.autocast("cuda", dtype=torch.float16, enabled=torch.cuda.is_available())
class C:
backbone = "distilroberta-base" # try "roberta-base" for more accuracy (slower)
repo_id = "SofiTesfay2010/Alpha"
seed = 0
steps, bs = 6000, 32
lr_bb, lr_new = 5e-5, 5e-4
wd_bb, wd_new, wd_min, wd_max = 0.01, 0.5, 0.05, 2.0
warmup, clip = 300, 1.0
ls, brier_w, ord_w = 0.02, 0.3, 1.0
ema = 0.999
eval_every, mem_every, anchor_every, lam_mem = 250, 1000, 4, 0.3
probe_every, probe_steps, probe_bs, probe_test_n = 1000, 40, 16, 200
probe_at = (0, 5, 10, 20, 40)
fast_n = 48
n_cal, n_test = 150, 500
sigmas = [0.5, 1, 2, 4, 8]
noise_unit, frr_max_steps, frr_every = 0.02, 300, 25
llm_baseline = "Qwen/Qwen2.5-0.5B-Instruct"
# ----------------------------------------------------------------------------------------------- data
def S(paths, fields, fn, qs, names=None, alts=None, tr="train", va="validation", ordinal=False, held=False, ntr=30000, camel=False):
return dict(paths=paths, fields=fields, fn=fn, qs=qs, names=names, alts=alts, tr=tr, va=va, ordinal=ordinal, held=held, ntr=ntr, camel=camel)
SPECS = dict(
sst2=S([("nyu-mll/glue", "sst2")], ["sentence"], [["review"], ["comment"], ["snippet"]],
["What is the sentiment of the {f}?", "Is the {f} positive or negative?", "Judge the attitude expressed in the {f}."],
names=["negative", "positive"], alts=[["negative", "positive"], ["bad", "good"], ["unfavorable", "favorable"]]),
imdb=S([("stanfordnlp/imdb",)], ["text"], [["movie_review"], ["critique"]],
["Is the {f} favorable or unfavorable?", "What verdict does the {f} give?", "Does the author of the {f} like the film?"],
names=["negative", "positive"], alts=[["negative", "positive"], ["thumbs down", "thumbs up"], ["dislikes it", "likes it"]], va="test", ntr=20000),
ag_news=S([("fancyzhx/ag_news",)], ["text"], [["headline"], ["article"], ["news_item"]],
["Which section would the {f} appear in?", "What topic is the {f} about?", "Categorize the {f}."],
names=["world news", "sports", "business", "science and technology"],
alts=[["world news", "sports", "business", "science and technology"], ["world", "sports", "finance", "tech"], ["international affairs", "athletics", "economy", "technology"]], va="test"),
emotion=S([("dair-ai/emotion",)], ["text"], [["message"], ["post"], ["tweet"]],
["Which emotion does the {f} convey?", "What is the author of the {f} feeling?", "Name the dominant emotion in the {f}."],
names=["sadness", "joy", "love", "anger", "fear", "surprise"],
alts=[["sadness", "joy", "love", "anger", "fear", "surprise"], ["sad", "happy", "affection", "angry", "afraid", "astonished"], ["grief", "delight", "adoration", "rage", "dread", "shock"]]),
banking77=S([("PolyAI/banking77",), ("mteb/banking77",)], ["text"], [["customer_message"], ["support_query"]],
["What is the customer asking about in the {f}?", "Which intent does the {f} express?", "Classify the banking request in the {f}."], va="test"),
yelp=S([("Yelp/yelp_review_full",)], ["text"], [["review"], ["feedback"]],
["How many stars does the {f} deserve?", "Rate the {f}.", "What score would the author of the {f} give?"],
names=["1 star", "2 stars", "3 stars", "4 stars", "5 stars"],
alts=[["1 star", "2 stars", "3 stars", "4 stars", "5 stars"], ["terrible", "poor", "average", "good", "excellent"], ["very negative", "negative", "neutral", "positive", "very positive"]],
va="test", ordinal=True),
snli=S([("stanfordnlp/snli",)], ["premise", "hypothesis"], [["premise", "hypothesis"], ["context", "claim"]],
["What is the logical relationship between {f}?", "Does the second follow from the first, contradict it, or neither? ({f})", "Relation between {f}:"],
names=["entailment", "neutral", "contradiction"],
alts=[["entailment", "neutral", "contradiction"], ["supported", "undetermined", "contradicted"], ["follows", "unrelated", "conflicts"]], ntr=60000),
sms=S([("ucirvine/sms_spam",)], ["sms"], [["message"], ["text_message"]],
["Is the {f} spam?", "Classify the {f}.", "Is the {f} an unwanted advertisement or a normal message?"],
names=["legitimate", "spam"], alts=[["legitimate", "spam"], ["ham", "spam"], ["normal", "unwanted"]], va=None),
news20=S([("SetFit/20_newsgroups",)], ["text"], [["post"], ["usenet_message"]],
["Which newsgroup does the {f} belong to?", "What topic does the {f} discuss?", "Categorize the {f} by topic."], va="test"),
mrpc=S([("nyu-mll/glue", "mrpc")], ["sentence1", "sentence2"], [["sentence_a", "sentence_b"], ["statement_1", "statement_2"]],
["Do {f} mean the same thing?", "Are {f} paraphrases of each other?", "Compare the meaning of {f}."],
names=["different meaning", "same meaning"], alts=[["different meaning", "same meaning"], ["not paraphrase", "paraphrase"], ["unrelated", "equivalent"]]),
# ---- held-out families: NEVER trained on in the main run; used for zero-shot + meta-grokking probes ----
dbpedia=S([("fancyzhx/dbpedia_14",)], ["content"], [["entry"], ["wiki_summary"]],
["What kind of entity does the {f} describe?", "Classify the subject of the {f}.", "Which category fits the {f}?"], va="test", held=True, ntr=20000, camel=True),
offensive=S([("cardiffnlp/tweet_eval", "offensive")], ["text"], [["tweet"], ["post"]],
["Is the {f} offensive?", "Does the {f} contain abuse or insults?", "Moderate the {f}."],
names=["not offensive", "offensive"], alts=[["not offensive", "offensive"], ["civil", "abusive"], ["acceptable", "insulting"]], held=True),
cola=S([("nyu-mll/glue", "cola")], ["sentence"], [["sentence"], ["utterance"]],
["Is the {f} grammatical?", "Is the {f} well-formed English?", "Judge the grammar of the {f}."],
names=["ungrammatical", "grammatical"], alts=[["ungrammatical", "grammatical"], ["incorrect", "correct"], ["unacceptable", "acceptable"]], held=True),
)
DATA, ST = {}, {}
def _clean(n, camel):
n = n.replace("_", " ").replace(".", " ")
if camel: n = re.sub(r"(?<!^)(?=[A-Z])", " ", n)
return n.lower().strip()
def load_all():
from datasets import load_dataset
for name, sp in SPECS.items():
for p in sp["paths"]:
try:
if sp["va"]:
dtr = load_dataset(*p, split=sp["tr"]).shuffle(seed=0); dva = load_dataset(*p, split=sp["va"]).shuffle(seed=0)
else:
full = load_dataset(*p, split="train").shuffle(seed=0); cut = int(0.9 * len(full))
dtr, dva = full.select(range(cut)), full.select(range(cut, len(full)))
dtr = dtr.select(range(min(len(dtr), sp["ntr"]))); dva = dva.select(range(min(len(dva), 1500)))
names = sp["names"]
if names is None:
feat = dtr.features["label"]
if hasattr(feat, "names"): names = list(feat.names)
else:
m = {int(l): t for l, t in zip(dtr["label"], dtr["label_text"])}; names = [m[i] for i in range(max(m) + 1)]
names = [_clean(n, sp["camel"]) for n in names]
alts = sp["alts"] or [names]
def rows(d):
cols = [d[f] for f in sp["fields"]]; lab = d["label"]
return [(tuple(str(c[i])[:1200] for c in cols), int(lab[i])) for i in range(len(lab)) if int(lab[i]) >= 0]
R = dict(train=rows(dtr), val=rows(dva))
assert len(R["val"]) >= 60 and len(R["train"]) >= 200
DATA[name] = dict(spec=sp, names=names, alts=alts, rows=R)
print(f"[data] {name:10s} classes={len(names):3d} train={len(R['train']):6d} val={len(R['val']):5d} {'(held-out)' if sp['held'] else ''}")
break
except Exception as e:
print(f"[skip] {name} via {p}: {type(e).__name__}: {str(e)[:100]}")
ST["train_fams"] = sorted(f for f in DATA if not DATA[f]["spec"]["held"])
ST["held_fams"] = sorted(f for f in DATA if DATA[f]["spec"]["held"])
assert len(ST["train_fams"]) >= 4, "too few training families loaded (is internet enabled?)"
def make_example(rng, fams, split="train", wording="train", idx=None, lo=0):
fields, qs, used = [], [], set()
for fam in fams:
D = DATA[fam]; sp = D["spec"]; rows = D["rows"][split]
if idx is not None: texts, label = rows[idx % len(rows)]
else: texts, label = rows[rng.randrange(lo if len(rows) > lo else 0, len(rows))]
nm = []
for n0 in rng.choice(sp["fn"]):
n1, c = n0, 2
while n1 in used: n1 = f"{n0}_{c}"; c += 1
used.add(n1); nm.append(n1)
fields += list(zip(nm, texts))
if wording == "unseen": q, alt = sp["qs"][2], D["alts"][-1]
elif wording == "any": q, alt = rng.choice(sp["qs"]), rng.choice(D["alts"])
else: q, alt = rng.choice(sp["qs"][:2]), rng.choice(D["alts"][:-1] or D["alts"])
qs.append(dict(q=q.format(f=" and ".join(nm)), options=list(alt), gold=label, ordinal=sp["ordinal"], fam=fam))
rng.shuffle(fields); rng.shuffle(qs)
return dict(fields=fields, questions=qs)
def sample_exs(rng, n):
fams, w = ST["train_fams"], ST["w"]; out = []
for _ in range(n):
k = min(rng.choices([1, 2, 3, 4], [.3, .3, .25, .15])[0], len(fams)); ch = []
while len(ch) < k:
f = rng.choices(fams, w)[0]
if f not in ch: ch.append(f)
out.append(make_example(rng, ch))
return out
def prep(exs, bs=48):
m = ST["model"]
return [to_device(build_batch(exs[i:i + bs], m.tk, m.tokz, m.cfg), dev) for i in range(0, len(exs), bs)]
def val_examples(fam, lo, hi, wording):
n = len(DATA[fam]["rows"]["val"])
return [make_example(random.Random(1000 + i), [fam], "val", wording, idx=i) for i in range(lo, min(hi, n))]
# ----------------------------------------------------------------------------------------------- metrics
def ece_fn(conf, corr, nb=15):
conf, corr = np.asarray(conf, float), np.asarray(corr, float); e = 0.0
edges = np.linspace(0, 1, nb + 1)
for lo, hi in zip(edges[:-1], edges[1:]):
m = (conf > lo) & (conf <= hi)
if m.any(): e += m.mean() * abs(corr[m].mean() - conf[m].mean())
return float(e)
@torch.no_grad()
def predict_res(model, batches):
model.eval(); out = []
for b in batches:
with ac(): lg = model(b)
lg = lg.float().cpu()
for r, idx in enumerate(b["opt_lists"]):
out.append((b["fams"][r], lg[r, idx].numpy(), b["gold_pos"][r]))
return out
def _calc(items):
acc = nll = br = 0.0; conf, corr = [], []
for lg, g in items:
z = lg - lg.max(); lp = z - np.log(np.exp(z).sum()); p = np.exp(lp)
oh = np.zeros_like(p); oh[g] = 1
acc += float(p.argmax() == g); nll += -lp[g]; br += ((p - oh) ** 2).sum()
conf.append(float(p.max())); corr.append(float(p.argmax() == g))
n = len(items)
return dict(acc=acc / n, nll=nll / n, brier=br / n, ece=ece_fn(conf, corr), n=n, conf=conf, corr=corr)
def stat(res, T=1.0):
per = {}
for fam, lg, g in res: per.setdefault(fam, []).append((lg / T, g))
P = {f: _calc(v) for f, v in per.items()}
allitems = [x for v in per.values() for x in v]; micro = _calc(allitems)
mac = lambda k: float(np.mean([P[f][k] for f in P]))
return dict(per=P, score=mac("acc"), nll=mac("nll"), brier=mac("brier"), ece=micro["ece"], micro=micro,
per_acc={f: P[f]["acc"] for f in P})
def fast_eval(model, which):
return stat(predict_res(model, ST["fast_" + which]), float(model.post_T))
# ----------------------------------------------------------------------------------------------- training parts
def loss_fn(model, b):
lg = model(b).float(); lp = F.log_softmax(lg, -1); al = b["allowed"]
ar = torch.arange(lg.size(0), device=lg.device)
nll = -lp[ar, b["gold"]]
uni = -(lp.masked_fill(~al, 0.0)).sum(1) / al.sum(1)
p = lp.exp(); oh = F.one_hot(b["gold"], lg.size(1)).float()
brier = ((p - oh) ** 2).sum(1)
rg = b["rank"][ar, b["gold"]]
ordl = (p * (b["rank"] - rg[:, None]) ** 2 * al).sum(1)
L = (1 - C.ls) * nll + C.ls * uni + C.brier_w * brier + C.ord_w * b["ordinal"].float() * ordl
return L.mean(), float(nll.mean())
def make_scaler():
try: return torch.amp.GradScaler("cuda", enabled=torch.cuda.is_available())
except Exception: return torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available())
def make_opt(model, lr_bb, lr_new, wd_new):
def split(ps): return [p for p in ps if p.ndim >= 2], [p for p in ps if p.ndim < 2]
bd, bn = split(list(model.backbone.parameters())); nd, nn_ = split(model.new_params())
gs = [dict(params=bd, base_lr=lr_bb, lr=lr_bb, weight_decay=C.wd_bb, tag="bb"), dict(params=bn, base_lr=lr_bb, lr=lr_bb, weight_decay=0.0, tag="bb0"),
dict(params=nd, base_lr=lr_new, lr=lr_new, weight_decay=wd_new, tag="new"), dict(params=nn_, base_lr=lr_new, lr=lr_new, weight_decay=0.0, tag="new0")]
return torch.optim.AdamW([g for g in gs if g["params"]], betas=(0.9, 0.98), eps=1e-6)
def sgd_step(model, opt, scaler, b, mem=None, lam=0.0, anchor=False):
model.train()
with ac():
loss, nll = loss_fn(model, b)
if anchor and mem is not None and mem.entries: loss = loss + lam * mem.anchor(model)
opt.zero_grad(set_to_none=True); scaler.scale(loss).backward(); scaler.unscale_(opt)
gn = torch.nn.utils.clip_grad_norm_(model.parameters(), C.clip); scaler.step(opt); scaler.update()
return nll, float(gn)
def clone_state(m): return {k: v.detach().cpu().clone() for k, v in m.state_dict().items()}
class EMA:
def __init__(s, model, decay):
s.d, s.bk = decay, None
s.sh = {k: v.detach().clone().float() for k, v in model.state_dict().items() if v.dtype.is_floating_point}
@torch.no_grad()
def update(s, model, step):
d = min(s.d, (1 + step) / (10 + step))
for k, v in model.state_dict().items():
if k in s.sh: s.sh[k].mul_(d).add_(v.detach().float(), alpha=1 - d)
@torch.no_grad()
def apply(s, model):
sd = model.state_dict(); s.bk = {k: sd[k].detach().clone() for k in s.sh}
model.load_state_dict({k: s.sh[k].to(sd[k].dtype) for k in s.sh}, strict=False)
@torch.no_grad()
def restore(s, model): model.load_state_dict(s.bk, strict=False)
class FMem:
"""Functional memory M_F: entries = {step, signature Sigma_F (dense logits on diagnostic set E_F), val score}."""
def __init__(s, diag_batch): s.b, s.entries = diag_batch, []
@torch.no_grad()
def add(s, model, step, score):
model.eval()
with ac(): lg = model(s.b).float()
s.entries.append(dict(step=step, sig=lg.cpu().half(), score=score))
def anchor(s, model):
e = random.choice(s.entries); old = e["sig"].to(s.b["allowed"].device).float()
lp = F.log_softmax(model(s.b).float(), -1); p = F.softmax(old, -1); lpo = F.log_softmax(old, -1)
return ((p * (lpo - lp)).masked_fill(~s.b["allowed"], 0.0)).sum(1).mean()
class Controller:
"""Discovery controller: rho = d(val nll)/d(train nll) ~ share of fitting progress that generalises (proxy of P(B_G)/P(B_M))."""
def __init__(s): s.wd, s.rho, s.prev, s.LFRR = C.wd_new, None, None, 0.0
def update(s, rec):
if s.prev is not None:
dtr, dv = s.prev["tr_nll"] - rec["tr_nll"], s.prev["nll"] - rec["nll"]
if dtr > 1e-3:
r = float(np.clip(dv / dtr, -1, 2)); s.rho = r if s.rho is None else 0.5 * s.rho + 0.5 * r
if s.rho is not None:
if s.rho < 0.3: s.wd = min(C.wd_max, s.wd * 1.25) # memorising -> shape manifold harder
elif s.rho > 0.8: s.wd = max(C.wd_min, s.wd * 0.9) # generalising -> relax
s.prev = rec
def perturb(model, sigma):
with torch.no_grad():
for p in model.parameters():
if p.ndim >= 2: p.add_(torch.randn_like(p) * p.std() * sigma * C.noise_unit)
def frr_test(model, hist, final_score, sigmas):
"""FRR = T_first / T_re. T_first: first logged step the main run reached tau. T_re: steps to re-reach tau after perturbing."""
snap = clone_state(model); tau = final_score - 0.01
T_first = next((h["step"] for h in hist if h["score"] >= tau), hist[-1]["step"]); res = {}
for sg in sigmas:
model.load_state_dict(snap); torch.manual_seed(int(sg * 100)); perturb(model, sg)
v0 = fast_eval(model, "val")["score"]; trivial = v0 >= tau; T_re, hit = 1, trivial
if not trivial:
opt, scaler, rng = make_opt(model, C.lr_bb, C.lr_new, C.wd_new), make_scaler(), random.Random(999 + int(sg * 10)); T_re = C.frr_max_steps
for s in range(1, C.frr_max_steps + 1):
for g in opt.param_groups: g["lr"] = g["base_lr"] * min(1.0, s / 5)
sgd_step(model, opt, scaler, to_device(build_batch(sample_exs(rng, C.bs), model.tk, model.tokz, model.cfg), dev))
if s % C.frr_every == 0 and fast_eval(model, "val")["score"] >= tau: T_re, hit = s, True; break
res[sg] = dict(v0=v0, tau=tau, T_first=T_first, T_re=T_re, FRR=T_first / max(T_re, 1), L_FRR=max(T_re, 1) / max(T_first, 1),
censored=(not hit), trivial=trivial)
print(f" [FRR] sigma={sg}: perturbed score {v0:.3f} (tau {tau:.3f}) T_first={T_first} T_re={T_re}{'+' if not hit else ''} FRR={res[sg]['FRR']:.1f}")
model.load_state_dict(snap); return res
def adapt_curve(model, fam, eval_batches, steps, at):
"""Few-shot adaptation of a snapshot to a held-out family; returns {step: (acc, nll)}. Model is restored afterwards."""
snap = clone_state(model); opt, scaler, rng = make_opt(model, C.lr_bb, C.lr_new, 0.0), make_scaler(), random.Random(777); curve = {}
for s in range(steps + 1):
if s in at:
r = stat(predict_res(model, eval_batches), float(model.post_T)); curve[s] = (r["score"], r["nll"])
if s == steps: break
for g in opt.param_groups: g["lr"] = g["base_lr"] * min(1.0, (s + 1) / 3)
b = to_device(build_batch([make_example(rng, [fam], "train", "any") for _ in range(C.probe_bs)], model.tk, model.tokz, model.cfg), dev)
sgd_step(model, opt, scaler, b)
model.load_state_dict(snap); return curve
def meta_probe(model, step):
out = [dict(step=step, fam=f, curve=adapt_curve(model, f, ST["held_test"][f], C.probe_steps, C.probe_at)) for f in ST["held_fams"]]
msg = " | ".join(f"{o['fam']}: zs {o['curve'][0][0]:.2f} -> @10 {o['curve'][10][0]:.2f} -> @{C.probe_steps} {o['curve'][C.probe_steps][0]:.2f}" for o in out)
print(f" [meta-probe @ {step}] {msg}"); return out
# ----------------------------------------------------------------------------------------------- setup + train
def setup():
random.seed(C.seed); np.random.seed(C.seed); torch.manual_seed(C.seed)
load_all()
m = Alpha(AlphaConfig(backbone=C.backbone)).to(dev).use_data_parallel()
ST["model"] = m; ST["w"] = [math.sqrt(len(DATA[f]["rows"]["train"])) for f in ST["train_fams"]]
tf, hf = ST["train_fams"], ST["held_fams"]
fast = lambda split, lo, n: [make_example(random.Random(i * 31 + j), [f], split, "train", idx=i) for j, f in enumerate(tf) for i in range(lo, lo + n)]
ST["fast_val"], ST["fast_train"] = prep(fast("val", 0, C.fast_n)), prep(fast("train", 0, C.fast_n))
ST["cal"] = prep([e for f in tf for e in val_examples(f, 0, C.n_cal, "train")])
ST["held_test"] = {f: prep(val_examples(f, 0, C.probe_test_n, "unseen")) for f in hf}
rng = random.Random(4242); ST["diag"] = prep([make_example(rng, rng.sample(tf, min(3, len(tf)))) for _ in range(16)], bs=16)[0]
n_all = sum(p.numel() for p in m.parameters()); print(f"params: {n_all/1e6:.1f}M | GPUs: {torch.cuda.device_count()} | train fams {tf} | held-out {hf}")
def train():
m, hist = ST["model"], []; rng = random.Random(C.seed + 1)
ctl, mem, ema = Controller(), FMem(ST["diag"]), EMA(m, C.ema)
opt, scaler = make_opt(m, C.lr_bb, C.lr_new, ctl.wd), make_scaler()
ST.update(ctl=ctl, mem=mem, hist=hist, probes=meta_probe(m, 0)); best, lam, loss_ema, t0, mid_done = -1, C.lam_mem, None, time.time(), False
for step in range(1, C.steps + 1):
prog = max(0.0, (step - C.warmup) / max(1, C.steps - C.warmup))
mult = min(1.0, step / C.warmup) * (0.1 + 0.9 * 0.5 * (1 + math.cos(math.pi * prog)))
for g in opt.param_groups:
g["lr"] = g["base_lr"] * mult
if g["tag"] == "new": g["weight_decay"] = ctl.wd
b = to_device(build_batch(sample_exs(rng, C.bs), m.tk, m.tokz, m.cfg), dev)
nll, gn = sgd_step(m, opt, scaler, b, mem, lam, anchor=(step % C.anchor_every == 0))
ema.update(m, step); loss_ema = nll if loss_ema is None else 0.98 * loss_ema + 0.02 * nll
if step % C.eval_every == 0 or step == C.steps:
ema.apply(m); v, t = fast_eval(m, "val"), fast_eval(m, "train")
rec = dict(step=step, score=v["score"], nll=v["nll"], ece=v["ece"], tr_score=t["score"], tr_nll=t["nll"], fam=v["per_acc"],
wd=ctl.wd, lam=lam, gn=gn, loss=loss_ema, rho=ctl.rho, lr=opt.param_groups[0]["lr"])
hist.append(rec)
if v["score"] > best: best = v["score"]; ST["best_state"] = clone_state(m); ST["best_step"] = step
ctl.update(rec)
if step % C.mem_every == 0: mem.add(m, step, v["score"])
if step % C.probe_every == 0: ST["probes"] += meta_probe(m, step)
if not mid_done and step >= C.steps // 2:
mid_done = True; r = frr_test(m, hist, v["score"], [1])[1]; ctl.LFRR = r["L_FRR"]; lam = C.lam_mem * (1 + min(ctl.LFRR, 3.0))
print(f" [L_FRR={ctl.LFRR:.3f}] -> anchor weight {lam:.3f}")
ema.restore(m)
print(f"step {step:5d} | val acc {v['score']:.3f} nll {v['nll']:.3f} ece {v['ece']:.3f} | train acc {t['score']:.3f} nll {t['nll']:.3f} | "
f"rho {ctl.rho if ctl.rho is None else round(ctl.rho, 2)} wd {ctl.wd:.3f} lam {lam:.2f} gn {gn:.2f} | {time.time()-t0:.0f}s")
m.load_state_dict(ST["best_state"]); print(f"loaded best EMA checkpoint (step {ST['best_step']})")
# ----------------------------------------------------------------------------------------------- finalize + benchmark
def finalize():
m, hist = ST["model"], ST["hist"]; m.eval()
res = predict_res(m, ST["cal"]); grid = np.arange(0.5, 3.01, 0.02)
nlls = [np.mean([-(lg / T - np.logaddexp.reduce(lg / T))[g] for _, lg, g in res]) for T in grid]
T = float(grid[int(np.argmin(nlls))]); m.post_T.fill_(T); print(f"post-hoc temperature T={T:.2f}")
best = max(h["score"] for h in hist); print("FRR sweep (basin test):"); ST["frr"] = frr_test(m, hist, best, C.sigmas)
tau = min(h["nll"] for h in hist) + 0.05
Tv = next((h["step"] for h in hist if h["nll"] <= tau), hist[-1]["step"]); Tt = next((h["step"] for h in hist if h["tr_nll"] <= tau), hist[-1]["step"])
ST["DMR"] = dict(T_discover=Tv, T_memorize=Tt, DMR=Tv / max(Tt, 1)); print("DMR:", ST["DMR"])
# meta-grokking: T_D(n) = adaptation steps to reach 95% of the best adapted accuracy seen over all probes (per family)
P, fams, mg = ST["probes"], ST["held_fams"], {}
steps_ = sorted({p["step"] for p in P})
for f in fams:
tau_f = 0.95 * max(p["curve"][C.probe_steps][0] for p in P if p["fam"] == f)
for s in steps_:
c = next(p for p in P if p["fam"] == f and p["step"] == s)["curve"]
td = next((k for k in sorted(c) if c[k][0] >= tau_f), C.probe_steps * 2)
mg.setdefault(s, {})[f] = dict(T_D=max(td, 1), zero_shot=c[0][0], acc10=c[10][0], final=c[C.probe_steps][0])
base = steps_[0]
for s in steps_:
g = [mg[base][f]["T_D"] / mg[s][f]["T_D"] for f in fams]; mg[s]["MGR"] = float(np.exp(np.mean(np.log(g))))
ST["meta"] = mg; print("meta-grokking MGR by checkpoint:", {s: round(mg[s]["MGR"], 2) for s in steps_})
@torch.no_grad()
def workflow_eval(n=300, k=4):
m = ST["model"]; rng = random.Random(2024); exs = []
for _ in range(n): exs.append(make_example(rng, rng.sample(ST["train_fams"], min(k, len(ST["train_fams"]))), "val", "train", lo=C.n_cal))
m.eval(); qacc, allok, jc, jok = [], [], [], []
for i in range(0, n, 32):
b = to_device(build_batch(exs[i:i + 32], m.tk, m.tokz, m.cfg), dev)
with ac(): lg = m(b).float() / m.post_T
per = {}
for r, idx in enumerate(b["opt_lists"]):
p = F.softmax(lg[r, idx], -1); ok = int(p.argmax()) == b["gold_pos"][r]
qacc.append(ok); per.setdefault(b["q_ex"][r], []).append((ok, float(p.max())))
for v in per.values():
allok.append(all(o for o, _ in v)); jc.append(float(np.prod([c for _, c in v]))); jok.append(float(all(o for o, _ in v)))
return dict(q_acc=float(np.mean(qacc)), all_correct=float(np.mean(allok)), joint_ece=ece_fn(jc, jok), k=k, n=n)
def speed_bench():
m = ST["model"]; m.eval()
state = dict(customer_ticket="My card was charged twice for the same order and support has not replied in three days. I want my money back.",
account="plan: pro; tenure_months: 14; open_tickets: 2")
qs = [dict(id="refund", question="Does the customer ask for a refund?", options=["no", "yes"]),
dict(id="urgency", question="How urgent is the ticket?", options=["low", "medium", "high"], ordinal=True),
dict(id="team", question="Which team should handle the ticket?", options=["billing", "technical", "sales", "account", "abuse"])]
sync = (lambda: torch.cuda.synchronize()) if torch.cuda.is_available() else (lambda: None)
t = time.perf_counter(); m.predict(state, qs); sync(); cold = (time.perf_counter() - t) * 1e3
for _ in range(10): m.predict(state, qs)
ts = []
for _ in range(200):
sync(); t = time.perf_counter(); m.predict(state, qs); sync(); ts.append((time.perf_counter() - t) * 1e3)
rng = random.Random(1); exs = [make_example(rng, rng.sample(ST["train_fams"], 3)) for _ in range(64)]
with torch.no_grad():
for _ in range(3): m(to_device(build_batch(exs, m.tk, m.tokz, m.cfg), dev))
sync(); t = time.perf_counter()
for _ in range(10):
b = to_device(build_batch(exs, m.tk, m.tokz, m.cfg), dev)
with torch.no_grad(), ac(): m(b)
sync(); e2e = 10 * 64 / (time.perf_counter() - t)
return dict(latency_ms_p50=float(np.percentile(ts, 50)), latency_ms_p95=float(np.percentile(ts, 95)), cold_first_call_ms=cold,
throughput_workflows_per_s_bs64=e2e, questions_per_workflow=3, device=torch.cuda.get_device_name(0) if torch.cuda.is_available() else "cpu")
def llm_baseline(n_per=30, fams=("sst2", "ag_news", "emotion", "snli")):
"""Generative small-LLM baseline on the SAME k=1 questions (answers parsed from free text) -> accuracy + latency."""
from transformers import AutoModelForCausalLM, AutoTokenizer
fams = [f for f in fams if f in DATA]; name = C.llm_baseline
t = AutoTokenizer.from_pretrained(name); lm = AutoModelForCausalLM.from_pretrained(name, torch_dtype=torch.float16).to(dev).eval()
m = ST["model"]; m.eval(); lat, ok_l, ok_a = [], [], []
for f in fams:
for i in range(n_per):
ex = make_example(random.Random(i), [f], "val", "train", idx=C.n_cal + i); q = ex["questions"][0]
state = "\n".join(f"{k}: {v}" for k, v in ex["fields"])
msg = [{"role": "user", "content": f"{state}\n\nQuestion: {q['q']}\nOptions: {', '.join(q['options'])}\nReply with exactly one option and nothing else."}]
ids = t.apply_chat_template(msg, add_generation_prompt=True, return_tensors="pt", return_dict=True).to(dev)
torch.cuda.synchronize(); t0 = time.perf_counter()
with torch.no_grad(): out = lm.generate(**ids, max_new_tokens=12, do_sample=False, pad_token_id=t.eos_token_id)
torch.cuda.synchronize(); lat.append((time.perf_counter() - t0) * 1e3)
txt = t.decode(out[0, ids["input_ids"].shape[1]:], skip_special_tokens=True).strip().lower()
cand = [o for o in q["options"] if o.lower() in txt]; pred = max(cand, key=len) if cand else None
ok_l.append(pred is not None and pred == q["options"][q["gold"]])
b = to_device(build_batch([ex], m.tk, m.tokz, m.cfg), dev)
with torch.no_grad(), ac(): lg = m(b)
ok_a.append(int(lg[0, b["opt_lists"][0]].argmax()) == b["gold_pos"][0])
del lm; torch.cuda.empty_cache()
return dict(model=name, n=len(ok_l), llm_acc=float(np.mean(ok_l)), alpha_acc=float(np.mean(ok_a)), llm_latency_ms_p50=float(np.percentile(lat, 50)),
llm_latency_ms_p95=float(np.percentile(lat, 95)), families=fams)
def benchmark(run_llm=True):
m = ST["model"]; m.eval(); T = float(m.post_T); R = dict(post_T=T, quality={}, calib={})
for wording in ("train", "unseen"):
allr = []
for f in ST["train_fams"]:
res = predict_res(m, prep(val_examples(f, C.n_cal, C.n_cal + C.n_test, wording))); allr += res
s = stat(res, T)["per"][f]; R["quality"].setdefault(wording, {})[f] = {k: s[k] for k in ("acc", "nll", "ece", "brier", "n")}
s0, s1 = stat(allr, 1.0), stat(allr, T)
R["quality"][wording]["_macro"] = dict(acc=s1["score"], nll=s1["nll"], ece=s1["ece"], brier=s1["brier"])
R["calib"][wording] = dict(ece_before_T=s0["ece"], ece_after_T=s1["ece"], conf=s1["micro"]["conf"], corr=s1["micro"]["corr"])
R["held_out"] = {}
for f in ST["held_fams"]:
eb = prep(val_examples(f, C.n_cal, C.n_cal + C.n_test, "unseen"))
c = adapt_curve(m, f, eb, C.probe_steps, C.probe_at); R["held_out"][f] = dict(zero_shot_acc=c[0][0], adapted_acc={k: v[0] for k, v in c.items()}, n=len(val_examples(f, C.n_cal, C.n_cal + C.n_test, "unseen")))
R["workflow"] = workflow_eval()
R["speed"] = speed_bench()
R["type_safety"] = dict(schema_violations=0, note="outputs are a masked softmax over the declared options; an out-of-schema answer cannot be produced")
if run_llm:
try: R["llm_baseline"] = llm_baseline()
except Exception as e: R["llm_baseline"] = dict(error=f"{type(e).__name__}: {str(e)[:200]}")
R["frr"] = ST["frr"]; R["dmr"] = ST["DMR"]; R["meta"] = {str(k): v for k, v in ST["meta"].items()}
ST["R"] = R
print(json.dumps({k: v for k, v in R.items() if k not in ("calib", "meta")}, indent=1, default=float)[:6000]); return R
def plots(path="alpha_report.png"):
import matplotlib; import matplotlib.pyplot as plt
H, R = ST["hist"], ST["R"]; st = [h["step"] for h in H]
fig, ax = plt.subplots(2, 4, figsize=(22, 9)); ax = ax.ravel()
ax[0].plot(st, [h["score"] for h in H], label="val acc"); ax[0].plot(st, [h["tr_score"] for h in H], label="train acc"); ax[0].set_title("Learning curve (EMA weights)"); ax[0].legend()
ax[1].plot(st, [h["nll"] for h in H], label="val nll"); ax[1].plot(st, [h["tr_nll"] for h in H], label="train nll"); ax[1].axvline(ST["DMR"]["T_discover"], color="g", ls=":"); ax[1].axvline(ST["DMR"]["T_memorize"], color="r", ls=":")
ax[1].set_title(f"DMR = {ST['DMR']['DMR']:.2f} (T_discover/T_memorize)"); ax[1].legend()
ax[2].plot(st, [h["wd"] for h in H], label="wd (new modules)"); ax[2].plot(st, [h["rho"] if h["rho"] is not None else 0 for h in H], label="rho"); ax[2].plot(st, [h["gn"] for h in H], label="grad norm"); ax[2].legend(); ax[2].set_title("Discovery controller")
sg = list(R["frr"]); ax[3].bar([str(s) for s in sg], [min(R["frr"][s]["FRR"], 100) for s in sg]); ax[3].axhline(1, color="k", ls="--"); ax[3].set_yscale("log"); ax[3].set_title("FRR vs perturbation sigma (basin test)")
mg = ST["meta"]; ss = sorted(k for k in mg); ax[4].plot(ss, [mg[s]["MGR"] for s in ss], "o-"); ax[4].axhline(1, color="k", ls="--"); ax[4].set_title("Meta-grokking rate MGR (held-out families)")
for f in ST["held_fams"]: ax[5].plot(ss, [mg[s][f]["zero_shot"] for s in ss], "o-", label=f"{f} zero-shot"); ax[5].plot(ss, [mg[s][f]["acc10"] for s in ss], "s--", label=f"{f} +10 steps")
ax[5].legend(fontsize=7); ax[5].set_title("Unseen families: zero-shot / after 10 steps")
conf, corr = np.array(R["calib"]["train"]["conf"]), np.array(R["calib"]["train"]["corr"]); e = np.linspace(0, 1, 11)
xs, ys = [], []
for lo, hi in zip(e[:-1], e[1:]):
mk = (conf > lo) & (conf <= hi)
if mk.sum() > 5: xs.append(conf[mk].mean()); ys.append(corr[mk].mean())
ax[6].plot([0, 1], [0, 1], "k--"); ax[6].plot(xs, ys, "o-"); ax[6].set_title(f"Reliability (ECE {R['calib']['train']['ece_after_T']:.3f})")
fams = ST["train_fams"]; x = np.arange(len(fams))
ax[7].bar(x - .2, [R["quality"]["train"][f]["acc"] for f in fams], .4, label="seen wording"); ax[7].bar(x + .2, [R["quality"]["unseen"][f]["acc"] for f in fams], .4, label="unseen wording")
ax[7].set_xticks(x); ax[7].set_xticklabels(fams, rotation=60); ax[7].legend(); ax[7].set_title("Test accuracy per family")
plt.tight_layout(); plt.savefig(path, dpi=110); plt.show()
# ----------------------------------------------------------------------------------------------- publish
def get_token():
t = os.environ.get("HF_TOKEN")
if not t:
try:
from kaggle_secrets import UserSecretsClient
t = UserSecretsClient().get_secret("HF_TOKEN")
except Exception: pass
return t
def model_card(R):
q, sp, wf, h = R["quality"], R["speed"], R["workflow"], R["held_out"]
rows = "\n".join(f"| {f} | {q['train'][f]['acc']:.3f} | {q['unseen'][f]['acc']:.3f} | {q['train'][f]['ece']:.3f} |" for f in ST["train_fams"])
hrows = "\n".join(f"| {f} | {v['zero_shot_acc']:.3f} | {v['adapted_acc'][10]:.3f} | {v['adapted_acc'][C.probe_steps]:.3f} |" for f, v in h.items())
llm = R.get("llm_baseline", {}); llm_s = (f"Qwen2.5-0.5B-Instruct (generative, same {llm['n']} questions): acc {llm['llm_acc']:.3f}, p50 latency {llm['llm_latency_ms_p50']:.0f} ms "
f"vs Alpha acc {llm['alpha_acc']:.3f}, p50 {sp['latency_ms_p50']:.1f} ms.") if "llm_acc" in llm else "LLM baseline not run."
fr = " ".join(f"sigma={s}: FRR {v['FRR']:.1f}{'*' if v['censored'] else ''}" for s, v in R["frr"].items())
return f"""---
license: apache-2.0
base_model: {C.backbone}
tags: [system-one, structured-output, calibrated, classification, parallel-decoding]
---
# Alpha (System-One-style typed decision model)
Alpha takes unstructured / structured state plus **typed questions with declared options** and returns calibrated probabilities for
**all questions in one parallel forward pass**. It cannot generate strings and cannot answer outside the declared options
(masked softmax -> schema violations are impossible by construction). Option embeddings are independent of the state, so a schema can be
compiled once and cached; cardinality is limited only by memory.
Architecture: `{C.backbone}` encoder (state + question slots with `<mask>` readout tokens) -> 2-layer **Slot Reasoner** (set transformer, squared-ReLU,
zero-init residuals) -> biaffine/cosine scorer against option embeddings with per-question learned temperature + post-hoc temperature (T={R['post_T']:.2f}).
Training: {C.steps} steps x {C.bs} workflow-style examples (1-4 questions over named fields, mixed from {len(ST['train_fams'])} public datasets with paraphrased
questions/option wordings), proper-scoring-rule loss (NLL + Brier + ordinal term), EMA weights, functional-memory anchor, discovery controller.
Held-out families (never trained on): {', '.join(ST['held_fams'])}.
## Results (test splits; see results.json, alpha_report.png)
| family | acc (seen wording) | acc (unseen wording) | ECE |
|---|---|---|---|
{rows}
Workflow eval ({wf['k']} questions per query): per-question acc {wf['q_acc']:.3f}, all-correct {wf['all_correct']:.3f}, joint-confidence ECE {wf['joint_ece']:.3f}.
Held-out task families (unseen schema wording): zero-shot / after 10 / after {C.probe_steps} gradient steps
| family | zero-shot | +10 steps | +{C.probe_steps} steps |
|---|---|---|---|
{hrows}
Speed on {sp['device']}: p50 {sp['latency_ms_p50']:.1f} ms, p95 {sp['latency_ms_p95']:.1f} ms for a 3-question query (local, no network); {sp['throughput_workflows_per_s_bs64']:.0f} workflows/s at batch 64.
{llm_s}
Framework diagnostics: DMR={R['dmr']['DMR']:.2f}; FRR by perturbation: {fr} (* = not re-reached within budget); MGR by checkpoint in results.json.
## Honest limits
* This is a small research model trained on public classification datasets. **It is not a replacement for, and has not been compared against, TypeSafe's Jev**
(a closed API; its claims were not verified here). Do not read the latency comparison as a Jev comparison.
* Narrow task coverage (sentiment/topic/intent/NLI/paraphrase/spam/rating); free-form reasoning is out of scope by design.
* Calibration is measured in-distribution; shift will degrade it.
## Usage
```python
from huggingface_hub import snapshot_download; import sys
p = snapshot_download("{C.repo_id}"); sys.path.insert(0, p)
from alpha_s1 import Alpha
m = Alpha.from_pretrained(p, device="cuda").eval()
print(m.predict({{"ticket": "I was charged twice, refund me now."}},
[{{"id": "refund", "question": "Does the customer want a refund?", "options": ["no", "yes"]}}]))
```
"""
def publish(out_dir="alpha_out"):
from huggingface_hub import HfApi
tok = get_token(); assert tok, "Set a Kaggle secret named HF_TOKEN (Add-ons > Secrets) or env var HF_TOKEN."
R = ST["R"]; m = ST["model"]; m.save_pretrained(out_dir)
shutil.copy("alpha_s1.py", out_dir); shutil.copy("alpha_train.py", out_dir)
if os.path.exists("alpha_report.png"): shutil.copy("alpha_report.png", out_dir)
slim = {k: v for k, v in R.items() if k != "calib"}; json.dump(slim, open(f"{out_dir}/results.json", "w"), indent=1, default=float)
json.dump(dict(hist=ST["hist"], probes=ST["probes"]), open(f"{out_dir}/training_log.json", "w"), default=float)
open(f"{out_dir}/README.md", "w").write(model_card(R))
api = HfApi(token=tok); api.create_repo(C.repo_id, exist_ok=True, private=False)
api.upload_folder(folder_path=out_dir, repo_id=C.repo_id, commit_message="Alpha: System-One-style typed decision model")
print(f"published -> https://huggingface.co/{C.repo_id}")
def demo():
m = ST["model"]; m.eval()
state = dict(review="The battery died after two days and support never answered. Total waste of money.", account="plan: pro; tenure_months: 14")
qs = [dict(id="sentiment", question="Is the review positive or negative?", options=["negative", "positive"]),
dict(id="stars", question="How many stars does the review deserve?", options=["1 star", "2 stars", "3 stars", "4 stars", "5 stars"], ordinal=True),
dict(id="topic", question="What is the review about?", options=["hardware", "billing", "support", "shipping"])]
print(json.dumps(m.predict(state, qs), indent=1))
def chat():
m = ST["model"]; print("Alpha structured console. Empty text to quit.")
while True:
text = input("state text> ").strip()
if not text: break
q = input("question> ").strip(); opts = [o.strip() for o in input("options (comma separated)> ").split(",") if o.strip()]
r = m.predict(dict(input=text), [dict(id="q", question=q, options=opts)])["q"]
print(f" answer={r['answer']} confidence={r['confidence']:.3f} certainty={r['certainty']:.3f}\n probs={ {k: round(v, 3) for k, v in r['probs'].items()} }")