""" Alpha training / benchmark / publish pipeline (Kaggle, 2x T4). Framework rules from the spec are used as *design priors and instruments*, not as proven facts: * Accessibility A(theta,F): squared-ReLU + zero-init Slot Reasoner, pretrained encoder, schema-compositional heads (the general function is the easiest one to reach), many paraphrased schemas (high information rate). * Regularisation shapes the manifold: very different weight decay for pretrained backbone vs new modules, adapted online by the Discovery Controller from rho = d(val)/d(train) (proxy for P(B_G)/P(B_M)). * Functional memory M_F: function-space signatures of past EMA checkpoints on diagnostic set E_F, used as a KL anchor (self-distillation to earlier functions); L_FRR measured mid-training scales the anchor weight. * FRR: perturb the discovered solution (basin test), retrain, time re-discovery -> FRR = T_first / T_re. * Meta-grokking: held-out task families never trained on; adaptation speed measured at checkpoints (MGR). * DMR: T_discover / T_memorize from train/val trajectories ( ~1 => no delayed generalisation, i.e. 'groks from the start'). """ import os, re, json, math, time, random, shutil import numpy as np import torch import torch.nn.functional as F from alpha_s1 import Alpha, AlphaConfig, build_batch, to_device dev = "cuda:0" if torch.cuda.is_available() else "cpu" ac = lambda: torch.autocast("cuda", dtype=torch.float16, enabled=torch.cuda.is_available()) class C: backbone = "distilroberta-base" # try "roberta-base" for more accuracy (slower) repo_id = "SofiTesfay2010/Alpha" seed = 0 steps, bs = 6000, 32 lr_bb, lr_new = 5e-5, 5e-4 wd_bb, wd_new, wd_min, wd_max = 0.01, 0.5, 0.05, 2.0 warmup, clip = 300, 1.0 ls, brier_w, ord_w = 0.02, 0.3, 1.0 ema = 0.999 eval_every, mem_every, anchor_every, lam_mem = 250, 1000, 4, 0.3 probe_every, probe_steps, probe_bs, probe_test_n = 1000, 40, 16, 200 probe_at = (0, 5, 10, 20, 40) fast_n = 48 n_cal, n_test = 150, 500 sigmas = [0.5, 1, 2, 4, 8] noise_unit, frr_max_steps, frr_every = 0.02, 300, 25 llm_baseline = "Qwen/Qwen2.5-0.5B-Instruct" # ----------------------------------------------------------------------------------------------- data def S(paths, fields, fn, qs, names=None, alts=None, tr="train", va="validation", ordinal=False, held=False, ntr=30000, camel=False): return dict(paths=paths, fields=fields, fn=fn, qs=qs, names=names, alts=alts, tr=tr, va=va, ordinal=ordinal, held=held, ntr=ntr, camel=camel) SPECS = dict( sst2=S([("nyu-mll/glue", "sst2")], ["sentence"], [["review"], ["comment"], ["snippet"]], ["What is the sentiment of the {f}?", "Is the {f} positive or negative?", "Judge the attitude expressed in the {f}."], names=["negative", "positive"], alts=[["negative", "positive"], ["bad", "good"], ["unfavorable", "favorable"]]), imdb=S([("stanfordnlp/imdb",)], ["text"], [["movie_review"], ["critique"]], ["Is the {f} favorable or unfavorable?", "What verdict does the {f} give?", "Does the author of the {f} like the film?"], names=["negative", "positive"], alts=[["negative", "positive"], ["thumbs down", "thumbs up"], ["dislikes it", "likes it"]], va="test", ntr=20000), ag_news=S([("fancyzhx/ag_news",)], ["text"], [["headline"], ["article"], ["news_item"]], ["Which section would the {f} appear in?", "What topic is the {f} about?", "Categorize the {f}."], names=["world news", "sports", "business", "science and technology"], alts=[["world news", "sports", "business", "science and technology"], ["world", "sports", "finance", "tech"], ["international affairs", "athletics", "economy", "technology"]], va="test"), emotion=S([("dair-ai/emotion",)], ["text"], [["message"], ["post"], ["tweet"]], ["Which emotion does the {f} convey?", "What is the author of the {f} feeling?", "Name the dominant emotion in the {f}."], names=["sadness", "joy", "love", "anger", "fear", "surprise"], alts=[["sadness", "joy", "love", "anger", "fear", "surprise"], ["sad", "happy", "affection", "angry", "afraid", "astonished"], ["grief", "delight", "adoration", "rage", "dread", "shock"]]), banking77=S([("PolyAI/banking77",), ("mteb/banking77",)], ["text"], [["customer_message"], ["support_query"]], ["What is the customer asking about in the {f}?", "Which intent does the {f} express?", "Classify the banking request in the {f}."], va="test"), yelp=S([("Yelp/yelp_review_full",)], ["text"], [["review"], ["feedback"]], ["How many stars does the {f} deserve?", "Rate the {f}.", "What score would the author of the {f} give?"], names=["1 star", "2 stars", "3 stars", "4 stars", "5 stars"], alts=[["1 star", "2 stars", "3 stars", "4 stars", "5 stars"], ["terrible", "poor", "average", "good", "excellent"], ["very negative", "negative", "neutral", "positive", "very positive"]], va="test", ordinal=True), snli=S([("stanfordnlp/snli",)], ["premise", "hypothesis"], [["premise", "hypothesis"], ["context", "claim"]], ["What is the logical relationship between {f}?", "Does the second follow from the first, contradict it, or neither? ({f})", "Relation between {f}:"], names=["entailment", "neutral", "contradiction"], alts=[["entailment", "neutral", "contradiction"], ["supported", "undetermined", "contradicted"], ["follows", "unrelated", "conflicts"]], ntr=60000), sms=S([("ucirvine/sms_spam",)], ["sms"], [["message"], ["text_message"]], ["Is the {f} spam?", "Classify the {f}.", "Is the {f} an unwanted advertisement or a normal message?"], names=["legitimate", "spam"], alts=[["legitimate", "spam"], ["ham", "spam"], ["normal", "unwanted"]], va=None), news20=S([("SetFit/20_newsgroups",)], ["text"], [["post"], ["usenet_message"]], ["Which newsgroup does the {f} belong to?", "What topic does the {f} discuss?", "Categorize the {f} by topic."], va="test"), mrpc=S([("nyu-mll/glue", "mrpc")], ["sentence1", "sentence2"], [["sentence_a", "sentence_b"], ["statement_1", "statement_2"]], ["Do {f} mean the same thing?", "Are {f} paraphrases of each other?", "Compare the meaning of {f}."], names=["different meaning", "same meaning"], alts=[["different meaning", "same meaning"], ["not paraphrase", "paraphrase"], ["unrelated", "equivalent"]]), # ---- held-out families: NEVER trained on in the main run; used for zero-shot + meta-grokking probes ---- dbpedia=S([("fancyzhx/dbpedia_14",)], ["content"], [["entry"], ["wiki_summary"]], ["What kind of entity does the {f} describe?", "Classify the subject of the {f}.", "Which category fits the {f}?"], va="test", held=True, ntr=20000, camel=True), offensive=S([("cardiffnlp/tweet_eval", "offensive")], ["text"], [["tweet"], ["post"]], ["Is the {f} offensive?", "Does the {f} contain abuse or insults?", "Moderate the {f}."], names=["not offensive", "offensive"], alts=[["not offensive", "offensive"], ["civil", "abusive"], ["acceptable", "insulting"]], held=True), cola=S([("nyu-mll/glue", "cola")], ["sentence"], [["sentence"], ["utterance"]], ["Is the {f} grammatical?", "Is the {f} well-formed English?", "Judge the grammar of the {f}."], names=["ungrammatical", "grammatical"], alts=[["ungrammatical", "grammatical"], ["incorrect", "correct"], ["unacceptable", "acceptable"]], held=True), ) DATA, ST = {}, {} def _clean(n, camel): n = n.replace("_", " ").replace(".", " ") if camel: n = re.sub(r"(?= 0] R = dict(train=rows(dtr), val=rows(dva)) assert len(R["val"]) >= 60 and len(R["train"]) >= 200 DATA[name] = dict(spec=sp, names=names, alts=alts, rows=R) print(f"[data] {name:10s} classes={len(names):3d} train={len(R['train']):6d} val={len(R['val']):5d} {'(held-out)' if sp['held'] else ''}") break except Exception as e: print(f"[skip] {name} via {p}: {type(e).__name__}: {str(e)[:100]}") ST["train_fams"] = sorted(f for f in DATA if not DATA[f]["spec"]["held"]) ST["held_fams"] = sorted(f for f in DATA if DATA[f]["spec"]["held"]) assert len(ST["train_fams"]) >= 4, "too few training families loaded (is internet enabled?)" def make_example(rng, fams, split="train", wording="train", idx=None, lo=0): fields, qs, used = [], [], set() for fam in fams: D = DATA[fam]; sp = D["spec"]; rows = D["rows"][split] if idx is not None: texts, label = rows[idx % len(rows)] else: texts, label = rows[rng.randrange(lo if len(rows) > lo else 0, len(rows))] nm = [] for n0 in rng.choice(sp["fn"]): n1, c = n0, 2 while n1 in used: n1 = f"{n0}_{c}"; c += 1 used.add(n1); nm.append(n1) fields += list(zip(nm, texts)) if wording == "unseen": q, alt = sp["qs"][2], D["alts"][-1] elif wording == "any": q, alt = rng.choice(sp["qs"]), rng.choice(D["alts"]) else: q, alt = rng.choice(sp["qs"][:2]), rng.choice(D["alts"][:-1] or D["alts"]) qs.append(dict(q=q.format(f=" and ".join(nm)), options=list(alt), gold=label, ordinal=sp["ordinal"], fam=fam)) rng.shuffle(fields); rng.shuffle(qs) return dict(fields=fields, questions=qs) def sample_exs(rng, n): fams, w = ST["train_fams"], ST["w"]; out = [] for _ in range(n): k = min(rng.choices([1, 2, 3, 4], [.3, .3, .25, .15])[0], len(fams)); ch = [] while len(ch) < k: f = rng.choices(fams, w)[0] if f not in ch: ch.append(f) out.append(make_example(rng, ch)) return out def prep(exs, bs=48): m = ST["model"] return [to_device(build_batch(exs[i:i + bs], m.tk, m.tokz, m.cfg), dev) for i in range(0, len(exs), bs)] def val_examples(fam, lo, hi, wording): n = len(DATA[fam]["rows"]["val"]) return [make_example(random.Random(1000 + i), [fam], "val", wording, idx=i) for i in range(lo, min(hi, n))] # ----------------------------------------------------------------------------------------------- metrics def ece_fn(conf, corr, nb=15): conf, corr = np.asarray(conf, float), np.asarray(corr, float); e = 0.0 edges = np.linspace(0, 1, nb + 1) for lo, hi in zip(edges[:-1], edges[1:]): m = (conf > lo) & (conf <= hi) if m.any(): e += m.mean() * abs(corr[m].mean() - conf[m].mean()) return float(e) @torch.no_grad() def predict_res(model, batches): model.eval(); out = [] for b in batches: with ac(): lg = model(b) lg = lg.float().cpu() for r, idx in enumerate(b["opt_lists"]): out.append((b["fams"][r], lg[r, idx].numpy(), b["gold_pos"][r])) return out def _calc(items): acc = nll = br = 0.0; conf, corr = [], [] for lg, g in items: z = lg - lg.max(); lp = z - np.log(np.exp(z).sum()); p = np.exp(lp) oh = np.zeros_like(p); oh[g] = 1 acc += float(p.argmax() == g); nll += -lp[g]; br += ((p - oh) ** 2).sum() conf.append(float(p.max())); corr.append(float(p.argmax() == g)) n = len(items) return dict(acc=acc / n, nll=nll / n, brier=br / n, ece=ece_fn(conf, corr), n=n, conf=conf, corr=corr) def stat(res, T=1.0): per = {} for fam, lg, g in res: per.setdefault(fam, []).append((lg / T, g)) P = {f: _calc(v) for f, v in per.items()} allitems = [x for v in per.values() for x in v]; micro = _calc(allitems) mac = lambda k: float(np.mean([P[f][k] for f in P])) return dict(per=P, score=mac("acc"), nll=mac("nll"), brier=mac("brier"), ece=micro["ece"], micro=micro, per_acc={f: P[f]["acc"] for f in P}) def fast_eval(model, which): return stat(predict_res(model, ST["fast_" + which]), float(model.post_T)) # ----------------------------------------------------------------------------------------------- training parts def loss_fn(model, b): lg = model(b).float(); lp = F.log_softmax(lg, -1); al = b["allowed"] ar = torch.arange(lg.size(0), device=lg.device) nll = -lp[ar, b["gold"]] uni = -(lp.masked_fill(~al, 0.0)).sum(1) / al.sum(1) p = lp.exp(); oh = F.one_hot(b["gold"], lg.size(1)).float() brier = ((p - oh) ** 2).sum(1) rg = b["rank"][ar, b["gold"]] ordl = (p * (b["rank"] - rg[:, None]) ** 2 * al).sum(1) L = (1 - C.ls) * nll + C.ls * uni + C.brier_w * brier + C.ord_w * b["ordinal"].float() * ordl return L.mean(), float(nll.mean()) def make_scaler(): try: return torch.amp.GradScaler("cuda", enabled=torch.cuda.is_available()) except Exception: return torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available()) def make_opt(model, lr_bb, lr_new, wd_new): def split(ps): return [p for p in ps if p.ndim >= 2], [p for p in ps if p.ndim < 2] bd, bn = split(list(model.backbone.parameters())); nd, nn_ = split(model.new_params()) gs = [dict(params=bd, base_lr=lr_bb, lr=lr_bb, weight_decay=C.wd_bb, tag="bb"), dict(params=bn, base_lr=lr_bb, lr=lr_bb, weight_decay=0.0, tag="bb0"), dict(params=nd, base_lr=lr_new, lr=lr_new, weight_decay=wd_new, tag="new"), dict(params=nn_, base_lr=lr_new, lr=lr_new, weight_decay=0.0, tag="new0")] return torch.optim.AdamW([g for g in gs if g["params"]], betas=(0.9, 0.98), eps=1e-6) def sgd_step(model, opt, scaler, b, mem=None, lam=0.0, anchor=False): model.train() with ac(): loss, nll = loss_fn(model, b) if anchor and mem is not None and mem.entries: loss = loss + lam * mem.anchor(model) opt.zero_grad(set_to_none=True); scaler.scale(loss).backward(); scaler.unscale_(opt) gn = torch.nn.utils.clip_grad_norm_(model.parameters(), C.clip); scaler.step(opt); scaler.update() return nll, float(gn) def clone_state(m): return {k: v.detach().cpu().clone() for k, v in m.state_dict().items()} class EMA: def __init__(s, model, decay): s.d, s.bk = decay, None s.sh = {k: v.detach().clone().float() for k, v in model.state_dict().items() if v.dtype.is_floating_point} @torch.no_grad() def update(s, model, step): d = min(s.d, (1 + step) / (10 + step)) for k, v in model.state_dict().items(): if k in s.sh: s.sh[k].mul_(d).add_(v.detach().float(), alpha=1 - d) @torch.no_grad() def apply(s, model): sd = model.state_dict(); s.bk = {k: sd[k].detach().clone() for k in s.sh} model.load_state_dict({k: s.sh[k].to(sd[k].dtype) for k in s.sh}, strict=False) @torch.no_grad() def restore(s, model): model.load_state_dict(s.bk, strict=False) class FMem: """Functional memory M_F: entries = {step, signature Sigma_F (dense logits on diagnostic set E_F), val score}.""" def __init__(s, diag_batch): s.b, s.entries = diag_batch, [] @torch.no_grad() def add(s, model, step, score): model.eval() with ac(): lg = model(s.b).float() s.entries.append(dict(step=step, sig=lg.cpu().half(), score=score)) def anchor(s, model): e = random.choice(s.entries); old = e["sig"].to(s.b["allowed"].device).float() lp = F.log_softmax(model(s.b).float(), -1); p = F.softmax(old, -1); lpo = F.log_softmax(old, -1) return ((p * (lpo - lp)).masked_fill(~s.b["allowed"], 0.0)).sum(1).mean() class Controller: """Discovery controller: rho = d(val nll)/d(train nll) ~ share of fitting progress that generalises (proxy of P(B_G)/P(B_M)).""" def __init__(s): s.wd, s.rho, s.prev, s.LFRR = C.wd_new, None, None, 0.0 def update(s, rec): if s.prev is not None: dtr, dv = s.prev["tr_nll"] - rec["tr_nll"], s.prev["nll"] - rec["nll"] if dtr > 1e-3: r = float(np.clip(dv / dtr, -1, 2)); s.rho = r if s.rho is None else 0.5 * s.rho + 0.5 * r if s.rho is not None: if s.rho < 0.3: s.wd = min(C.wd_max, s.wd * 1.25) # memorising -> shape manifold harder elif s.rho > 0.8: s.wd = max(C.wd_min, s.wd * 0.9) # generalising -> relax s.prev = rec def perturb(model, sigma): with torch.no_grad(): for p in model.parameters(): if p.ndim >= 2: p.add_(torch.randn_like(p) * p.std() * sigma * C.noise_unit) def frr_test(model, hist, final_score, sigmas): """FRR = T_first / T_re. T_first: first logged step the main run reached tau. T_re: steps to re-reach tau after perturbing.""" snap = clone_state(model); tau = final_score - 0.01 T_first = next((h["step"] for h in hist if h["score"] >= tau), hist[-1]["step"]); res = {} for sg in sigmas: model.load_state_dict(snap); torch.manual_seed(int(sg * 100)); perturb(model, sg) v0 = fast_eval(model, "val")["score"]; trivial = v0 >= tau; T_re, hit = 1, trivial if not trivial: opt, scaler, rng = make_opt(model, C.lr_bb, C.lr_new, C.wd_new), make_scaler(), random.Random(999 + int(sg * 10)); T_re = C.frr_max_steps for s in range(1, C.frr_max_steps + 1): for g in opt.param_groups: g["lr"] = g["base_lr"] * min(1.0, s / 5) sgd_step(model, opt, scaler, to_device(build_batch(sample_exs(rng, C.bs), model.tk, model.tokz, model.cfg), dev)) if s % C.frr_every == 0 and fast_eval(model, "val")["score"] >= tau: T_re, hit = s, True; break res[sg] = dict(v0=v0, tau=tau, T_first=T_first, T_re=T_re, FRR=T_first / max(T_re, 1), L_FRR=max(T_re, 1) / max(T_first, 1), censored=(not hit), trivial=trivial) print(f" [FRR] sigma={sg}: perturbed score {v0:.3f} (tau {tau:.3f}) T_first={T_first} T_re={T_re}{'+' if not hit else ''} FRR={res[sg]['FRR']:.1f}") model.load_state_dict(snap); return res def adapt_curve(model, fam, eval_batches, steps, at): """Few-shot adaptation of a snapshot to a held-out family; returns {step: (acc, nll)}. Model is restored afterwards.""" snap = clone_state(model); opt, scaler, rng = make_opt(model, C.lr_bb, C.lr_new, 0.0), make_scaler(), random.Random(777); curve = {} for s in range(steps + 1): if s in at: r = stat(predict_res(model, eval_batches), float(model.post_T)); curve[s] = (r["score"], r["nll"]) if s == steps: break for g in opt.param_groups: g["lr"] = g["base_lr"] * min(1.0, (s + 1) / 3) b = to_device(build_batch([make_example(rng, [fam], "train", "any") for _ in range(C.probe_bs)], model.tk, model.tokz, model.cfg), dev) sgd_step(model, opt, scaler, b) model.load_state_dict(snap); return curve def meta_probe(model, step): out = [dict(step=step, fam=f, curve=adapt_curve(model, f, ST["held_test"][f], C.probe_steps, C.probe_at)) for f in ST["held_fams"]] msg = " | ".join(f"{o['fam']}: zs {o['curve'][0][0]:.2f} -> @10 {o['curve'][10][0]:.2f} -> @{C.probe_steps} {o['curve'][C.probe_steps][0]:.2f}" for o in out) print(f" [meta-probe @ {step}] {msg}"); return out # ----------------------------------------------------------------------------------------------- setup + train def setup(): random.seed(C.seed); np.random.seed(C.seed); torch.manual_seed(C.seed) load_all() m = Alpha(AlphaConfig(backbone=C.backbone)).to(dev).use_data_parallel() ST["model"] = m; ST["w"] = [math.sqrt(len(DATA[f]["rows"]["train"])) for f in ST["train_fams"]] tf, hf = ST["train_fams"], ST["held_fams"] fast = lambda split, lo, n: [make_example(random.Random(i * 31 + j), [f], split, "train", idx=i) for j, f in enumerate(tf) for i in range(lo, lo + n)] ST["fast_val"], ST["fast_train"] = prep(fast("val", 0, C.fast_n)), prep(fast("train", 0, C.fast_n)) ST["cal"] = prep([e for f in tf for e in val_examples(f, 0, C.n_cal, "train")]) ST["held_test"] = {f: prep(val_examples(f, 0, C.probe_test_n, "unseen")) for f in hf} rng = random.Random(4242); ST["diag"] = prep([make_example(rng, rng.sample(tf, min(3, len(tf)))) for _ in range(16)], bs=16)[0] n_all = sum(p.numel() for p in m.parameters()); print(f"params: {n_all/1e6:.1f}M | GPUs: {torch.cuda.device_count()} | train fams {tf} | held-out {hf}") def train(): m, hist = ST["model"], []; rng = random.Random(C.seed + 1) ctl, mem, ema = Controller(), FMem(ST["diag"]), EMA(m, C.ema) opt, scaler = make_opt(m, C.lr_bb, C.lr_new, ctl.wd), make_scaler() ST.update(ctl=ctl, mem=mem, hist=hist, probes=meta_probe(m, 0)); best, lam, loss_ema, t0, mid_done = -1, C.lam_mem, None, time.time(), False for step in range(1, C.steps + 1): prog = max(0.0, (step - C.warmup) / max(1, C.steps - C.warmup)) mult = min(1.0, step / C.warmup) * (0.1 + 0.9 * 0.5 * (1 + math.cos(math.pi * prog))) for g in opt.param_groups: g["lr"] = g["base_lr"] * mult if g["tag"] == "new": g["weight_decay"] = ctl.wd b = to_device(build_batch(sample_exs(rng, C.bs), m.tk, m.tokz, m.cfg), dev) nll, gn = sgd_step(m, opt, scaler, b, mem, lam, anchor=(step % C.anchor_every == 0)) ema.update(m, step); loss_ema = nll if loss_ema is None else 0.98 * loss_ema + 0.02 * nll if step % C.eval_every == 0 or step == C.steps: ema.apply(m); v, t = fast_eval(m, "val"), fast_eval(m, "train") rec = dict(step=step, score=v["score"], nll=v["nll"], ece=v["ece"], tr_score=t["score"], tr_nll=t["nll"], fam=v["per_acc"], wd=ctl.wd, lam=lam, gn=gn, loss=loss_ema, rho=ctl.rho, lr=opt.param_groups[0]["lr"]) hist.append(rec) if v["score"] > best: best = v["score"]; ST["best_state"] = clone_state(m); ST["best_step"] = step ctl.update(rec) if step % C.mem_every == 0: mem.add(m, step, v["score"]) if step % C.probe_every == 0: ST["probes"] += meta_probe(m, step) if not mid_done and step >= C.steps // 2: mid_done = True; r = frr_test(m, hist, v["score"], [1])[1]; ctl.LFRR = r["L_FRR"]; lam = C.lam_mem * (1 + min(ctl.LFRR, 3.0)) print(f" [L_FRR={ctl.LFRR:.3f}] -> anchor weight {lam:.3f}") ema.restore(m) print(f"step {step:5d} | val acc {v['score']:.3f} nll {v['nll']:.3f} ece {v['ece']:.3f} | train acc {t['score']:.3f} nll {t['nll']:.3f} | " f"rho {ctl.rho if ctl.rho is None else round(ctl.rho, 2)} wd {ctl.wd:.3f} lam {lam:.2f} gn {gn:.2f} | {time.time()-t0:.0f}s") m.load_state_dict(ST["best_state"]); print(f"loaded best EMA checkpoint (step {ST['best_step']})") # ----------------------------------------------------------------------------------------------- finalize + benchmark def finalize(): m, hist = ST["model"], ST["hist"]; m.eval() res = predict_res(m, ST["cal"]); grid = np.arange(0.5, 3.01, 0.02) nlls = [np.mean([-(lg / T - np.logaddexp.reduce(lg / T))[g] for _, lg, g in res]) for T in grid] T = float(grid[int(np.argmin(nlls))]); m.post_T.fill_(T); print(f"post-hoc temperature T={T:.2f}") best = max(h["score"] for h in hist); print("FRR sweep (basin test):"); ST["frr"] = frr_test(m, hist, best, C.sigmas) tau = min(h["nll"] for h in hist) + 0.05 Tv = next((h["step"] for h in hist if h["nll"] <= tau), hist[-1]["step"]); Tt = next((h["step"] for h in hist if h["tr_nll"] <= tau), hist[-1]["step"]) ST["DMR"] = dict(T_discover=Tv, T_memorize=Tt, DMR=Tv / max(Tt, 1)); print("DMR:", ST["DMR"]) # meta-grokking: T_D(n) = adaptation steps to reach 95% of the best adapted accuracy seen over all probes (per family) P, fams, mg = ST["probes"], ST["held_fams"], {} steps_ = sorted({p["step"] for p in P}) for f in fams: tau_f = 0.95 * max(p["curve"][C.probe_steps][0] for p in P if p["fam"] == f) for s in steps_: c = next(p for p in P if p["fam"] == f and p["step"] == s)["curve"] td = next((k for k in sorted(c) if c[k][0] >= tau_f), C.probe_steps * 2) mg.setdefault(s, {})[f] = dict(T_D=max(td, 1), zero_shot=c[0][0], acc10=c[10][0], final=c[C.probe_steps][0]) base = steps_[0] for s in steps_: g = [mg[base][f]["T_D"] / mg[s][f]["T_D"] for f in fams]; mg[s]["MGR"] = float(np.exp(np.mean(np.log(g)))) ST["meta"] = mg; print("meta-grokking MGR by checkpoint:", {s: round(mg[s]["MGR"], 2) for s in steps_}) @torch.no_grad() def workflow_eval(n=300, k=4): m = ST["model"]; rng = random.Random(2024); exs = [] for _ in range(n): exs.append(make_example(rng, rng.sample(ST["train_fams"], min(k, len(ST["train_fams"]))), "val", "train", lo=C.n_cal)) m.eval(); qacc, allok, jc, jok = [], [], [], [] for i in range(0, n, 32): b = to_device(build_batch(exs[i:i + 32], m.tk, m.tokz, m.cfg), dev) with ac(): lg = m(b).float() / m.post_T per = {} for r, idx in enumerate(b["opt_lists"]): p = F.softmax(lg[r, idx], -1); ok = int(p.argmax()) == b["gold_pos"][r] qacc.append(ok); per.setdefault(b["q_ex"][r], []).append((ok, float(p.max()))) for v in per.values(): allok.append(all(o for o, _ in v)); jc.append(float(np.prod([c for _, c in v]))); jok.append(float(all(o for o, _ in v))) return dict(q_acc=float(np.mean(qacc)), all_correct=float(np.mean(allok)), joint_ece=ece_fn(jc, jok), k=k, n=n) def speed_bench(): m = ST["model"]; m.eval() state = dict(customer_ticket="My card was charged twice for the same order and support has not replied in three days. I want my money back.", account="plan: pro; tenure_months: 14; open_tickets: 2") qs = [dict(id="refund", question="Does the customer ask for a refund?", options=["no", "yes"]), dict(id="urgency", question="How urgent is the ticket?", options=["low", "medium", "high"], ordinal=True), dict(id="team", question="Which team should handle the ticket?", options=["billing", "technical", "sales", "account", "abuse"])] sync = (lambda: torch.cuda.synchronize()) if torch.cuda.is_available() else (lambda: None) t = time.perf_counter(); m.predict(state, qs); sync(); cold = (time.perf_counter() - t) * 1e3 for _ in range(10): m.predict(state, qs) ts = [] for _ in range(200): sync(); t = time.perf_counter(); m.predict(state, qs); sync(); ts.append((time.perf_counter() - t) * 1e3) rng = random.Random(1); exs = [make_example(rng, rng.sample(ST["train_fams"], 3)) for _ in range(64)] with torch.no_grad(): for _ in range(3): m(to_device(build_batch(exs, m.tk, m.tokz, m.cfg), dev)) sync(); t = time.perf_counter() for _ in range(10): b = to_device(build_batch(exs, m.tk, m.tokz, m.cfg), dev) with torch.no_grad(), ac(): m(b) sync(); e2e = 10 * 64 / (time.perf_counter() - t) return dict(latency_ms_p50=float(np.percentile(ts, 50)), latency_ms_p95=float(np.percentile(ts, 95)), cold_first_call_ms=cold, throughput_workflows_per_s_bs64=e2e, questions_per_workflow=3, device=torch.cuda.get_device_name(0) if torch.cuda.is_available() else "cpu") def llm_baseline(n_per=30, fams=("sst2", "ag_news", "emotion", "snli")): """Generative small-LLM baseline on the SAME k=1 questions (answers parsed from free text) -> accuracy + latency.""" from transformers import AutoModelForCausalLM, AutoTokenizer fams = [f for f in fams if f in DATA]; name = C.llm_baseline t = AutoTokenizer.from_pretrained(name); lm = AutoModelForCausalLM.from_pretrained(name, torch_dtype=torch.float16).to(dev).eval() m = ST["model"]; m.eval(); lat, ok_l, ok_a = [], [], [] for f in fams: for i in range(n_per): ex = make_example(random.Random(i), [f], "val", "train", idx=C.n_cal + i); q = ex["questions"][0] state = "\n".join(f"{k}: {v}" for k, v in ex["fields"]) msg = [{"role": "user", "content": f"{state}\n\nQuestion: {q['q']}\nOptions: {', '.join(q['options'])}\nReply with exactly one option and nothing else."}] ids = t.apply_chat_template(msg, add_generation_prompt=True, return_tensors="pt", return_dict=True).to(dev) torch.cuda.synchronize(); t0 = time.perf_counter() with torch.no_grad(): out = lm.generate(**ids, max_new_tokens=12, do_sample=False, pad_token_id=t.eos_token_id) torch.cuda.synchronize(); lat.append((time.perf_counter() - t0) * 1e3) txt = t.decode(out[0, ids["input_ids"].shape[1]:], skip_special_tokens=True).strip().lower() cand = [o for o in q["options"] if o.lower() in txt]; pred = max(cand, key=len) if cand else None ok_l.append(pred is not None and pred == q["options"][q["gold"]]) b = to_device(build_batch([ex], m.tk, m.tokz, m.cfg), dev) with torch.no_grad(), ac(): lg = m(b) ok_a.append(int(lg[0, b["opt_lists"][0]].argmax()) == b["gold_pos"][0]) del lm; torch.cuda.empty_cache() return dict(model=name, n=len(ok_l), llm_acc=float(np.mean(ok_l)), alpha_acc=float(np.mean(ok_a)), llm_latency_ms_p50=float(np.percentile(lat, 50)), llm_latency_ms_p95=float(np.percentile(lat, 95)), families=fams) def benchmark(run_llm=True): m = ST["model"]; m.eval(); T = float(m.post_T); R = dict(post_T=T, quality={}, calib={}) for wording in ("train", "unseen"): allr = [] for f in ST["train_fams"]: res = predict_res(m, prep(val_examples(f, C.n_cal, C.n_cal + C.n_test, wording))); allr += res s = stat(res, T)["per"][f]; R["quality"].setdefault(wording, {})[f] = {k: s[k] for k in ("acc", "nll", "ece", "brier", "n")} s0, s1 = stat(allr, 1.0), stat(allr, T) R["quality"][wording]["_macro"] = dict(acc=s1["score"], nll=s1["nll"], ece=s1["ece"], brier=s1["brier"]) R["calib"][wording] = dict(ece_before_T=s0["ece"], ece_after_T=s1["ece"], conf=s1["micro"]["conf"], corr=s1["micro"]["corr"]) R["held_out"] = {} for f in ST["held_fams"]: eb = prep(val_examples(f, C.n_cal, C.n_cal + C.n_test, "unseen")) c = adapt_curve(m, f, eb, C.probe_steps, C.probe_at); R["held_out"][f] = dict(zero_shot_acc=c[0][0], adapted_acc={k: v[0] for k, v in c.items()}, n=len(val_examples(f, C.n_cal, C.n_cal + C.n_test, "unseen"))) R["workflow"] = workflow_eval() R["speed"] = speed_bench() R["type_safety"] = dict(schema_violations=0, note="outputs are a masked softmax over the declared options; an out-of-schema answer cannot be produced") if run_llm: try: R["llm_baseline"] = llm_baseline() except Exception as e: R["llm_baseline"] = dict(error=f"{type(e).__name__}: {str(e)[:200]}") R["frr"] = ST["frr"]; R["dmr"] = ST["DMR"]; R["meta"] = {str(k): v for k, v in ST["meta"].items()} ST["R"] = R print(json.dumps({k: v for k, v in R.items() if k not in ("calib", "meta")}, indent=1, default=float)[:6000]); return R def plots(path="alpha_report.png"): import matplotlib; import matplotlib.pyplot as plt H, R = ST["hist"], ST["R"]; st = [h["step"] for h in H] fig, ax = plt.subplots(2, 4, figsize=(22, 9)); ax = ax.ravel() ax[0].plot(st, [h["score"] for h in H], label="val acc"); ax[0].plot(st, [h["tr_score"] for h in H], label="train acc"); ax[0].set_title("Learning curve (EMA weights)"); ax[0].legend() ax[1].plot(st, [h["nll"] for h in H], label="val nll"); ax[1].plot(st, [h["tr_nll"] for h in H], label="train nll"); ax[1].axvline(ST["DMR"]["T_discover"], color="g", ls=":"); ax[1].axvline(ST["DMR"]["T_memorize"], color="r", ls=":") ax[1].set_title(f"DMR = {ST['DMR']['DMR']:.2f} (T_discover/T_memorize)"); ax[1].legend() ax[2].plot(st, [h["wd"] for h in H], label="wd (new modules)"); ax[2].plot(st, [h["rho"] if h["rho"] is not None else 0 for h in H], label="rho"); ax[2].plot(st, [h["gn"] for h in H], label="grad norm"); ax[2].legend(); ax[2].set_title("Discovery controller") sg = list(R["frr"]); ax[3].bar([str(s) for s in sg], [min(R["frr"][s]["FRR"], 100) for s in sg]); ax[3].axhline(1, color="k", ls="--"); ax[3].set_yscale("log"); ax[3].set_title("FRR vs perturbation sigma (basin test)") mg = ST["meta"]; ss = sorted(k for k in mg); ax[4].plot(ss, [mg[s]["MGR"] for s in ss], "o-"); ax[4].axhline(1, color="k", ls="--"); ax[4].set_title("Meta-grokking rate MGR (held-out families)") for f in ST["held_fams"]: ax[5].plot(ss, [mg[s][f]["zero_shot"] for s in ss], "o-", label=f"{f} zero-shot"); ax[5].plot(ss, [mg[s][f]["acc10"] for s in ss], "s--", label=f"{f} +10 steps") ax[5].legend(fontsize=7); ax[5].set_title("Unseen families: zero-shot / after 10 steps") conf, corr = np.array(R["calib"]["train"]["conf"]), np.array(R["calib"]["train"]["corr"]); e = np.linspace(0, 1, 11) xs, ys = [], [] for lo, hi in zip(e[:-1], e[1:]): mk = (conf > lo) & (conf <= hi) if mk.sum() > 5: xs.append(conf[mk].mean()); ys.append(corr[mk].mean()) ax[6].plot([0, 1], [0, 1], "k--"); ax[6].plot(xs, ys, "o-"); ax[6].set_title(f"Reliability (ECE {R['calib']['train']['ece_after_T']:.3f})") fams = ST["train_fams"]; x = np.arange(len(fams)) ax[7].bar(x - .2, [R["quality"]["train"][f]["acc"] for f in fams], .4, label="seen wording"); ax[7].bar(x + .2, [R["quality"]["unseen"][f]["acc"] for f in fams], .4, label="unseen wording") ax[7].set_xticks(x); ax[7].set_xticklabels(fams, rotation=60); ax[7].legend(); ax[7].set_title("Test accuracy per family") plt.tight_layout(); plt.savefig(path, dpi=110); plt.show() # ----------------------------------------------------------------------------------------------- publish def get_token(): t = os.environ.get("HF_TOKEN") if not t: try: from kaggle_secrets import UserSecretsClient t = UserSecretsClient().get_secret("HF_TOKEN") except Exception: pass return t def model_card(R): q, sp, wf, h = R["quality"], R["speed"], R["workflow"], R["held_out"] rows = "\n".join(f"| {f} | {q['train'][f]['acc']:.3f} | {q['unseen'][f]['acc']:.3f} | {q['train'][f]['ece']:.3f} |" for f in ST["train_fams"]) hrows = "\n".join(f"| {f} | {v['zero_shot_acc']:.3f} | {v['adapted_acc'][10]:.3f} | {v['adapted_acc'][C.probe_steps]:.3f} |" for f, v in h.items()) llm = R.get("llm_baseline", {}); llm_s = (f"Qwen2.5-0.5B-Instruct (generative, same {llm['n']} questions): acc {llm['llm_acc']:.3f}, p50 latency {llm['llm_latency_ms_p50']:.0f} ms " f"vs Alpha acc {llm['alpha_acc']:.3f}, p50 {sp['latency_ms_p50']:.1f} ms.") if "llm_acc" in llm else "LLM baseline not run." fr = " ".join(f"sigma={s}: FRR {v['FRR']:.1f}{'*' if v['censored'] else ''}" for s, v in R["frr"].items()) return f"""--- license: apache-2.0 base_model: {C.backbone} tags: [system-one, structured-output, calibrated, classification, parallel-decoding] --- # Alpha (System-One-style typed decision model) Alpha takes unstructured / structured state plus **typed questions with declared options** and returns calibrated probabilities for **all questions in one parallel forward pass**. It cannot generate strings and cannot answer outside the declared options (masked softmax -> schema violations are impossible by construction). Option embeddings are independent of the state, so a schema can be compiled once and cached; cardinality is limited only by memory. Architecture: `{C.backbone}` encoder (state + question slots with `` readout tokens) -> 2-layer **Slot Reasoner** (set transformer, squared-ReLU, zero-init residuals) -> biaffine/cosine scorer against option embeddings with per-question learned temperature + post-hoc temperature (T={R['post_T']:.2f}). Training: {C.steps} steps x {C.bs} workflow-style examples (1-4 questions over named fields, mixed from {len(ST['train_fams'])} public datasets with paraphrased questions/option wordings), proper-scoring-rule loss (NLL + Brier + ordinal term), EMA weights, functional-memory anchor, discovery controller. Held-out families (never trained on): {', '.join(ST['held_fams'])}. ## Results (test splits; see results.json, alpha_report.png) | family | acc (seen wording) | acc (unseen wording) | ECE | |---|---|---|---| {rows} Workflow eval ({wf['k']} questions per query): per-question acc {wf['q_acc']:.3f}, all-correct {wf['all_correct']:.3f}, joint-confidence ECE {wf['joint_ece']:.3f}. Held-out task families (unseen schema wording): zero-shot / after 10 / after {C.probe_steps} gradient steps | family | zero-shot | +10 steps | +{C.probe_steps} steps | |---|---|---|---| {hrows} Speed on {sp['device']}: p50 {sp['latency_ms_p50']:.1f} ms, p95 {sp['latency_ms_p95']:.1f} ms for a 3-question query (local, no network); {sp['throughput_workflows_per_s_bs64']:.0f} workflows/s at batch 64. {llm_s} Framework diagnostics: DMR={R['dmr']['DMR']:.2f}; FRR by perturbation: {fr} (* = not re-reached within budget); MGR by checkpoint in results.json. ## Honest limits * This is a small research model trained on public classification datasets. **It is not a replacement for, and has not been compared against, TypeSafe's Jev** (a closed API; its claims were not verified here). Do not read the latency comparison as a Jev comparison. * Narrow task coverage (sentiment/topic/intent/NLI/paraphrase/spam/rating); free-form reasoning is out of scope by design. * Calibration is measured in-distribution; shift will degrade it. ## Usage ```python from huggingface_hub import snapshot_download; import sys p = snapshot_download("{C.repo_id}"); sys.path.insert(0, p) from alpha_s1 import Alpha m = Alpha.from_pretrained(p, device="cuda").eval() print(m.predict({{"ticket": "I was charged twice, refund me now."}}, [{{"id": "refund", "question": "Does the customer want a refund?", "options": ["no", "yes"]}}])) ``` """ def publish(out_dir="alpha_out"): from huggingface_hub import HfApi tok = get_token(); assert tok, "Set a Kaggle secret named HF_TOKEN (Add-ons > Secrets) or env var HF_TOKEN." R = ST["R"]; m = ST["model"]; m.save_pretrained(out_dir) shutil.copy("alpha_s1.py", out_dir); shutil.copy("alpha_train.py", out_dir) if os.path.exists("alpha_report.png"): shutil.copy("alpha_report.png", out_dir) slim = {k: v for k, v in R.items() if k != "calib"}; json.dump(slim, open(f"{out_dir}/results.json", "w"), indent=1, default=float) json.dump(dict(hist=ST["hist"], probes=ST["probes"]), open(f"{out_dir}/training_log.json", "w"), default=float) open(f"{out_dir}/README.md", "w").write(model_card(R)) api = HfApi(token=tok); api.create_repo(C.repo_id, exist_ok=True, private=False) api.upload_folder(folder_path=out_dir, repo_id=C.repo_id, commit_message="Alpha: System-One-style typed decision model") print(f"published -> https://huggingface.co/{C.repo_id}") def demo(): m = ST["model"]; m.eval() state = dict(review="The battery died after two days and support never answered. Total waste of money.", account="plan: pro; tenure_months: 14") qs = [dict(id="sentiment", question="Is the review positive or negative?", options=["negative", "positive"]), dict(id="stars", question="How many stars does the review deserve?", options=["1 star", "2 stars", "3 stars", "4 stars", "5 stars"], ordinal=True), dict(id="topic", question="What is the review about?", options=["hardware", "billing", "support", "shipping"])] print(json.dumps(m.predict(state, qs), indent=1)) def chat(): m = ST["model"]; print("Alpha structured console. Empty text to quit.") while True: text = input("state text> ").strip() if not text: break q = input("question> ").strip(); opts = [o.strip() for o in input("options (comma separated)> ").split(",") if o.strip()] r = m.predict(dict(input=text), [dict(id="q", question=q, options=opts)])["q"] print(f" answer={r['answer']} confidence={r['confidence']:.3f} certainty={r['certainty']:.3f}\n probs={ {k: round(v, 3) for k, v in r['probs'].items()} }")