Download alpha_train.py from SofiTesfay2010/Alpha: direct link, hf CLI and curl.
- Browser
- Download file 41.4 kB
-
https://huggingface.co/SofiTesfay2010/Alpha/resolve/main/alpha_train.py
- Command line
-
hf download hf://SofiTesfay2010/Alpha/alpha_train.py
-
curl -L -o alpha_train.py https://huggingface.co/SofiTesfay2010/Alpha/resolve/main/alpha_train.py
41.4 kB
| """ | |
| Alpha training / benchmark / publish pipeline (Kaggle, 2x T4). | |
| Framework rules from the spec are used as *design priors and instruments*, not as proven facts: | |
| * Accessibility A(theta,F): squared-ReLU + zero-init Slot Reasoner, pretrained encoder, schema-compositional heads | |
| (the general function is the easiest one to reach), many paraphrased schemas (high information rate). | |
| * Regularisation shapes the manifold: very different weight decay for pretrained backbone vs new modules, adapted online | |
| by the Discovery Controller from rho = d(val)/d(train) (proxy for P(B_G)/P(B_M)). | |
| * Functional memory M_F: function-space signatures of past EMA checkpoints on diagnostic set E_F, used as a KL anchor | |
| (self-distillation to earlier functions); L_FRR measured mid-training scales the anchor weight. | |
| * FRR: perturb the discovered solution (basin test), retrain, time re-discovery -> FRR = T_first / T_re. | |
| * Meta-grokking: held-out task families never trained on; adaptation speed measured at checkpoints (MGR). | |
| * DMR: T_discover / T_memorize from train/val trajectories ( ~1 => no delayed generalisation, i.e. 'groks from the start'). | |
| """ | |
| import os, re, json, math, time, random, shutil | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from alpha_s1 import Alpha, AlphaConfig, build_batch, to_device | |
| dev = "cuda:0" if torch.cuda.is_available() else "cpu" | |
| ac = lambda: torch.autocast("cuda", dtype=torch.float16, enabled=torch.cuda.is_available()) | |
| class C: | |
| backbone = "distilroberta-base" # try "roberta-base" for more accuracy (slower) | |
| repo_id = "SofiTesfay2010/Alpha" | |
| seed = 0 | |
| steps, bs = 6000, 32 | |
| lr_bb, lr_new = 5e-5, 5e-4 | |
| wd_bb, wd_new, wd_min, wd_max = 0.01, 0.5, 0.05, 2.0 | |
| warmup, clip = 300, 1.0 | |
| ls, brier_w, ord_w = 0.02, 0.3, 1.0 | |
| ema = 0.999 | |
| eval_every, mem_every, anchor_every, lam_mem = 250, 1000, 4, 0.3 | |
| probe_every, probe_steps, probe_bs, probe_test_n = 1000, 40, 16, 200 | |
| probe_at = (0, 5, 10, 20, 40) | |
| fast_n = 48 | |
| n_cal, n_test = 150, 500 | |
| sigmas = [0.5, 1, 2, 4, 8] | |
| noise_unit, frr_max_steps, frr_every = 0.02, 300, 25 | |
| llm_baseline = "Qwen/Qwen2.5-0.5B-Instruct" | |
| # ----------------------------------------------------------------------------------------------- data | |
| def S(paths, fields, fn, qs, names=None, alts=None, tr="train", va="validation", ordinal=False, held=False, ntr=30000, camel=False): | |
| return dict(paths=paths, fields=fields, fn=fn, qs=qs, names=names, alts=alts, tr=tr, va=va, ordinal=ordinal, held=held, ntr=ntr, camel=camel) | |
| SPECS = dict( | |
| sst2=S([("nyu-mll/glue", "sst2")], ["sentence"], [["review"], ["comment"], ["snippet"]], | |
| ["What is the sentiment of the {f}?", "Is the {f} positive or negative?", "Judge the attitude expressed in the {f}."], | |
| names=["negative", "positive"], alts=[["negative", "positive"], ["bad", "good"], ["unfavorable", "favorable"]]), | |
| imdb=S([("stanfordnlp/imdb",)], ["text"], [["movie_review"], ["critique"]], | |
| ["Is the {f} favorable or unfavorable?", "What verdict does the {f} give?", "Does the author of the {f} like the film?"], | |
| names=["negative", "positive"], alts=[["negative", "positive"], ["thumbs down", "thumbs up"], ["dislikes it", "likes it"]], va="test", ntr=20000), | |
| ag_news=S([("fancyzhx/ag_news",)], ["text"], [["headline"], ["article"], ["news_item"]], | |
| ["Which section would the {f} appear in?", "What topic is the {f} about?", "Categorize the {f}."], | |
| names=["world news", "sports", "business", "science and technology"], | |
| alts=[["world news", "sports", "business", "science and technology"], ["world", "sports", "finance", "tech"], ["international affairs", "athletics", "economy", "technology"]], va="test"), | |
| emotion=S([("dair-ai/emotion",)], ["text"], [["message"], ["post"], ["tweet"]], | |
| ["Which emotion does the {f} convey?", "What is the author of the {f} feeling?", "Name the dominant emotion in the {f}."], | |
| names=["sadness", "joy", "love", "anger", "fear", "surprise"], | |
| alts=[["sadness", "joy", "love", "anger", "fear", "surprise"], ["sad", "happy", "affection", "angry", "afraid", "astonished"], ["grief", "delight", "adoration", "rage", "dread", "shock"]]), | |
| banking77=S([("PolyAI/banking77",), ("mteb/banking77",)], ["text"], [["customer_message"], ["support_query"]], | |
| ["What is the customer asking about in the {f}?", "Which intent does the {f} express?", "Classify the banking request in the {f}."], va="test"), | |
| yelp=S([("Yelp/yelp_review_full",)], ["text"], [["review"], ["feedback"]], | |
| ["How many stars does the {f} deserve?", "Rate the {f}.", "What score would the author of the {f} give?"], | |
| names=["1 star", "2 stars", "3 stars", "4 stars", "5 stars"], | |
| alts=[["1 star", "2 stars", "3 stars", "4 stars", "5 stars"], ["terrible", "poor", "average", "good", "excellent"], ["very negative", "negative", "neutral", "positive", "very positive"]], | |
| va="test", ordinal=True), | |
| snli=S([("stanfordnlp/snli",)], ["premise", "hypothesis"], [["premise", "hypothesis"], ["context", "claim"]], | |
| ["What is the logical relationship between {f}?", "Does the second follow from the first, contradict it, or neither? ({f})", "Relation between {f}:"], | |
| names=["entailment", "neutral", "contradiction"], | |
| alts=[["entailment", "neutral", "contradiction"], ["supported", "undetermined", "contradicted"], ["follows", "unrelated", "conflicts"]], ntr=60000), | |
| sms=S([("ucirvine/sms_spam",)], ["sms"], [["message"], ["text_message"]], | |
| ["Is the {f} spam?", "Classify the {f}.", "Is the {f} an unwanted advertisement or a normal message?"], | |
| names=["legitimate", "spam"], alts=[["legitimate", "spam"], ["ham", "spam"], ["normal", "unwanted"]], va=None), | |
| news20=S([("SetFit/20_newsgroups",)], ["text"], [["post"], ["usenet_message"]], | |
| ["Which newsgroup does the {f} belong to?", "What topic does the {f} discuss?", "Categorize the {f} by topic."], va="test"), | |
| mrpc=S([("nyu-mll/glue", "mrpc")], ["sentence1", "sentence2"], [["sentence_a", "sentence_b"], ["statement_1", "statement_2"]], | |
| ["Do {f} mean the same thing?", "Are {f} paraphrases of each other?", "Compare the meaning of {f}."], | |
| names=["different meaning", "same meaning"], alts=[["different meaning", "same meaning"], ["not paraphrase", "paraphrase"], ["unrelated", "equivalent"]]), | |
| # ---- held-out families: NEVER trained on in the main run; used for zero-shot + meta-grokking probes ---- | |
| dbpedia=S([("fancyzhx/dbpedia_14",)], ["content"], [["entry"], ["wiki_summary"]], | |
| ["What kind of entity does the {f} describe?", "Classify the subject of the {f}.", "Which category fits the {f}?"], va="test", held=True, ntr=20000, camel=True), | |
| offensive=S([("cardiffnlp/tweet_eval", "offensive")], ["text"], [["tweet"], ["post"]], | |
| ["Is the {f} offensive?", "Does the {f} contain abuse or insults?", "Moderate the {f}."], | |
| names=["not offensive", "offensive"], alts=[["not offensive", "offensive"], ["civil", "abusive"], ["acceptable", "insulting"]], held=True), | |
| cola=S([("nyu-mll/glue", "cola")], ["sentence"], [["sentence"], ["utterance"]], | |
| ["Is the {f} grammatical?", "Is the {f} well-formed English?", "Judge the grammar of the {f}."], | |
| names=["ungrammatical", "grammatical"], alts=[["ungrammatical", "grammatical"], ["incorrect", "correct"], ["unacceptable", "acceptable"]], held=True), | |
| ) | |
| DATA, ST = {}, {} | |
| def _clean(n, camel): | |
| n = n.replace("_", " ").replace(".", " ") | |
| if camel: n = re.sub(r"(?<!^)(?=[A-Z])", " ", n) | |
| return n.lower().strip() | |
| def load_all(): | |
| from datasets import load_dataset | |
| for name, sp in SPECS.items(): | |
| for p in sp["paths"]: | |
| try: | |
| if sp["va"]: | |
| dtr = load_dataset(*p, split=sp["tr"]).shuffle(seed=0); dva = load_dataset(*p, split=sp["va"]).shuffle(seed=0) | |
| else: | |
| full = load_dataset(*p, split="train").shuffle(seed=0); cut = int(0.9 * len(full)) | |
| dtr, dva = full.select(range(cut)), full.select(range(cut, len(full))) | |
| dtr = dtr.select(range(min(len(dtr), sp["ntr"]))); dva = dva.select(range(min(len(dva), 1500))) | |
| names = sp["names"] | |
| if names is None: | |
| feat = dtr.features["label"] | |
| if hasattr(feat, "names"): names = list(feat.names) | |
| else: | |
| m = {int(l): t for l, t in zip(dtr["label"], dtr["label_text"])}; names = [m[i] for i in range(max(m) + 1)] | |
| names = [_clean(n, sp["camel"]) for n in names] | |
| alts = sp["alts"] or [names] | |
| def rows(d): | |
| cols = [d[f] for f in sp["fields"]]; lab = d["label"] | |
| return [(tuple(str(c[i])[:1200] for c in cols), int(lab[i])) for i in range(len(lab)) if int(lab[i]) >= 0] | |
| R = dict(train=rows(dtr), val=rows(dva)) | |
| assert len(R["val"]) >= 60 and len(R["train"]) >= 200 | |
| DATA[name] = dict(spec=sp, names=names, alts=alts, rows=R) | |
| print(f"[data] {name:10s} classes={len(names):3d} train={len(R['train']):6d} val={len(R['val']):5d} {'(held-out)' if sp['held'] else ''}") | |
| break | |
| except Exception as e: | |
| print(f"[skip] {name} via {p}: {type(e).__name__}: {str(e)[:100]}") | |
| ST["train_fams"] = sorted(f for f in DATA if not DATA[f]["spec"]["held"]) | |
| ST["held_fams"] = sorted(f for f in DATA if DATA[f]["spec"]["held"]) | |
| assert len(ST["train_fams"]) >= 4, "too few training families loaded (is internet enabled?)" | |
| def make_example(rng, fams, split="train", wording="train", idx=None, lo=0): | |
| fields, qs, used = [], [], set() | |
| for fam in fams: | |
| D = DATA[fam]; sp = D["spec"]; rows = D["rows"][split] | |
| if idx is not None: texts, label = rows[idx % len(rows)] | |
| else: texts, label = rows[rng.randrange(lo if len(rows) > lo else 0, len(rows))] | |
| nm = [] | |
| for n0 in rng.choice(sp["fn"]): | |
| n1, c = n0, 2 | |
| while n1 in used: n1 = f"{n0}_{c}"; c += 1 | |
| used.add(n1); nm.append(n1) | |
| fields += list(zip(nm, texts)) | |
| if wording == "unseen": q, alt = sp["qs"][2], D["alts"][-1] | |
| elif wording == "any": q, alt = rng.choice(sp["qs"]), rng.choice(D["alts"]) | |
| else: q, alt = rng.choice(sp["qs"][:2]), rng.choice(D["alts"][:-1] or D["alts"]) | |
| qs.append(dict(q=q.format(f=" and ".join(nm)), options=list(alt), gold=label, ordinal=sp["ordinal"], fam=fam)) | |
| rng.shuffle(fields); rng.shuffle(qs) | |
| return dict(fields=fields, questions=qs) | |
| def sample_exs(rng, n): | |
| fams, w = ST["train_fams"], ST["w"]; out = [] | |
| for _ in range(n): | |
| k = min(rng.choices([1, 2, 3, 4], [.3, .3, .25, .15])[0], len(fams)); ch = [] | |
| while len(ch) < k: | |
| f = rng.choices(fams, w)[0] | |
| if f not in ch: ch.append(f) | |
| out.append(make_example(rng, ch)) | |
| return out | |
| def prep(exs, bs=48): | |
| m = ST["model"] | |
| return [to_device(build_batch(exs[i:i + bs], m.tk, m.tokz, m.cfg), dev) for i in range(0, len(exs), bs)] | |
| def val_examples(fam, lo, hi, wording): | |
| n = len(DATA[fam]["rows"]["val"]) | |
| return [make_example(random.Random(1000 + i), [fam], "val", wording, idx=i) for i in range(lo, min(hi, n))] | |
| # ----------------------------------------------------------------------------------------------- metrics | |
| def ece_fn(conf, corr, nb=15): | |
| conf, corr = np.asarray(conf, float), np.asarray(corr, float); e = 0.0 | |
| edges = np.linspace(0, 1, nb + 1) | |
| for lo, hi in zip(edges[:-1], edges[1:]): | |
| m = (conf > lo) & (conf <= hi) | |
| if m.any(): e += m.mean() * abs(corr[m].mean() - conf[m].mean()) | |
| return float(e) | |
| def predict_res(model, batches): | |
| model.eval(); out = [] | |
| for b in batches: | |
| with ac(): lg = model(b) | |
| lg = lg.float().cpu() | |
| for r, idx in enumerate(b["opt_lists"]): | |
| out.append((b["fams"][r], lg[r, idx].numpy(), b["gold_pos"][r])) | |
| return out | |
| def _calc(items): | |
| acc = nll = br = 0.0; conf, corr = [], [] | |
| for lg, g in items: | |
| z = lg - lg.max(); lp = z - np.log(np.exp(z).sum()); p = np.exp(lp) | |
| oh = np.zeros_like(p); oh[g] = 1 | |
| acc += float(p.argmax() == g); nll += -lp[g]; br += ((p - oh) ** 2).sum() | |
| conf.append(float(p.max())); corr.append(float(p.argmax() == g)) | |
| n = len(items) | |
| return dict(acc=acc / n, nll=nll / n, brier=br / n, ece=ece_fn(conf, corr), n=n, conf=conf, corr=corr) | |
| def stat(res, T=1.0): | |
| per = {} | |
| for fam, lg, g in res: per.setdefault(fam, []).append((lg / T, g)) | |
| P = {f: _calc(v) for f, v in per.items()} | |
| allitems = [x for v in per.values() for x in v]; micro = _calc(allitems) | |
| mac = lambda k: float(np.mean([P[f][k] for f in P])) | |
| return dict(per=P, score=mac("acc"), nll=mac("nll"), brier=mac("brier"), ece=micro["ece"], micro=micro, | |
| per_acc={f: P[f]["acc"] for f in P}) | |
| def fast_eval(model, which): | |
| return stat(predict_res(model, ST["fast_" + which]), float(model.post_T)) | |
| # ----------------------------------------------------------------------------------------------- training parts | |
| def loss_fn(model, b): | |
| lg = model(b).float(); lp = F.log_softmax(lg, -1); al = b["allowed"] | |
| ar = torch.arange(lg.size(0), device=lg.device) | |
| nll = -lp[ar, b["gold"]] | |
| uni = -(lp.masked_fill(~al, 0.0)).sum(1) / al.sum(1) | |
| p = lp.exp(); oh = F.one_hot(b["gold"], lg.size(1)).float() | |
| brier = ((p - oh) ** 2).sum(1) | |
| rg = b["rank"][ar, b["gold"]] | |
| ordl = (p * (b["rank"] - rg[:, None]) ** 2 * al).sum(1) | |
| L = (1 - C.ls) * nll + C.ls * uni + C.brier_w * brier + C.ord_w * b["ordinal"].float() * ordl | |
| return L.mean(), float(nll.mean()) | |
| def make_scaler(): | |
| try: return torch.amp.GradScaler("cuda", enabled=torch.cuda.is_available()) | |
| except Exception: return torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available()) | |
| def make_opt(model, lr_bb, lr_new, wd_new): | |
| def split(ps): return [p for p in ps if p.ndim >= 2], [p for p in ps if p.ndim < 2] | |
| bd, bn = split(list(model.backbone.parameters())); nd, nn_ = split(model.new_params()) | |
| gs = [dict(params=bd, base_lr=lr_bb, lr=lr_bb, weight_decay=C.wd_bb, tag="bb"), dict(params=bn, base_lr=lr_bb, lr=lr_bb, weight_decay=0.0, tag="bb0"), | |
| dict(params=nd, base_lr=lr_new, lr=lr_new, weight_decay=wd_new, tag="new"), dict(params=nn_, base_lr=lr_new, lr=lr_new, weight_decay=0.0, tag="new0")] | |
| return torch.optim.AdamW([g for g in gs if g["params"]], betas=(0.9, 0.98), eps=1e-6) | |
| def sgd_step(model, opt, scaler, b, mem=None, lam=0.0, anchor=False): | |
| model.train() | |
| with ac(): | |
| loss, nll = loss_fn(model, b) | |
| if anchor and mem is not None and mem.entries: loss = loss + lam * mem.anchor(model) | |
| opt.zero_grad(set_to_none=True); scaler.scale(loss).backward(); scaler.unscale_(opt) | |
| gn = torch.nn.utils.clip_grad_norm_(model.parameters(), C.clip); scaler.step(opt); scaler.update() | |
| return nll, float(gn) | |
| def clone_state(m): return {k: v.detach().cpu().clone() for k, v in m.state_dict().items()} | |
| class EMA: | |
| def __init__(s, model, decay): | |
| s.d, s.bk = decay, None | |
| s.sh = {k: v.detach().clone().float() for k, v in model.state_dict().items() if v.dtype.is_floating_point} | |
| def update(s, model, step): | |
| d = min(s.d, (1 + step) / (10 + step)) | |
| for k, v in model.state_dict().items(): | |
| if k in s.sh: s.sh[k].mul_(d).add_(v.detach().float(), alpha=1 - d) | |
| def apply(s, model): | |
| sd = model.state_dict(); s.bk = {k: sd[k].detach().clone() for k in s.sh} | |
| model.load_state_dict({k: s.sh[k].to(sd[k].dtype) for k in s.sh}, strict=False) | |
| def restore(s, model): model.load_state_dict(s.bk, strict=False) | |
| class FMem: | |
| """Functional memory M_F: entries = {step, signature Sigma_F (dense logits on diagnostic set E_F), val score}.""" | |
| def __init__(s, diag_batch): s.b, s.entries = diag_batch, [] | |
| def add(s, model, step, score): | |
| model.eval() | |
| with ac(): lg = model(s.b).float() | |
| s.entries.append(dict(step=step, sig=lg.cpu().half(), score=score)) | |
| def anchor(s, model): | |
| e = random.choice(s.entries); old = e["sig"].to(s.b["allowed"].device).float() | |
| lp = F.log_softmax(model(s.b).float(), -1); p = F.softmax(old, -1); lpo = F.log_softmax(old, -1) | |
| return ((p * (lpo - lp)).masked_fill(~s.b["allowed"], 0.0)).sum(1).mean() | |
| class Controller: | |
| """Discovery controller: rho = d(val nll)/d(train nll) ~ share of fitting progress that generalises (proxy of P(B_G)/P(B_M)).""" | |
| def __init__(s): s.wd, s.rho, s.prev, s.LFRR = C.wd_new, None, None, 0.0 | |
| def update(s, rec): | |
| if s.prev is not None: | |
| dtr, dv = s.prev["tr_nll"] - rec["tr_nll"], s.prev["nll"] - rec["nll"] | |
| if dtr > 1e-3: | |
| r = float(np.clip(dv / dtr, -1, 2)); s.rho = r if s.rho is None else 0.5 * s.rho + 0.5 * r | |
| if s.rho is not None: | |
| if s.rho < 0.3: s.wd = min(C.wd_max, s.wd * 1.25) # memorising -> shape manifold harder | |
| elif s.rho > 0.8: s.wd = max(C.wd_min, s.wd * 0.9) # generalising -> relax | |
| s.prev = rec | |
| def perturb(model, sigma): | |
| with torch.no_grad(): | |
| for p in model.parameters(): | |
| if p.ndim >= 2: p.add_(torch.randn_like(p) * p.std() * sigma * C.noise_unit) | |
| def frr_test(model, hist, final_score, sigmas): | |
| """FRR = T_first / T_re. T_first: first logged step the main run reached tau. T_re: steps to re-reach tau after perturbing.""" | |
| snap = clone_state(model); tau = final_score - 0.01 | |
| T_first = next((h["step"] for h in hist if h["score"] >= tau), hist[-1]["step"]); res = {} | |
| for sg in sigmas: | |
| model.load_state_dict(snap); torch.manual_seed(int(sg * 100)); perturb(model, sg) | |
| v0 = fast_eval(model, "val")["score"]; trivial = v0 >= tau; T_re, hit = 1, trivial | |
| if not trivial: | |
| opt, scaler, rng = make_opt(model, C.lr_bb, C.lr_new, C.wd_new), make_scaler(), random.Random(999 + int(sg * 10)); T_re = C.frr_max_steps | |
| for s in range(1, C.frr_max_steps + 1): | |
| for g in opt.param_groups: g["lr"] = g["base_lr"] * min(1.0, s / 5) | |
| sgd_step(model, opt, scaler, to_device(build_batch(sample_exs(rng, C.bs), model.tk, model.tokz, model.cfg), dev)) | |
| if s % C.frr_every == 0 and fast_eval(model, "val")["score"] >= tau: T_re, hit = s, True; break | |
| res[sg] = dict(v0=v0, tau=tau, T_first=T_first, T_re=T_re, FRR=T_first / max(T_re, 1), L_FRR=max(T_re, 1) / max(T_first, 1), | |
| censored=(not hit), trivial=trivial) | |
| print(f" [FRR] sigma={sg}: perturbed score {v0:.3f} (tau {tau:.3f}) T_first={T_first} T_re={T_re}{'+' if not hit else ''} FRR={res[sg]['FRR']:.1f}") | |
| model.load_state_dict(snap); return res | |
| def adapt_curve(model, fam, eval_batches, steps, at): | |
| """Few-shot adaptation of a snapshot to a held-out family; returns {step: (acc, nll)}. Model is restored afterwards.""" | |
| snap = clone_state(model); opt, scaler, rng = make_opt(model, C.lr_bb, C.lr_new, 0.0), make_scaler(), random.Random(777); curve = {} | |
| for s in range(steps + 1): | |
| if s in at: | |
| r = stat(predict_res(model, eval_batches), float(model.post_T)); curve[s] = (r["score"], r["nll"]) | |
| if s == steps: break | |
| for g in opt.param_groups: g["lr"] = g["base_lr"] * min(1.0, (s + 1) / 3) | |
| b = to_device(build_batch([make_example(rng, [fam], "train", "any") for _ in range(C.probe_bs)], model.tk, model.tokz, model.cfg), dev) | |
| sgd_step(model, opt, scaler, b) | |
| model.load_state_dict(snap); return curve | |
| def meta_probe(model, step): | |
| out = [dict(step=step, fam=f, curve=adapt_curve(model, f, ST["held_test"][f], C.probe_steps, C.probe_at)) for f in ST["held_fams"]] | |
| msg = " | ".join(f"{o['fam']}: zs {o['curve'][0][0]:.2f} -> @10 {o['curve'][10][0]:.2f} -> @{C.probe_steps} {o['curve'][C.probe_steps][0]:.2f}" for o in out) | |
| print(f" [meta-probe @ {step}] {msg}"); return out | |
| # ----------------------------------------------------------------------------------------------- setup + train | |
| def setup(): | |
| random.seed(C.seed); np.random.seed(C.seed); torch.manual_seed(C.seed) | |
| load_all() | |
| m = Alpha(AlphaConfig(backbone=C.backbone)).to(dev).use_data_parallel() | |
| ST["model"] = m; ST["w"] = [math.sqrt(len(DATA[f]["rows"]["train"])) for f in ST["train_fams"]] | |
| tf, hf = ST["train_fams"], ST["held_fams"] | |
| fast = lambda split, lo, n: [make_example(random.Random(i * 31 + j), [f], split, "train", idx=i) for j, f in enumerate(tf) for i in range(lo, lo + n)] | |
| ST["fast_val"], ST["fast_train"] = prep(fast("val", 0, C.fast_n)), prep(fast("train", 0, C.fast_n)) | |
| ST["cal"] = prep([e for f in tf for e in val_examples(f, 0, C.n_cal, "train")]) | |
| ST["held_test"] = {f: prep(val_examples(f, 0, C.probe_test_n, "unseen")) for f in hf} | |
| rng = random.Random(4242); ST["diag"] = prep([make_example(rng, rng.sample(tf, min(3, len(tf)))) for _ in range(16)], bs=16)[0] | |
| n_all = sum(p.numel() for p in m.parameters()); print(f"params: {n_all/1e6:.1f}M | GPUs: {torch.cuda.device_count()} | train fams {tf} | held-out {hf}") | |
| def train(): | |
| m, hist = ST["model"], []; rng = random.Random(C.seed + 1) | |
| ctl, mem, ema = Controller(), FMem(ST["diag"]), EMA(m, C.ema) | |
| opt, scaler = make_opt(m, C.lr_bb, C.lr_new, ctl.wd), make_scaler() | |
| ST.update(ctl=ctl, mem=mem, hist=hist, probes=meta_probe(m, 0)); best, lam, loss_ema, t0, mid_done = -1, C.lam_mem, None, time.time(), False | |
| for step in range(1, C.steps + 1): | |
| prog = max(0.0, (step - C.warmup) / max(1, C.steps - C.warmup)) | |
| mult = min(1.0, step / C.warmup) * (0.1 + 0.9 * 0.5 * (1 + math.cos(math.pi * prog))) | |
| for g in opt.param_groups: | |
| g["lr"] = g["base_lr"] * mult | |
| if g["tag"] == "new": g["weight_decay"] = ctl.wd | |
| b = to_device(build_batch(sample_exs(rng, C.bs), m.tk, m.tokz, m.cfg), dev) | |
| nll, gn = sgd_step(m, opt, scaler, b, mem, lam, anchor=(step % C.anchor_every == 0)) | |
| ema.update(m, step); loss_ema = nll if loss_ema is None else 0.98 * loss_ema + 0.02 * nll | |
| if step % C.eval_every == 0 or step == C.steps: | |
| ema.apply(m); v, t = fast_eval(m, "val"), fast_eval(m, "train") | |
| rec = dict(step=step, score=v["score"], nll=v["nll"], ece=v["ece"], tr_score=t["score"], tr_nll=t["nll"], fam=v["per_acc"], | |
| wd=ctl.wd, lam=lam, gn=gn, loss=loss_ema, rho=ctl.rho, lr=opt.param_groups[0]["lr"]) | |
| hist.append(rec) | |
| if v["score"] > best: best = v["score"]; ST["best_state"] = clone_state(m); ST["best_step"] = step | |
| ctl.update(rec) | |
| if step % C.mem_every == 0: mem.add(m, step, v["score"]) | |
| if step % C.probe_every == 0: ST["probes"] += meta_probe(m, step) | |
| if not mid_done and step >= C.steps // 2: | |
| mid_done = True; r = frr_test(m, hist, v["score"], [1])[1]; ctl.LFRR = r["L_FRR"]; lam = C.lam_mem * (1 + min(ctl.LFRR, 3.0)) | |
| print(f" [L_FRR={ctl.LFRR:.3f}] -> anchor weight {lam:.3f}") | |
| ema.restore(m) | |
| print(f"step {step:5d} | val acc {v['score']:.3f} nll {v['nll']:.3f} ece {v['ece']:.3f} | train acc {t['score']:.3f} nll {t['nll']:.3f} | " | |
| f"rho {ctl.rho if ctl.rho is None else round(ctl.rho, 2)} wd {ctl.wd:.3f} lam {lam:.2f} gn {gn:.2f} | {time.time()-t0:.0f}s") | |
| m.load_state_dict(ST["best_state"]); print(f"loaded best EMA checkpoint (step {ST['best_step']})") | |
| # ----------------------------------------------------------------------------------------------- finalize + benchmark | |
| def finalize(): | |
| m, hist = ST["model"], ST["hist"]; m.eval() | |
| res = predict_res(m, ST["cal"]); grid = np.arange(0.5, 3.01, 0.02) | |
| nlls = [np.mean([-(lg / T - np.logaddexp.reduce(lg / T))[g] for _, lg, g in res]) for T in grid] | |
| T = float(grid[int(np.argmin(nlls))]); m.post_T.fill_(T); print(f"post-hoc temperature T={T:.2f}") | |
| best = max(h["score"] for h in hist); print("FRR sweep (basin test):"); ST["frr"] = frr_test(m, hist, best, C.sigmas) | |
| tau = min(h["nll"] for h in hist) + 0.05 | |
| Tv = next((h["step"] for h in hist if h["nll"] <= tau), hist[-1]["step"]); Tt = next((h["step"] for h in hist if h["tr_nll"] <= tau), hist[-1]["step"]) | |
| ST["DMR"] = dict(T_discover=Tv, T_memorize=Tt, DMR=Tv / max(Tt, 1)); print("DMR:", ST["DMR"]) | |
| # meta-grokking: T_D(n) = adaptation steps to reach 95% of the best adapted accuracy seen over all probes (per family) | |
| P, fams, mg = ST["probes"], ST["held_fams"], {} | |
| steps_ = sorted({p["step"] for p in P}) | |
| for f in fams: | |
| tau_f = 0.95 * max(p["curve"][C.probe_steps][0] for p in P if p["fam"] == f) | |
| for s in steps_: | |
| c = next(p for p in P if p["fam"] == f and p["step"] == s)["curve"] | |
| td = next((k for k in sorted(c) if c[k][0] >= tau_f), C.probe_steps * 2) | |
| mg.setdefault(s, {})[f] = dict(T_D=max(td, 1), zero_shot=c[0][0], acc10=c[10][0], final=c[C.probe_steps][0]) | |
| base = steps_[0] | |
| for s in steps_: | |
| g = [mg[base][f]["T_D"] / mg[s][f]["T_D"] for f in fams]; mg[s]["MGR"] = float(np.exp(np.mean(np.log(g)))) | |
| ST["meta"] = mg; print("meta-grokking MGR by checkpoint:", {s: round(mg[s]["MGR"], 2) for s in steps_}) | |
| def workflow_eval(n=300, k=4): | |
| m = ST["model"]; rng = random.Random(2024); exs = [] | |
| for _ in range(n): exs.append(make_example(rng, rng.sample(ST["train_fams"], min(k, len(ST["train_fams"]))), "val", "train", lo=C.n_cal)) | |
| m.eval(); qacc, allok, jc, jok = [], [], [], [] | |
| for i in range(0, n, 32): | |
| b = to_device(build_batch(exs[i:i + 32], m.tk, m.tokz, m.cfg), dev) | |
| with ac(): lg = m(b).float() / m.post_T | |
| per = {} | |
| for r, idx in enumerate(b["opt_lists"]): | |
| p = F.softmax(lg[r, idx], -1); ok = int(p.argmax()) == b["gold_pos"][r] | |
| qacc.append(ok); per.setdefault(b["q_ex"][r], []).append((ok, float(p.max()))) | |
| for v in per.values(): | |
| allok.append(all(o for o, _ in v)); jc.append(float(np.prod([c for _, c in v]))); jok.append(float(all(o for o, _ in v))) | |
| return dict(q_acc=float(np.mean(qacc)), all_correct=float(np.mean(allok)), joint_ece=ece_fn(jc, jok), k=k, n=n) | |
| def speed_bench(): | |
| m = ST["model"]; m.eval() | |
| state = dict(customer_ticket="My card was charged twice for the same order and support has not replied in three days. I want my money back.", | |
| account="plan: pro; tenure_months: 14; open_tickets: 2") | |
| qs = [dict(id="refund", question="Does the customer ask for a refund?", options=["no", "yes"]), | |
| dict(id="urgency", question="How urgent is the ticket?", options=["low", "medium", "high"], ordinal=True), | |
| dict(id="team", question="Which team should handle the ticket?", options=["billing", "technical", "sales", "account", "abuse"])] | |
| sync = (lambda: torch.cuda.synchronize()) if torch.cuda.is_available() else (lambda: None) | |
| t = time.perf_counter(); m.predict(state, qs); sync(); cold = (time.perf_counter() - t) * 1e3 | |
| for _ in range(10): m.predict(state, qs) | |
| ts = [] | |
| for _ in range(200): | |
| sync(); t = time.perf_counter(); m.predict(state, qs); sync(); ts.append((time.perf_counter() - t) * 1e3) | |
| rng = random.Random(1); exs = [make_example(rng, rng.sample(ST["train_fams"], 3)) for _ in range(64)] | |
| with torch.no_grad(): | |
| for _ in range(3): m(to_device(build_batch(exs, m.tk, m.tokz, m.cfg), dev)) | |
| sync(); t = time.perf_counter() | |
| for _ in range(10): | |
| b = to_device(build_batch(exs, m.tk, m.tokz, m.cfg), dev) | |
| with torch.no_grad(), ac(): m(b) | |
| sync(); e2e = 10 * 64 / (time.perf_counter() - t) | |
| return dict(latency_ms_p50=float(np.percentile(ts, 50)), latency_ms_p95=float(np.percentile(ts, 95)), cold_first_call_ms=cold, | |
| throughput_workflows_per_s_bs64=e2e, questions_per_workflow=3, device=torch.cuda.get_device_name(0) if torch.cuda.is_available() else "cpu") | |
| def llm_baseline(n_per=30, fams=("sst2", "ag_news", "emotion", "snli")): | |
| """Generative small-LLM baseline on the SAME k=1 questions (answers parsed from free text) -> accuracy + latency.""" | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| fams = [f for f in fams if f in DATA]; name = C.llm_baseline | |
| t = AutoTokenizer.from_pretrained(name); lm = AutoModelForCausalLM.from_pretrained(name, torch_dtype=torch.float16).to(dev).eval() | |
| m = ST["model"]; m.eval(); lat, ok_l, ok_a = [], [], [] | |
| for f in fams: | |
| for i in range(n_per): | |
| ex = make_example(random.Random(i), [f], "val", "train", idx=C.n_cal + i); q = ex["questions"][0] | |
| state = "\n".join(f"{k}: {v}" for k, v in ex["fields"]) | |
| msg = [{"role": "user", "content": f"{state}\n\nQuestion: {q['q']}\nOptions: {', '.join(q['options'])}\nReply with exactly one option and nothing else."}] | |
| ids = t.apply_chat_template(msg, add_generation_prompt=True, return_tensors="pt", return_dict=True).to(dev) | |
| torch.cuda.synchronize(); t0 = time.perf_counter() | |
| with torch.no_grad(): out = lm.generate(**ids, max_new_tokens=12, do_sample=False, pad_token_id=t.eos_token_id) | |
| torch.cuda.synchronize(); lat.append((time.perf_counter() - t0) * 1e3) | |
| txt = t.decode(out[0, ids["input_ids"].shape[1]:], skip_special_tokens=True).strip().lower() | |
| cand = [o for o in q["options"] if o.lower() in txt]; pred = max(cand, key=len) if cand else None | |
| ok_l.append(pred is not None and pred == q["options"][q["gold"]]) | |
| b = to_device(build_batch([ex], m.tk, m.tokz, m.cfg), dev) | |
| with torch.no_grad(), ac(): lg = m(b) | |
| ok_a.append(int(lg[0, b["opt_lists"][0]].argmax()) == b["gold_pos"][0]) | |
| del lm; torch.cuda.empty_cache() | |
| return dict(model=name, n=len(ok_l), llm_acc=float(np.mean(ok_l)), alpha_acc=float(np.mean(ok_a)), llm_latency_ms_p50=float(np.percentile(lat, 50)), | |
| llm_latency_ms_p95=float(np.percentile(lat, 95)), families=fams) | |
| def benchmark(run_llm=True): | |
| m = ST["model"]; m.eval(); T = float(m.post_T); R = dict(post_T=T, quality={}, calib={}) | |
| for wording in ("train", "unseen"): | |
| allr = [] | |
| for f in ST["train_fams"]: | |
| res = predict_res(m, prep(val_examples(f, C.n_cal, C.n_cal + C.n_test, wording))); allr += res | |
| s = stat(res, T)["per"][f]; R["quality"].setdefault(wording, {})[f] = {k: s[k] for k in ("acc", "nll", "ece", "brier", "n")} | |
| s0, s1 = stat(allr, 1.0), stat(allr, T) | |
| R["quality"][wording]["_macro"] = dict(acc=s1["score"], nll=s1["nll"], ece=s1["ece"], brier=s1["brier"]) | |
| R["calib"][wording] = dict(ece_before_T=s0["ece"], ece_after_T=s1["ece"], conf=s1["micro"]["conf"], corr=s1["micro"]["corr"]) | |
| R["held_out"] = {} | |
| for f in ST["held_fams"]: | |
| eb = prep(val_examples(f, C.n_cal, C.n_cal + C.n_test, "unseen")) | |
| c = adapt_curve(m, f, eb, C.probe_steps, C.probe_at); R["held_out"][f] = dict(zero_shot_acc=c[0][0], adapted_acc={k: v[0] for k, v in c.items()}, n=len(val_examples(f, C.n_cal, C.n_cal + C.n_test, "unseen"))) | |
| R["workflow"] = workflow_eval() | |
| R["speed"] = speed_bench() | |
| R["type_safety"] = dict(schema_violations=0, note="outputs are a masked softmax over the declared options; an out-of-schema answer cannot be produced") | |
| if run_llm: | |
| try: R["llm_baseline"] = llm_baseline() | |
| except Exception as e: R["llm_baseline"] = dict(error=f"{type(e).__name__}: {str(e)[:200]}") | |
| R["frr"] = ST["frr"]; R["dmr"] = ST["DMR"]; R["meta"] = {str(k): v for k, v in ST["meta"].items()} | |
| ST["R"] = R | |
| print(json.dumps({k: v for k, v in R.items() if k not in ("calib", "meta")}, indent=1, default=float)[:6000]); return R | |
| def plots(path="alpha_report.png"): | |
| import matplotlib; import matplotlib.pyplot as plt | |
| H, R = ST["hist"], ST["R"]; st = [h["step"] for h in H] | |
| fig, ax = plt.subplots(2, 4, figsize=(22, 9)); ax = ax.ravel() | |
| ax[0].plot(st, [h["score"] for h in H], label="val acc"); ax[0].plot(st, [h["tr_score"] for h in H], label="train acc"); ax[0].set_title("Learning curve (EMA weights)"); ax[0].legend() | |
| ax[1].plot(st, [h["nll"] for h in H], label="val nll"); ax[1].plot(st, [h["tr_nll"] for h in H], label="train nll"); ax[1].axvline(ST["DMR"]["T_discover"], color="g", ls=":"); ax[1].axvline(ST["DMR"]["T_memorize"], color="r", ls=":") | |
| ax[1].set_title(f"DMR = {ST['DMR']['DMR']:.2f} (T_discover/T_memorize)"); ax[1].legend() | |
| ax[2].plot(st, [h["wd"] for h in H], label="wd (new modules)"); ax[2].plot(st, [h["rho"] if h["rho"] is not None else 0 for h in H], label="rho"); ax[2].plot(st, [h["gn"] for h in H], label="grad norm"); ax[2].legend(); ax[2].set_title("Discovery controller") | |
| sg = list(R["frr"]); ax[3].bar([str(s) for s in sg], [min(R["frr"][s]["FRR"], 100) for s in sg]); ax[3].axhline(1, color="k", ls="--"); ax[3].set_yscale("log"); ax[3].set_title("FRR vs perturbation sigma (basin test)") | |
| mg = ST["meta"]; ss = sorted(k for k in mg); ax[4].plot(ss, [mg[s]["MGR"] for s in ss], "o-"); ax[4].axhline(1, color="k", ls="--"); ax[4].set_title("Meta-grokking rate MGR (held-out families)") | |
| for f in ST["held_fams"]: ax[5].plot(ss, [mg[s][f]["zero_shot"] for s in ss], "o-", label=f"{f} zero-shot"); ax[5].plot(ss, [mg[s][f]["acc10"] for s in ss], "s--", label=f"{f} +10 steps") | |
| ax[5].legend(fontsize=7); ax[5].set_title("Unseen families: zero-shot / after 10 steps") | |
| conf, corr = np.array(R["calib"]["train"]["conf"]), np.array(R["calib"]["train"]["corr"]); e = np.linspace(0, 1, 11) | |
| xs, ys = [], [] | |
| for lo, hi in zip(e[:-1], e[1:]): | |
| mk = (conf > lo) & (conf <= hi) | |
| if mk.sum() > 5: xs.append(conf[mk].mean()); ys.append(corr[mk].mean()) | |
| ax[6].plot([0, 1], [0, 1], "k--"); ax[6].plot(xs, ys, "o-"); ax[6].set_title(f"Reliability (ECE {R['calib']['train']['ece_after_T']:.3f})") | |
| fams = ST["train_fams"]; x = np.arange(len(fams)) | |
| ax[7].bar(x - .2, [R["quality"]["train"][f]["acc"] for f in fams], .4, label="seen wording"); ax[7].bar(x + .2, [R["quality"]["unseen"][f]["acc"] for f in fams], .4, label="unseen wording") | |
| ax[7].set_xticks(x); ax[7].set_xticklabels(fams, rotation=60); ax[7].legend(); ax[7].set_title("Test accuracy per family") | |
| plt.tight_layout(); plt.savefig(path, dpi=110); plt.show() | |
| # ----------------------------------------------------------------------------------------------- publish | |
| def get_token(): | |
| t = os.environ.get("HF_TOKEN") | |
| if not t: | |
| try: | |
| from kaggle_secrets import UserSecretsClient | |
| t = UserSecretsClient().get_secret("HF_TOKEN") | |
| except Exception: pass | |
| return t | |
| def model_card(R): | |
| q, sp, wf, h = R["quality"], R["speed"], R["workflow"], R["held_out"] | |
| rows = "\n".join(f"| {f} | {q['train'][f]['acc']:.3f} | {q['unseen'][f]['acc']:.3f} | {q['train'][f]['ece']:.3f} |" for f in ST["train_fams"]) | |
| hrows = "\n".join(f"| {f} | {v['zero_shot_acc']:.3f} | {v['adapted_acc'][10]:.3f} | {v['adapted_acc'][C.probe_steps]:.3f} |" for f, v in h.items()) | |
| llm = R.get("llm_baseline", {}); llm_s = (f"Qwen2.5-0.5B-Instruct (generative, same {llm['n']} questions): acc {llm['llm_acc']:.3f}, p50 latency {llm['llm_latency_ms_p50']:.0f} ms " | |
| f"vs Alpha acc {llm['alpha_acc']:.3f}, p50 {sp['latency_ms_p50']:.1f} ms.") if "llm_acc" in llm else "LLM baseline not run." | |
| fr = " ".join(f"sigma={s}: FRR {v['FRR']:.1f}{'*' if v['censored'] else ''}" for s, v in R["frr"].items()) | |
| return f"""--- | |
| license: apache-2.0 | |
| base_model: {C.backbone} | |
| tags: [system-one, structured-output, calibrated, classification, parallel-decoding] | |
| --- | |
| # Alpha (System-One-style typed decision model) | |
| Alpha takes unstructured / structured state plus **typed questions with declared options** and returns calibrated probabilities for | |
| **all questions in one parallel forward pass**. It cannot generate strings and cannot answer outside the declared options | |
| (masked softmax -> schema violations are impossible by construction). Option embeddings are independent of the state, so a schema can be | |
| compiled once and cached; cardinality is limited only by memory. | |
| Architecture: `{C.backbone}` encoder (state + question slots with `<mask>` readout tokens) -> 2-layer **Slot Reasoner** (set transformer, squared-ReLU, | |
| zero-init residuals) -> biaffine/cosine scorer against option embeddings with per-question learned temperature + post-hoc temperature (T={R['post_T']:.2f}). | |
| Training: {C.steps} steps x {C.bs} workflow-style examples (1-4 questions over named fields, mixed from {len(ST['train_fams'])} public datasets with paraphrased | |
| questions/option wordings), proper-scoring-rule loss (NLL + Brier + ordinal term), EMA weights, functional-memory anchor, discovery controller. | |
| Held-out families (never trained on): {', '.join(ST['held_fams'])}. | |
| ## Results (test splits; see results.json, alpha_report.png) | |
| | family | acc (seen wording) | acc (unseen wording) | ECE | | |
| |---|---|---|---| | |
| {rows} | |
| Workflow eval ({wf['k']} questions per query): per-question acc {wf['q_acc']:.3f}, all-correct {wf['all_correct']:.3f}, joint-confidence ECE {wf['joint_ece']:.3f}. | |
| Held-out task families (unseen schema wording): zero-shot / after 10 / after {C.probe_steps} gradient steps | |
| | family | zero-shot | +10 steps | +{C.probe_steps} steps | | |
| |---|---|---|---| | |
| {hrows} | |
| Speed on {sp['device']}: p50 {sp['latency_ms_p50']:.1f} ms, p95 {sp['latency_ms_p95']:.1f} ms for a 3-question query (local, no network); {sp['throughput_workflows_per_s_bs64']:.0f} workflows/s at batch 64. | |
| {llm_s} | |
| Framework diagnostics: DMR={R['dmr']['DMR']:.2f}; FRR by perturbation: {fr} (* = not re-reached within budget); MGR by checkpoint in results.json. | |
| ## Honest limits | |
| * This is a small research model trained on public classification datasets. **It is not a replacement for, and has not been compared against, TypeSafe's Jev** | |
| (a closed API; its claims were not verified here). Do not read the latency comparison as a Jev comparison. | |
| * Narrow task coverage (sentiment/topic/intent/NLI/paraphrase/spam/rating); free-form reasoning is out of scope by design. | |
| * Calibration is measured in-distribution; shift will degrade it. | |
| ## Usage | |
| ```python | |
| from huggingface_hub import snapshot_download; import sys | |
| p = snapshot_download("{C.repo_id}"); sys.path.insert(0, p) | |
| from alpha_s1 import Alpha | |
| m = Alpha.from_pretrained(p, device="cuda").eval() | |
| print(m.predict({{"ticket": "I was charged twice, refund me now."}}, | |
| [{{"id": "refund", "question": "Does the customer want a refund?", "options": ["no", "yes"]}}])) | |
| ``` | |
| """ | |
| def publish(out_dir="alpha_out"): | |
| from huggingface_hub import HfApi | |
| tok = get_token(); assert tok, "Set a Kaggle secret named HF_TOKEN (Add-ons > Secrets) or env var HF_TOKEN." | |
| R = ST["R"]; m = ST["model"]; m.save_pretrained(out_dir) | |
| shutil.copy("alpha_s1.py", out_dir); shutil.copy("alpha_train.py", out_dir) | |
| if os.path.exists("alpha_report.png"): shutil.copy("alpha_report.png", out_dir) | |
| slim = {k: v for k, v in R.items() if k != "calib"}; json.dump(slim, open(f"{out_dir}/results.json", "w"), indent=1, default=float) | |
| json.dump(dict(hist=ST["hist"], probes=ST["probes"]), open(f"{out_dir}/training_log.json", "w"), default=float) | |
| open(f"{out_dir}/README.md", "w").write(model_card(R)) | |
| api = HfApi(token=tok); api.create_repo(C.repo_id, exist_ok=True, private=False) | |
| api.upload_folder(folder_path=out_dir, repo_id=C.repo_id, commit_message="Alpha: System-One-style typed decision model") | |
| print(f"published -> https://huggingface.co/{C.repo_id}") | |
| def demo(): | |
| m = ST["model"]; m.eval() | |
| state = dict(review="The battery died after two days and support never answered. Total waste of money.", account="plan: pro; tenure_months: 14") | |
| qs = [dict(id="sentiment", question="Is the review positive or negative?", options=["negative", "positive"]), | |
| dict(id="stars", question="How many stars does the review deserve?", options=["1 star", "2 stars", "3 stars", "4 stars", "5 stars"], ordinal=True), | |
| dict(id="topic", question="What is the review about?", options=["hardware", "billing", "support", "shipping"])] | |
| print(json.dumps(m.predict(state, qs), indent=1)) | |
| def chat(): | |
| m = ST["model"]; print("Alpha structured console. Empty text to quit.") | |
| while True: | |
| text = input("state text> ").strip() | |
| if not text: break | |
| q = input("question> ").strip(); opts = [o.strip() for o in input("options (comma separated)> ").split(",") if o.strip()] | |
| r = m.predict(dict(input=text), [dict(id="q", question=q, options=opts)])["q"] | |
| print(f" answer={r['answer']} confidence={r['confidence']:.3f} certainty={r['certainty']:.3f}\n probs={ {k: round(v, 3) for k, v in r['probs'].items()} }") | |