Buckets:
| """Paired-bootstrap 95% intervals for the final report's headline comparisons, and the final hypothesis table. | |
| CUDA_VISIBLE_DEVICES="" python analysis/final_stats.py # CPU only; numpy, no torch / vllm | |
| Reads the saved predictions only (nothing is re-run) and writes results/final_stats.json and results/final_stats.md. | |
| Method | |
| - Point estimates are recomputed exactly as runs/v2/metrics_v2.py does (same eval / calib files, same helpers from | |
| metrics.py and common.py) and checked against runs/v2/results/metrics_{v2,rl}.json and results/FINAL_REPORT.md. | |
| - Temperatures (fast readout and post-chain, per question type) are fitted once on the calibration items and then held | |
| fixed: the calibration set is not resampled. | |
| - Paired bootstrap: questions are resampled with replacement and every model is scored on the same resample. A | |
| resample is represented by its multiplicity vector w (w_i = times item i was drawn), so accuracy is w.ok / n, ECE | |
| is sum_bins |sum_i w_i (ok_i - conf_i)| / sum_i w_i, routing sends the lowest-margin round(f n) draws to thinking, | |
| and AUROC counts weighted positive-negative pairs (ties half). Each weighted statistic is checked against the plain | |
| formula on explicitly resampled arrays before use. Percentile 95% intervals. | |
| - S1MB: benchmarks (not judgments) are the resampled unit, stratified by task for Task Avg, so the intervals cover | |
| which benchmarks happen to be in the suite, not item noise inside a benchmark. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import sys | |
| sys.dont_write_bytecode = True # importing the project's modules must not touch its __pycache__ | |
| os.environ["CUDA_VISIBLE_DEVICES"] = "" | |
| import numpy as np | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| sys.path.insert(0, ROOT) | |
| from common import auroc, ece, read_jsonl, softmax # noqa: E402 | |
| from metrics import calibrated, fit_temps, gap_recovered, margin, routing_curve # noqa: E402 | |
| B = 10_000 # bootstrap resamples per item set | |
| CHUNK = 1_000 | |
| SEED = 20261101 | |
| V2 = f"{ROOT}/runs/v2" | |
| P0 = f"{ROOT}/results/predictions" # v1-era: base and v1 (as run_v2.sh / run_rl_eval.sh pass them) | |
| P = f"{V2}/results/predictions" | |
| READOUT_FILES = {"base": f"{P0}/readout_base.jsonl", "v1": f"{P0}/readout_joint.jsonl", | |
| "v2": f"{P}/readout_v2_joint.jsonl", "rl60": f"{P}/readout_rl60.jsonl", "rl40": f"{P}/readout_rl40.jsonl", | |
| "abl_gold": f"{P}/readout_abl_gold.jsonl", "abl_mix": f"{P}/readout_abl_mix.jsonl"} | |
| REASON_FILES = {"base": f"{P0}/reason_base.jsonl", "v1": f"{P0}/reason_joint.jsonl", "v2": f"{P}/reason_v2_joint.jsonl", | |
| "rl60": f"{P}/reason_rl60.jsonl", "rl40": f"{P}/reason_rl40.jsonl"} | |
| STORED_NAME = {"base": "base", "v1": "v1_joint", "v2": "v2_joint", "rl60": "rl60", "rl40": "rl40", | |
| "abl_gold": "abl_gold", "abl_mix": "abl_mix"} | |
| MAIN = ["base", "v1", "v2", "rl60", "rl40"] | |
| IN_DIST = {"eval_proc", "eval_ts"} | |
| SEEN_PREFIXES = ("laya--ag-news, arc-choice, arct-, babi-nli, banking77, civil-comments, clinc, contract-nli, corr2cause, " | |
| "creak, dbpedia, defeasible-nli, ethics-, ethos, fol-nli, go-emotions, hans, hh-rlhf, logical-entailment, " | |
| "lonli, massive, mtop, openbookqa, paws, poem-sentiment, qasper, quartz, ruletaker, scicite, scitail, scone, " | |
| "sdoh-nli, snli, spartqa, stepgame, temporal-nli, tracie, winowhy, wiqa, patent-similarity").split(", ") | |
| # Numbers as printed in results/FINAL_REPORT.md (percent or plain), for the reproduction check. | |
| REPORT = { | |
| "fast_acc_all": {"base": 62.7, "v1": 69.5, "v2": 72.7, "rl60": 72.5, "rl40": 72.7}, | |
| "fast_acc_subset": {"base": 61.8, "v1": 70.3, "v2": 76.0, "rl60": 75.9, "rl40": 76.0}, | |
| "think_acc": {"base": 84.0, "v1": 84.4, "v2": 81.2, "rl60": 83.1, "rl40": 83.6}, | |
| "loop_rate": {"base": 0.9, "v1": 1.4, "v2": 5.1, "rl60": 0.2, "rl40": 2.0}, | |
| "routed30_acc": {"base": 73.0, "v1": 78.0, "v2": 81.8, "rl60": 81.8, "rl40": 82.0}, | |
| "routed30_tokens": {"base": 1069, "v1": 1166, "v2": 1749, "rl60": 729, "rl40": 1079}, | |
| "gap_recovered": {"rl60": 0.79, "v1": 0.52}, | |
| "fast_ece": {"rl60": 0.030}, | |
| "post_ece": {"rl60": 0.057, "v2": 0.043}, | |
| "abl_acc": {"abl_mix": 70.5, "abl_gold": 71.7}, | |
| "abl_ece": {"abl_mix": 0.047, "abl_gold": 0.031}, | |
| "s1mb_task_avg": {"rl60": 47.9, "v1_joint": 43.5, "v2_2b_ro": 37.3, "base": 32.8, "base_2b": 18.1}, | |
| "s1mb_seen_mean": {"base": 44.5, "rl60": 52.8}, | |
| "s1mb_unseen_mean": {"base": 33.0, "rl60": 50.4}, | |
| } | |
| # ------------------------------------------------------------------ weighted statistics (w = resample multiplicities) | |
| def w_mean(x): | |
| x = np.asarray(x, float) | |
| return lambda W: W @ x / W.sum(1) | |
| def w_ece(conf, correct, incl, bins=15): | |
| """ECE (common.ece, 15 bins) over the included items; per-model inclusion (post-chain readout may be missing).""" | |
| n = len(conf) | |
| conf = np.where(incl, np.asarray(conf, float), 0.0) | |
| b = np.minimum((conf * bins).astype(int), bins - 1) | |
| M = np.zeros((n, bins)) | |
| M[np.arange(n)[incl], b[incl]] = (np.asarray(correct, float) - conf)[incl] | |
| cnt = incl.astype(float) | |
| return lambda W: np.abs(W @ M).sum(1) / (W @ cnt) | |
| def w_routed(r_ok, t_ok, route_score, f, tokens=None): | |
| """metrics.routing_curve at one fraction: the lowest-score round(f n) draws take the thinking outcome.""" | |
| order = np.argsort(route_score, kind="mergesort") | |
| r, t = np.asarray(r_ok, float)[order], np.asarray(t_ok, float)[order] | |
| tk = None if tokens is None else np.asarray(tokens, float)[order] | |
| n = len(r) | |
| m = int(round(f * n)) | |
| def fn(W): | |
| Ws = W[:, order] | |
| routed = np.clip(m - (np.cumsum(Ws, 1) - Ws), 0, Ws) | |
| if tk is not None: | |
| return routed @ tk / n | |
| return (routed @ t + (Ws - routed) @ r) / n | |
| return fn | |
| def w_gap(r_ok, t_ok, route_score, f=0.3): | |
| acc_f, acc_0, acc_1 = w_routed(r_ok, t_ok, route_score, f), w_mean(r_ok), w_mean(t_ok) | |
| def fn(W): | |
| a0, a1 = acc_0(W), acc_1(W) | |
| d = a1 - a0 | |
| return np.where(np.abs(d) > 1e-12, (acc_f(W) - a0) / np.where(np.abs(d) > 1e-12, d, 1.0), np.nan) | |
| return fn | |
| def w_auroc(score, label, incl): | |
| """common.auroc (P(random positive outranks random negative), ties half) with multiplicity weights.""" | |
| score = np.asarray(score, float) | |
| order = np.argsort(score, kind="mergesort") | |
| s = score[order] | |
| starts = np.r_[0, np.nonzero(np.diff(s))[0] + 1] | |
| pos = (np.asarray(label, bool) & incl)[order].astype(float) | |
| neg = (~np.asarray(label, bool) & incl)[order].astype(float) | |
| def fn(W): | |
| Ws = W[:, order] | |
| pg = np.add.reduceat(Ws * pos, starts, axis=1) | |
| ng = np.add.reduceat(Ws * neg, starts, axis=1) | |
| below = np.cumsum(ng, 1) - ng | |
| return (pg * (below + 0.5 * ng)).sum(1) / (pg.sum(1) * ng.sum(1)) | |
| return fn | |
| def diff(fa, fb): | |
| return lambda W: fa(W) - fb(W) | |
| def weights_from_idx(idx, n): | |
| c = idx.shape[0] | |
| return np.bincount((idx + (np.arange(c) * n)[:, None]).ravel(), minlength=c * n).reshape(c, n).astype(float) | |
| def run_boot(n, fns, seed): | |
| rng = np.random.default_rng(seed) | |
| out = {k: [] for k in fns} | |
| for start in range(0, B, CHUNK): | |
| W = weights_from_idx(rng.integers(0, n, size=(min(CHUNK, B - start), n)), n) | |
| for k, fn in fns.items(): | |
| out[k].append(fn(W)) | |
| return {k: np.concatenate(v) for k, v in out.items()} | |
| def point(fn, n): | |
| return float(fn(np.ones((1, n)))[0]) | |
| def summ(pt, vals, scale=1.0): | |
| v = vals[~np.isnan(vals)] | |
| lo, hi = np.percentile(v, [2.5, 97.5]) | |
| return {"point": pt * scale, "ci95": [float(lo) * scale, float(hi) * scale], "excludes_0": bool(lo > 0 or hi < 0), | |
| "boot_share_le_0": float(np.mean(v <= 0)), "n_boot": int(len(v))} | |
| # ------------------------------------------------------------------ loading | |
| def load_readout(path): | |
| return {r["id"]: r["lp"] for r in read_jsonl(path)} | |
| def load_reason(path): | |
| out = {} | |
| with open(path, encoding="utf-8") as f: | |
| for line in f: | |
| if line.strip(): | |
| r = json.loads(line) | |
| out[r["id"]] = {k: r[k] for k in ("answer", "looped", "tokens", "post_lp")} | |
| return out | |
| def main() -> None: | |
| ev = read_jsonl(f"{V2}/data/eval.jsonl") | |
| cal = read_jsonl(f"{V2}/data/calib.jsonl") | |
| subset = set(open(f"{V2}/data/eval_subset_ids.txt").read().split()) | |
| stored = {**json.load(open(f"{V2}/results/metrics_v2.json"))["readout"], **json.load(open(f"{V2}/results/metrics_rl.json"))["readout"]} | |
| stored_rl = json.load(open(f"{V2}/results/metrics_rl.json")) | |
| stored_v2 = json.load(open(f"{V2}/results/metrics_v2.json")) | |
| n = len(ev) | |
| gold = np.array([it["gold"] for it in ev]) | |
| repro, checks = [], [] | |
| def rep(label, ours, stored_value=None, report_value=None, tol=None): | |
| row = {"what": label, "recomputed": ours} | |
| if stored_value is not None: | |
| row["metrics_json"] = stored_value | |
| row["matches_json"] = bool(abs(ours - stored_value) < 1e-9) | |
| if report_value is not None: | |
| row["final_report"] = report_value | |
| row["matches_report"] = bool(abs(ours - report_value) <= tol + 1e-9) | |
| repro.append(row) | |
| # ---------------------------------------------------------------- fast readout, all 5,442 eval questions | |
| R = {} | |
| for name, path in READOUT_FILES.items(): | |
| ro = load_readout(path) | |
| temps = fit_temps(cal, [ro[it["id"]] for it in cal]) | |
| st = stored[STORED_NAME[name]]["temps"] | |
| assert all(abs(temps[k] - st[k]) < 1e-9 for k in temps), (name, temps, st) | |
| probs = calibrated(ev, [ro[it["id"]] for it in ev], temps) | |
| ok = np.array([int(np.argmax(p)) == g for p, g in zip(probs, gold)]) | |
| conf = np.array([p.max() for p in probs]) | |
| R[name] = {"temps": temps, "probs": probs, "ok": ok, "conf": conf, "margin": np.array([margin(p) for p in probs])} | |
| a, e = float(ok.mean()), ece(conf, ok) | |
| rep(f"fast readout accuracy, all 5,442: {name}", 100 * a, 100 * stored[STORED_NAME[name]]["acc"]["all"], | |
| REPORT["fast_acc_all"].get(name), 0.05) | |
| rep(f"fast readout ECE15: {name}", e, stored[STORED_NAME[name]]["ece15"], | |
| REPORT["fast_ece"].get(name) if name in REPORT["fast_ece"] else REPORT["abl_ece"].get(name), 0.0005) | |
| if name in REPORT["abl_acc"]: | |
| rep(f"ablation readout accuracy: {name}", 100 * a, None, REPORT["abl_acc"][name], 0.05) | |
| allones = np.ones(n, bool) | |
| ind = np.array([it["set"] in IN_DIST for it in ev]) | |
| # ---------------------------------------------------------------- thinking mode on the items every reasoner answered | |
| T = {name: load_reason(path) for name, path in REASON_FILES.items()} | |
| common_ids = set.intersection(*[set(r) for r in T.values()]) | |
| ev_r = [it for it in ev if it["id"] in common_ids] | |
| assert {it["id"] for it in ev_r} == subset and len(ev_r) == stored_rl["reasoning_items"] == 1498 | |
| m = len(ev_r) | |
| pos = {it["id"]: i for i, it in enumerate(ev)} | |
| sub_idx = np.array([pos[it["id"]] for it in ev_r]) | |
| gold_r = gold[sub_idx] | |
| TH = {} | |
| for name, rr in T.items(): | |
| rows = [rr[it["id"]] for it in ev_r] | |
| has = np.array([r["answer"] is not None for r in rows]) | |
| t_ok = np.array([r["answer"] == g for r, g in zip(rows, gold_r)]) | |
| looped = np.array([bool(r["looped"]) for r in rows]) | |
| toks = np.array([r["tokens"] for r in rows], float) | |
| cal_rows = [(it, rr[it["id"]]) for it in cal if it["id"] in rr and rr[it["id"]]["post_lp"] is not None] | |
| pt = fit_temps([it for it, _ in cal_rows], [r["post_lp"] for _, r in cal_rows]) | |
| sname = STORED_NAME[name] | |
| assert all(abs(pt[k] - stored_rl["reasoning"][sname]["post_temps"][k]) < 1e-9 for k in pt), name | |
| has_post = np.array([r["post_lp"] is not None for r in rows]) | |
| pconf, pok = np.zeros(m), np.zeros(m, bool) | |
| for i in np.nonzero(has_post)[0]: | |
| p = softmax(np.asarray(rows[i]["post_lp"]), pt[ev_r[i]["qtype"]]) | |
| pconf[i], pok[i] = p.max(), int(np.argmax(p)) == gold_r[i] | |
| # the pair: this model's readout with its own reasoner (metrics_v2 'pairs') | |
| P = [R[name]["probs"][j] for j in sub_idx] | |
| r_ok = R[name]["ok"][sub_idx] | |
| agree = np.array([int(np.argmax(p)) == r["answer"] for p, r in zip(P, rows)]) & has | |
| t_route = np.where(has, t_ok, r_ok) | |
| mg = R[name]["margin"][sub_idx] | |
| unsure = np.array([1 - p.max() for p in P]) | |
| TH[name] = {"has": has, "t_ok": t_ok, "looped": looped, "toks": toks, "pconf": pconf, "pok": pok, "has_post": has_post, | |
| "r_ok": r_ok, "agree": agree, "t_route": t_route, "margin": mg, "unsure": unsure} | |
| sr = stored_rl["reasoning"][sname] | |
| rep(f"thinking accuracy, 1,498: {name}", 100 * float(t_ok.mean()), 100 * sr["acc"]["all"], REPORT["think_acc"][name], 0.05) | |
| rep(f"looped chains %: {name}", 100 * float(looped.mean()), sr["loops_per_1k"] / 10, REPORT["loop_rate"][name], 0.05) | |
| rep(f"post-chain ECE15: {name}", ece(pconf[has_post], pok[has_post]), sr["post_ece15"], REPORT["post_ece"].get(name), 0.0005) | |
| curve = routing_curve(r_ok, t_route, toks, mg) | |
| f3 = next(r for r in curve if abs(r["f"] - 0.3) < 1e-9) | |
| sp = stored_rl["pairs"][sname] | |
| sf3 = next(r for r in sp["routing"] if abs(r["f"] - 0.3) < 1e-9) | |
| rep(f"fast readout accuracy on the 1,498 subset: {name}", 100 * float(r_ok.mean()), 100 * sp["routing"][0]["accuracy"], | |
| REPORT["fast_acc_subset"][name], 0.05) | |
| rep(f"routed 30% accuracy: {name}", 100 * f3["accuracy"], 100 * sf3["accuracy"], REPORT["routed30_acc"][name], 0.05) | |
| rep(f"routed 30% thinking tokens per question: {name}", f3["mean_tokens"], sf3["mean_tokens"], REPORT["routed30_tokens"][name], 1.0) | |
| rep(f"gap recovered at f=0.3: {name}", gap_recovered(curve), sp["gap_recovered_f0.3"], REPORT["gap_recovered"].get(name), 0.005) | |
| rep(f"disagreement AUROC: {name}", auroc(unsure[has], ~agree[has]), sp["dis_auroc"]) | |
| # ---------------------------------------------------------------- check the weighted statistics on explicit resamples | |
| rng = np.random.default_rng(1) | |
| for _ in range(3): | |
| ix = rng.integers(0, m, m) | |
| W = weights_from_idx(ix[None, :], m) | |
| x = TH["rl60"] | |
| checks.append(("acc", float(w_mean(x["t_ok"])(W)[0]), float(x["t_ok"][ix].mean()))) | |
| hp = x["has_post"][ix] | |
| checks.append(("ece", float(w_ece(x["pconf"], x["pok"], x["has_post"])(W)[0]), ece(x["pconf"][ix][hp], x["pok"][ix][hp]))) | |
| cv = routing_curve(x["r_ok"][ix], x["t_route"][ix], x["toks"][ix], x["margin"][ix]) | |
| checks.append(("routed30", float(w_routed(x["r_ok"], x["t_route"], x["margin"], 0.3)(W)[0]), cv[3]["accuracy"])) | |
| checks.append(("tokens30", float(w_routed(x["r_ok"], x["t_route"], x["margin"], 0.3, x["toks"])(W)[0]), cv[3]["mean_tokens"])) | |
| checks.append(("gap", float(w_gap(x["r_ok"], x["t_route"], x["margin"])(W)[0]), gap_recovered(cv))) | |
| h = x["has"][ix] | |
| checks.append(("auroc", float(w_auroc(x["unsure"], ~x["agree"], x["has"])(W)[0]), auroc(x["unsure"][ix][h], ~x["agree"][ix][h]))) | |
| ix = rng.integers(0, n, n) | |
| W = weights_from_idx(ix[None, :], n) | |
| checks.append(("ece_all", float(w_ece(R["rl60"]["conf"], R["rl60"]["ok"], allones)(W)[0]), ece(R["rl60"]["conf"][ix], R["rl60"]["ok"][ix]))) | |
| worst = max(abs(a - b) for _, a, b in checks) | |
| assert worst < 1e-9, checks | |
| # ---------------------------------------------------------------- bootstrap: fast readout (5,442 questions) | |
| acc = {k: w_mean(R[k]["ok"]) for k in READOUT_FILES} | |
| ecef = {k: w_ece(R[k]["conf"], R[k]["ok"], allones) for k in READOUT_FILES} | |
| fast_fns = {f"acc_{k}": acc[k] for k in READOUT_FILES} | {f"ece_{k}": ecef[k] for k in READOUT_FILES} | |
| fast_pairs = [("rl60", "base"), ("rl60", "v1"), ("v2", "v1"), ("rl60", "v2"), ("rl40", "rl60"), ("abl_mix", "abl_gold")] | |
| for a, b in fast_pairs: | |
| fast_fns[f"acc_{a}-{b}"] = diff(acc[a], acc[b]) | |
| fast_fns["ece_abl_mix-abl_gold"] = diff(ecef["abl_mix"], ecef["abl_gold"]) | |
| fast_fns["ece_rl60-v2"] = diff(ecef["rl60"], ecef["v2"]) | |
| bs = run_boot(n, fast_fns, SEED + 1) | |
| fast = {k: summ(point(fn, n), bs[k], 100 if k.startswith("acc") else 1) for k, fn in fast_fns.items()} | |
| # in-distribution items (procedural + tasksource) for H1, as metrics.py's H1 | |
| nin = int(ind.sum()) | |
| in_fns = {f"acc_{k}": w_mean(R[k]["ok"][ind]) for k in ("abl_mix", "abl_gold")} | |
| in_fns["acc_abl_mix-abl_gold"] = diff(in_fns["acc_abl_mix"], in_fns["acc_abl_gold"]) | |
| bs_in = run_boot(nin, in_fns, SEED + 2) | |
| in_dist = {k: summ(point(fn, nin), bs_in[k], 100) for k, fn in in_fns.items()} | |
| # the out-of-distribution sets (OpenDecisions + LocalLLaMA), and in-dist minus OOD (strata resampled independently) | |
| ood = ~ind | |
| nood = int(ood.sum()) | |
| ood_fns = {f"acc_{k}": w_mean(R[k]["ok"][ood]) for k in ("abl_mix", "abl_gold")} | |
| ood_fns["acc_abl_mix-abl_gold"] = diff(ood_fns["acc_abl_mix"], ood_fns["acc_abl_gold"]) | |
| bs_ood = run_boot(nood, ood_fns, SEED + 5) | |
| out_dist = {k: summ(point(fn, nood), bs_ood[k], 100) for k, fn in ood_fns.items()} | |
| k = "acc_abl_mix-abl_gold" | |
| in_dist["in_minus_ood_" + k] = summ(in_dist[k]["point"] / 100 - out_dist[k]["point"] / 100, bs_in[k] - bs_ood[k], 100) | |
| # ---------------------------------------------------------------- bootstrap: thinking subset (1,498 questions) | |
| th_fns = {} | |
| for k in MAIN: | |
| x = TH[k] | |
| th_fns[f"think_{k}"] = w_mean(x["t_ok"]) | |
| th_fns[f"loop_{k}"] = w_mean(x["looped"]) | |
| th_fns[f"postece_{k}"] = w_ece(x["pconf"], x["pok"], x["has_post"]) | |
| th_fns[f"routed30_{k}"] = w_routed(x["r_ok"], x["t_route"], x["margin"], 0.3) | |
| th_fns[f"tokens30_{k}"] = w_routed(x["r_ok"], x["t_route"], x["margin"], 0.3, x["toks"]) | |
| th_fns[f"gap_{k}"] = w_gap(x["r_ok"], x["t_route"], x["margin"]) | |
| th_fns[f"disauroc_{k}"] = w_auroc(x["unsure"], ~x["agree"], x["has"]) | |
| th_fns[f"fastsub_{k}"] = w_mean(x["r_ok"]) | |
| th_pairs = {"think": [("rl60", "v2"), ("rl40", "rl60"), ("rl60", "base"), ("v2", "base"), ("rl40", "base")], | |
| "loop": [("rl60", "v2"), ("rl60", "base")], | |
| "postece": [("rl60", "v2")], | |
| "routed30": [("rl60", "v2"), ("rl60", "v1"), ("rl60", "base"), ("rl40", "rl60")], | |
| "tokens30": [("rl60", "v2"), ("rl60", "v1")], | |
| "gap": [("rl60", "v1")], | |
| "disauroc": [("rl60", "base"), ("rl60", "v1"), ("v2", "base")]} | |
| for stat, prs in th_pairs.items(): | |
| for a, b in prs: | |
| th_fns[f"{stat}_{a}-{b}"] = diff(th_fns[f"{stat}_{a}"], th_fns[f"{stat}_{b}"]) | |
| bs = run_boot(m, th_fns, SEED + 3) | |
| pct = ("think", "loop", "routed30", "fastsub") | |
| thinking = {k: summ(point(fn, m), bs[k], 100 if k.split("_")[0] in pct else 1) for k, fn in th_fns.items()} | |
| # ---------------------------------------------------------------- S1MB, benchmarks as the unit | |
| s = json.load(open(f"{ROOT}/results/s1mb.json")) | |
| cat = json.load(open(f"{ROOT}/data/s1mb/english-v1.json"))["benchmarks"] | |
| tasks = {b: json.load(open(f"{ROOT}/data/s1mb/benchmarks/{b}.json"))["task"] for b in cat} | |
| adj = {k: np.array([s["ours"][k]["calibrated"]["adjusted"][b] for b in cat], float) for k in s["ours"]} | |
| assert not any(np.isnan(v).any() for v in adj.values()) | |
| task_of = np.array([tasks[b] for b in cat]) | |
| TASKS = ("noul", "choice", "score") | |
| tidx = {t: np.nonzero(task_of == t)[0] for t in TASKS} | |
| for k, ref in REPORT["s1mb_task_avg"].items(): | |
| ta = 100 * np.mean([adj[k][tidx[t]].mean() for t in TASKS]) | |
| rep(f"S1MB Task Avg: {k}", float(ta), 100 * s["ours"][k]["calibrated"]["task_avg"], ref, 0.05) | |
| rng = np.random.default_rng(SEED + 4) | |
| draws = {t: rng.integers(0, len(tidx[t]), size=(B, len(tidx[t]))) for t in TASKS} | |
| def task_means(v): # (B, 3) resampled task means, and the point estimate | |
| return (np.stack([v[tidx[t]][draws[t]].mean(1) for t in TASKS], 1), np.array([v[tidx[t]].mean() for t in TASKS])) | |
| s1 = {} | |
| for a, b in [("rl60", "base"), ("rl60", "v1_joint"), ("v2_2b_ro", "base_2b"), ("v1_joint", "base")]: | |
| bt, pt_ = task_means(adj[a] - adj[b]) | |
| s1[f"task_avg_{a}-{b}"] = summ(float(pt_.mean()), bt.mean(1), 100) | |
| for j, t in enumerate(TASKS): | |
| s1[f"{t}_{a}-{b}"] = summ(float(pt_[j]), bt[:, j], 100) | |
| seen = np.array([any(b.startswith(p) for p in SEEN_PREFIXES) for b in cat]) | |
| assert seen.sum() == 41 | |
| gain = adj["rl60"] - adj["base"] | |
| sidx, uidx = np.nonzero(seen)[0], np.nonzero(~seen)[0] | |
| ds, du = rng.integers(0, len(sidx), (B, len(sidx))), rng.integers(0, len(uidx), (B, len(uidx))) | |
| for nm, ix, d in (("seen", sidx, ds), ("unseen", uidx, du)): | |
| for k in ("base", "rl60"): | |
| s1[f"{nm}_mean_{k}"] = summ(float(adj[k][ix].mean()), adj[k][ix][d].mean(1), 100) | |
| s1[f"{nm}_gain"] = summ(float(gain[ix].mean()), gain[ix][d].mean(1), 100) | |
| rep("S1MB seen mean adjusted: base", s1["seen_mean_base"]["point"], None, REPORT["s1mb_seen_mean"]["base"], 0.05) | |
| rep("S1MB seen mean adjusted: rl60", s1["seen_mean_rl60"]["point"], None, REPORT["s1mb_seen_mean"]["rl60"], 0.05) | |
| rep("S1MB unseen mean adjusted: base", s1["unseen_mean_base"]["point"], None, REPORT["s1mb_unseen_mean"]["base"], 0.05) | |
| rep("S1MB unseen mean adjusted: rl60", s1["unseen_mean_rl60"]["point"], None, REPORT["s1mb_unseen_mean"]["rl60"], 0.05) | |
| s1["unseen_minus_seen_gain"] = summ(float(gain[uidx].mean() - gain[sidx].mean()), gain[uidx][du].mean(1) - gain[sidx][ds].mean(1), 100) | |
| # task mix differs (seen: 25 choice / 15 noul / 1 score; unseen: 32 / 44 / 20): the same contrast inside each task | |
| comp = {} | |
| for t in ("noul", "choice"): | |
| si, ui = sidx[task_of[sidx] == t], uidx[task_of[uidx] == t] | |
| a_, b_ = rng.integers(0, len(si), (B, len(si))), rng.integers(0, len(ui), (B, len(ui))) | |
| comp[t] = (gain[ui][b_].mean(1) - gain[si][a_].mean(1), float(gain[ui].mean() - gain[si].mean())) | |
| s1[f"unseen_minus_seen_gain_{t}_only"] = summ(comp[t][1], comp[t][0], 100) | {"n_seen": int(len(si)), "n_unseen": int(len(ui))} | |
| s1["unseen_minus_seen_gain_task_matched"] = summ((comp["noul"][1] + comp["choice"][1]) / 2, (comp["noul"][0] + comp["choice"][0]) / 2, 100) | |
| s1["split_counts"] = {"seen": {t: int(((task_of == t) & seen).sum()) for t in TASKS}, "unseen": {t: int(((task_of == t) & ~seen).sum()) for t in TASKS}} | |
| s1["seen_benchmarks"] = [b for b, x in zip(cat, seen) if x] | |
| # ---------------------------------------------------------------- hypotheses for the final models | |
| H = {} | |
| h1 = in_dist["acc_abl_mix-abl_gold"] | |
| H["H1"] = {"test": "V2 ablations (same 12K questions): votes-mixed readout minus gold-only readout, in-distribution accuracy " | |
| "(procedural + tasksource, n=%d), pts" % nin, | |
| **h1, "all_eval": fast["acc_abl_mix-abl_gold"], "rule": "CI lower bound > 0", "confirmed": h1["ci95"][0] > 0} | |
| h2 = {k: thinking[f"disauroc_{k}"] for k in ("rl60-base", "rl60-v1", "v2-base")} | |
| for v in h2.values(): | |
| v["ok"] = bool(v["point"] >= 0.03 and v["ci95"][0] > 0) | |
| v["ci_lower_ge_0.03"] = bool(v["ci95"][0] >= 0.03) | |
| H["H2"] = {"test": "disagreement AUROC (readout 1 - max p vs its own reasoner's answer, 1,498), rl60 minus base and minus v1", | |
| "by_comparison": h2, "levels": {k: thinking[f"disauroc_{k}"]["point"] for k in MAIN}, | |
| "rule": "delta >= +0.03 and CI lower bound > 0 for both rl60 comparisons (metrics.py rule)", | |
| "confirmed": bool(h2["rl60-base"]["ok"] and h2["rl60-v1"]["ok"])} | |
| h3 = thinking["think_rl60-base"] | |
| H["H3"] = {"test": "trained reasoner (rl60) minus base reasoner, thinking accuracy on 1,498, pts", **h3, | |
| "v2_minus_base": thinking["think_v2-base"], "rl40_minus_base": thinking["think_rl40-base"], | |
| "rule": "point >= -1.0 pts (metrics.py rule); stricter non-inferiority: CI lower bound >= -1.0", | |
| "confirmed": bool(h3["point"] >= -1.0), "non_inferiority_shown": bool(h3["ci95"][0] >= -1.0), | |
| "v2_confirmed": bool(thinking["think_v2-base"]["point"] >= -1.0)} | |
| h4 = thinking["gap_rl60"] | |
| H["H4"] = {"test": "rl60 gap recovered at f=0.3 (lowest-margin routing, 1,498)", **h4, | |
| "rule": "point >= 0.80 (metrics.py rule)", "confirmed": bool(h4["point"] >= 0.8), | |
| "ci_contains_0.80": bool(h4["ci95"][0] <= 0.8 <= h4["ci95"][1])} | |
| mism = [r for r in repro if r.get("matches_json") is False or r.get("matches_report") is False] | |
| out = {"config": {"n_boot": B, "seed": SEED, "ci": "percentile 95%", "eval_items": n, "thinking_items": m, "in_dist_items": nin, | |
| "ood_items": nood, | |
| "calib_items": len(cal), "s1mb_benchmarks": len(cat), "weighted_stat_check_max_abs_err": worst, | |
| "eval_file": "runs/v2/data/eval.jsonl", "calib_file": "runs/v2/data/calib.jsonl (identical to data/calib.jsonl)", | |
| "readout_files": {k: os.path.relpath(v, ROOT) for k, v in READOUT_FILES.items()}, | |
| "reason_files": {k: os.path.relpath(v, ROOT) for k, v in REASON_FILES.items()}, | |
| "temperatures": {k: R[k]["temps"] for k in R}}, | |
| "reproduction": {"all_match": not mism, "mismatches": mism, "rows": repro}, | |
| "fast_readout": fast, "in_distribution": in_dist, "out_of_distribution": out_dist, "thinking_subset": thinking, | |
| "s1mb": s1, "hypotheses": H} | |
| json.dump(out, open(f"{ROOT}/results/final_stats.json", "w"), indent=1) | |
| write_md(out) | |
| print(json.dumps({"reproduction_mismatches": mism, "weighted_check": worst, | |
| "H": {k: v["confirmed"] for k, v in H.items()}}, indent=1)) | |
| # ------------------------------------------------------------------ markdown | |
| def write_md(o) -> None: | |
| F, TH, S, H = o["fast_readout"], o["thinking_subset"], o["s1mb"], o["hypotheses"] | |
| def pts(d, nd=1, sign=True): | |
| f = f"{{:+.{nd}f}}" if sign else f"{{:.{nd}f}}" | |
| return f"{f.format(d['point'])} [{f.format(d['ci95'][0])}, {f.format(d['ci95'][1])}]" | |
| def yn(d): | |
| return "yes" if d["excludes_0"] else "**no**" | |
| L = ["# Final statistics: paired-bootstrap intervals and hypotheses", "", | |
| f"{o['config']['n_boot']:,} paired bootstrap resamples of questions (seed {o['config']['seed']}), percentile 95% CI; every model " | |
| "scored on the same resample. Temperatures fixed at their calibration-fitted values. Single training seed throughout: " | |
| "the intervals cover question sampling only, not training variance. Script `analysis/final_stats.py`; all numbers in " | |
| "`results/final_stats.json`.", ""] | |
| rp = o["reproduction"] | |
| L += [f"**Reproduction:** {len(rp['rows'])} point estimates recomputed from the saved predictions; " | |
| + ("all match metrics_v2/metrics_rl.json exactly and the FINAL_REPORT figures to the printed precision." | |
| if rp["all_match"] else f"{len(rp['mismatches'])} mismatch(es): " + | |
| "; ".join(f"{r['what']} {r['recomputed']:.4f} vs report {r.get('final_report')}" for r in rp["mismatches"])) | |
| + " (Only rounding nit: the report's 1,069 thinking tokens/question for base at 30% is 1,069.7, truncated.)", ""] | |
| L += ["## Fast readout, 5,442 questions (accuracy, points)", "", "| Comparison | Difference [95% CI] | Excludes 0 |", "|---|---|---|"] | |
| for k, lab in [("rl60-base", "V2+RL (rl60) - base"), ("rl60-v1", "rl60 - v1"), ("v2-v1", "V2 - v1"), | |
| ("rl60-v2", "rl60 - V2"), ("rl40-rl60", "rl40 - rl60")]: | |
| L.append(f"| {lab} | {pts(F['acc_' + k], 2)} | {yn(F['acc_' + k])} |") | |
| L += ["", "## Thinking mode, 1,498 questions", "", "| Comparison | Difference [95% CI] | Excludes 0 |", "|---|---|---|"] | |
| for k, lab in [("think_rl60-v2", "Thinking accuracy, rl60 - V2 (pts)"), ("think_rl40-rl60", "Thinking accuracy, rl40 - rl60 (pts)"), | |
| ("think_rl60-base", "Thinking accuracy, rl60 - base (pts)"), ("think_v2-base", "Thinking accuracy, V2 - base (pts)"), | |
| ("loop_rl60-v2", "Looped-chain rate, rl60 - V2 (pts)"), | |
| ("routed30_rl60-v2", "Routed 30%, rl60 - V2 (pts)"), ("routed30_rl60-v1", "Routed 30%, rl60 - v1 (pts)"), | |
| ("routed30_rl60-base", "Routed 30%, rl60 - base (pts)"), | |
| ("tokens30_rl60-v2", "Thinking tokens/question at 30%, rl60 - V2")]: | |
| nd = 0 if k.startswith("tokens") else 1 | |
| L.append(f"| {lab} | {pts(TH[k], nd)} | {yn(TH[k])} |") | |
| L.append(f"| Gap recovered at 30%, rl60 (level) | {pts(TH['gap_rl60'], 3, False)} | (vs 0.80 target: CI {'contains' if H['H4']['ci_contains_0.80'] else 'excludes'} 0.80) |") | |
| L.append(f"| Gap recovered at 30%, rl60 - v1 | {pts(TH['gap_rl60-v1'], 3)} | {yn(TH['gap_rl60-v1'])} |") | |
| L += ["", "## Calibration (ECE, 15 bins)", "", "| Quantity | Estimate [95% CI] | Excludes 0 |", "|---|---|---|", | |
| f"| Fast-readout ECE, rl60 (level, 5,442) | {pts(F['ece_rl60'], 3, False)} | |", | |
| f"| Post-chain ECE, rl60 (level) / V2 (level) | {pts(TH['postece_rl60'], 3, False)} / {pts(TH['postece_v2'], 3, False)} | |", | |
| f"| Post-chain ECE, rl60 - V2 | {pts(TH['postece_rl60-v2'], 3)} | {yn(TH['postece_rl60-v2'])} |", | |
| f"| Fast-readout ECE, rl60 - V2 (5,442) | {pts(F['ece_rl60-v2'], 3)} | {yn(F['ece_rl60-v2'])} |", | |
| "", "ECE is biased upward under resampling (bin noise only adds), so intervals on ECE levels sit mostly above the point " | |
| "estimate; differences between models are much less affected.", | |
| "", "## V2 ablations, same 12K training questions (votes-mixed `abl_mix` vs gold-only `abl_gold`)", "", | |
| "| Quantity | abl_mix - abl_gold [95% CI] | Excludes 0 |", "|---|---|---|", | |
| f"| Accuracy, all 5,442 (pts) | {pts(F['acc_abl_mix-abl_gold'], 2)} | {yn(F['acc_abl_mix-abl_gold'])} |", | |
| f"| Accuracy, in-distribution (procedural + tasksource, {o['config']['in_dist_items']:,}) | {pts(o['in_distribution']['acc_abl_mix-abl_gold'], 2)} | {yn(o['in_distribution']['acc_abl_mix-abl_gold'])} |", | |
| f"| Accuracy, out-of-distribution (OpenDecisions + LocalLLaMA, {o['config']['ood_items']:,}) | {pts(o['out_of_distribution']['acc_abl_mix-abl_gold'], 2)} | {yn(o['out_of_distribution']['acc_abl_mix-abl_gold'])} |", | |
| f"| In-distribution effect minus OOD effect | {pts(o['in_distribution']['in_minus_ood_acc_abl_mix-abl_gold'], 2)} | {yn(o['in_distribution']['in_minus_ood_acc_abl_mix-abl_gold'])} |", | |
| f"| ECE, all 5,442 | {pts(F['ece_abl_mix-abl_gold'], 3)} | {yn(F['ece_abl_mix-abl_gold'])} |"] | |
| c = S["split_counts"] | |
| L += ["", "## S1MB (137 benchmarks; benchmarks resampled, stratified by task; adjusted score x100)", "", | |
| "| Comparison | Task Avg diff [95% CI] | Yes/no | Choice | Score | Task Avg excludes 0 |", "|---|---|---|---|---|---|"] | |
| for a, b in [("rl60", "base"), ("rl60", "v1_joint"), ("v2_2b_ro", "base_2b")]: | |
| k = f"{a}-{b}" | |
| L.append(f"| {a} - {b} | {pts(S['task_avg_' + k])} | {pts(S['noul_' + k])} | {pts(S['choice_' + k])} | {pts(S['score_' + k])} | {yn(S['task_avg_' + k])} |") | |
| L += ["", f"Seen vs unseen source (seen: {sum(c['seen'].values())} benchmarks, {c['seen']['choice']} choice / {c['seen']['noul']} yes-no / " | |
| f"{c['seen']['score']} score; unseen: {sum(c['unseen'].values())}, {c['unseen']['choice']} / {c['unseen']['noul']} / {c['unseen']['score']}); " | |
| "plain mean over benchmarks, base -> rl60:", "", | |
| "| Split | base | rl60 | Gain [95% CI] |", "|---|---|---|---|"] | |
| for nm in ("seen", "unseen"): | |
| L.append(f"| {nm} | {S[nm + '_mean_base']['point']:.1f} | {S[nm + '_mean_rl60']['point']:.1f} | {pts(S[nm + '_gain'])} |") | |
| L += [f"| unseen gain - seen gain | | | {pts(S['unseen_minus_seen_gain'])} |", | |
| f"| same, within yes/no only ({S['unseen_minus_seen_gain_noul_only']['n_seen']} vs {S['unseen_minus_seen_gain_noul_only']['n_unseen']}) | | | {pts(S['unseen_minus_seen_gain_noul_only'])} |", | |
| f"| same, within choice only ({S['unseen_minus_seen_gain_choice_only']['n_seen']} vs {S['unseen_minus_seen_gain_choice_only']['n_unseen']}) | | | {pts(S['unseen_minus_seen_gain_choice_only'])} |", | |
| f"| same, task-matched (mean of the yes/no and choice contrasts) | | | {pts(S['unseen_minus_seen_gain_task_matched'])} |", | |
| "", "Seen holds 1 score benchmark against 20 unseen, so the task-matched row (score left out) is the composition-fair " | |
| "comparison."] | |
| h2 = H["H2"]["by_comparison"] | |
| L += ["", "## Hypotheses (final models)", "", "| # | Test | Result [95% CI] | Rule | Confirmed |", "|---|---|---|---|---|", | |
| f"| H1 | votes-mixed vs gold-only readout (V2 ablations), in-distribution accuracy | {pts(H['H1'])} pts (all eval {pts(H['H1']['all_eval'])}) | CI > 0 | {'yes' if H['H1']['confirmed'] else 'no'} |", | |
| f"| H2 | disagreement AUROC, rl60 vs base / vs v1 (need +0.03) | vs base {pts(h2['rl60-base'], 3)}; vs v1 {pts(h2['rl60-v1'], 3)} | delta >= 0.03 and CI > 0 | {'yes' if H['H2']['confirmed'] else 'no'} |", | |
| f"| H3 | trained reasoner not worse than base: rl60 - base thinking | {pts(H['H3'])} pts (V2 - base {pts(H['H3']['v2_minus_base'])}) | point >= -1 pt | " | |
| f"{'yes' if H['H3']['confirmed'] else 'no'} (rl60); {'yes' if H['H3']['v2_confirmed'] else 'no'} (V2) |", | |
| f"| H4 | gap recovered at f=0.3, rl60 (need 80%) | {pts(H['H4'], 3, False)} | point >= 0.80 | {'yes' if H['H4']['confirmed'] else 'no'} |"] | |
| L += ["", "Rules are the ones metrics.py applied to v1 (H1 CI > 0; H2 delta >= 0.03 and CI > 0; H3 point >= -1 pt; H4 point >= 0.80). " | |
| f"Stricter readings: H2's lower bound clears 0.03 vs base ({h2['rl60-base']['ci95'][0]:.3f}) but not vs v1 " | |
| f"({h2['rl60-v1']['ci95'][0]:.3f}); H3 non-inferiority (lower bound >= -1 pt) is " | |
| f"{'shown' if H['H3']['non_inferiority_shown'] else 'not shown'}; H4's interval contains 0.80, so the miss is not " | |
| "distinguishable from meeting the target. H2 here compares whole recipes (each readout against its own reasoner), not " | |
| "votes vs gold only as in v1; the gold-only V2 ablation has no thinking-mode run."] | |
| L += ["", "## Findings", ""] + findings(o) | |
| open(f"{ROOT}/results/final_stats.md", "w").write("\n".join(L) + "\n") | |
| def findings(o): | |
| F, TH, S, H = o["fast_readout"], o["thinking_subset"], o["s1mb"], o["hypotheses"] | |
| I, OD = o["in_distribution"], o["out_of_distribution"] | |
| def ci(d, nd=1): | |
| return f"{d['point']:+.{nd}f} [{d['ci95'][0]:+.{nd}f}, {d['ci95'][1]:+.{nd}f}]" | |
| def v(d, yes="holds", no="not established (CI includes 0)"): | |
| return yes if d["excludes_0"] else no | |
| return [ | |
| f"- Fast readout gains are real: rl60 - base {ci(F['acc_rl60-base'])}, rl60 - v1 {ci(F['acc_rl60-v1'])}, " | |
| f"V2 - v1 {ci(F['acc_v2-v1'])} pts: {v(F['acc_rl60-base'])} / {v(F['acc_rl60-v1'])} / {v(F['acc_v2-v1'])}.", | |
| f"- RL and the readout: rl60 - V2 {ci(F['acc_rl60-v2'], 2)}: {v(F['acc_rl60-v2'], 'RL changed the readout', 'no detectable change')}; " | |
| f"rl40 - rl60 {ci(F['acc_rl40-rl60'], 2)}: {v(F['acc_rl40-rl60'], 'the extra 20 high-rate steps cost a small but detectable 0.3 pts', 'no detectable change')}.", | |
| f"- RL's thinking gain over V2 {ci(TH['think_rl60-v2'])} pts {v(TH['think_rl60-v2'])}; rl40 - rl60 {ci(TH['think_rl40-rl60'])} " | |
| f"{v(TH['think_rl40-rl60'], 'differs', 'is within noise')}.", | |
| f"- Thinking vs base: V2 {ci(TH['think_v2-base'])} is {v(TH['think_v2-base'], 'a real loss', 'within noise')}; " | |
| f"rl60 {ci(TH['think_rl60-base'])} is {v(TH['think_rl60-base'], 'a real loss', 'within noise, but a loss of up to 2.6 pts is not ruled out')}.", | |
| f"- Loops: rl60 - V2 looped-chain rate {ci(TH['loop_rl60-v2'])} pts {v(TH['loop_rl60-v2'])}; rl60 also loops less than base " | |
| f"({ci(TH['loop_rl60-base'])}, {v(TH['loop_rl60-base'], 'CI excludes 0', 'CI includes 0')}).", | |
| f"- Routed 30%: rl60 - V2 {ci(TH['routed30_rl60-v2'])} pts is a tie at {TH['tokens30_rl60-v2']['point']:+.0f} " | |
| f"[{TH['tokens30_rl60-v2']['ci95'][0]:+.0f}, {TH['tokens30_rl60-v2']['ci95'][1]:+.0f}] thinking tokens/question; " | |
| f"rl60 - v1 {ci(TH['routed30_rl60-v1'])} {v(TH['routed30_rl60-v1'])}.", | |
| f"- Gap recovered: rl60 {TH['gap_rl60']['point']:.3f} [{TH['gap_rl60']['ci95'][0]:.3f}, {TH['gap_rl60']['ci95'][1]:.3f}], " | |
| f"0.80 {'inside' if H['H4']['ci_contains_0.80'] else 'outside'} the CI; rl60 - v1 {ci(TH['gap_rl60-v1'], 3)} {v(TH['gap_rl60-v1'])}.", | |
| f"- Post-chain calibration: rl60 - V2 ECE {ci(TH['postece_rl60-v2'], 3)}: " | |
| f"{v(TH['postece_rl60-v2'], 'RL worsened post-chain calibration', 'the RL calibration loss is suggestive, not established')}; " | |
| f"fast-readout ECE rl60 - V2 {ci(F['ece_rl60-v2'], 3)}, {v(F['ece_rl60-v2'], 'differs', 'no change')}.", | |
| f"- Votes-mix vs gold-only (V2 ablations): accuracy {ci(F['acc_abl_mix-abl_gold'])} pts on all eval " | |
| f"({v(F['acc_abl_mix-abl_gold'], 'votes-mixing lowered accuracy', 'within noise')}), driven by OOD {ci(OD['acc_abl_mix-abl_gold'])} " | |
| f"while in-distribution is {ci(I['acc_abl_mix-abl_gold'])} ({v(I['acc_abl_mix-abl_gold'], 'holds', 'within noise')}); " | |
| f"the in-dist vs OOD contrast {ci(I['in_minus_ood_acc_abl_mix-abl_gold'])} {v(I['in_minus_ood_acc_abl_mix-abl_gold'])}; " | |
| f"ECE {ci(F['ece_abl_mix-abl_gold'], 3)} {v(F['ece_abl_mix-abl_gold'], 'holds (votes-mixing worsened eval calibration)')}.", | |
| f"- S1MB Task Avg: rl60 - base {ci(S['task_avg_rl60-base'])}, rl60 - v1 {ci(S['task_avg_rl60-v1_joint'])}, 2B readout-only - 2B base " | |
| f"{ci(S['task_avg_v2_2b_ro-base_2b'])}: {v(S['task_avg_rl60-base'])} / {v(S['task_avg_rl60-v1_joint'])} / {v(S['task_avg_v2_2b_ro-base_2b'])}; " | |
| f"rl60's yes/no gain over v1 {ci(S['noul_rl60-v1_joint'])} {v(S['noul_rl60-v1_joint'])}.", | |
| f"- S1MB seen vs unseen: gain {ci(S['seen_gain'])} seen, {ci(S['unseen_gain'])} unseen; unseen gained more by " | |
| f"{ci(S['unseen_minus_seen_gain'])} ({v(S['unseen_minus_seen_gain'])}), task-matched {ci(S['unseen_minus_seen_gain_task_matched'])} " | |
| f"({v(S['unseen_minus_seen_gain_task_matched'])}); neither single-task contrast excludes 0 on its own.", | |
| f"- Hypotheses: H1 {'confirmed' if H['H1']['confirmed'] else 'not confirmed'}; H2 {'confirmed' if H['H2']['confirmed'] else 'not confirmed'}; " | |
| f"H3 {'confirmed' if H['H3']['confirmed'] else 'not confirmed'} for rl60 (by 0.07 pts on the point rule; " | |
| f"non-inferiority {'shown' if H['H3']['non_inferiority_shown'] else 'not shown'}), " | |
| f"{'confirmed' if H['H3']['v2_confirmed'] else 'not confirmed'} for V2; H4 {'confirmed' if H['H4']['confirmed'] else 'not confirmed'}.", | |
| ] | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 39.4 kB
- Xet hash:
- e89d4e53d4b883534728cf7492ff89f70d4b6f5cf3445e9c837b1d3a810a5cf6
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.