BurnyCoder's picture
Publish the verified paper-review protocol before model inference
387a4cf verified
Raw History Blame Contribute Delete
3.93 kB
"""Generate all empirical values from event and per-paper result files."""
from __future__ import annotations
import numpy as np
from .data import load
from .search import run_name
def crossed_interval(left, right, labels, draws=10000, seed=1701):
"""Resample common seeds and common paper columns within their decision strata."""
diffs = np.asarray(left, dtype=float) - np.asarray(right, dtype=float)
labels = np.asarray(labels)
if (
diffs.ndim != 2
or len(labels) != diffs.shape[1]
or set(labels) != {"accept", "reject"}
):
raise ValueError("Expected paired binary predictions on both decision classes")
rng = np.random.default_rng(seed)
strata = [np.flatnonzero(labels == label) for label in ("accept", "reject")]
samples = []
for start in range(0, draws, 200):
batch = min(200, draws - start)
rows = rng.integers(0, len(diffs), size=(batch, len(diffs)))
cols = np.concatenate(
[rng.choice(s, size=(batch, len(s)), replace=True) for s in strata], axis=1
)
samples.extend(
diffs[rows[:, :, None], cols[:, None, :]].mean(axis=(1, 2)) * 100
)
low, high = np.quantile(samples, [0.025, 0.975])
return dict(
mean_pp=float(diffs.mean() * 100),
low=float(low),
high=float(high),
pairs=diffs.shape[0],
per_seed_pp=(diffs.mean(axis=1) * 100).tolist(),
)
def summary(project):
"""Include in-progress searches but count only terminal evaluations for inference."""
runs = [load(p) for p in sorted((project / "results" / "runs").glob("*.json"))]
evals = [
load(p) for p in sorted((project / "results" / "evaluations").glob("*.json"))
]
table = []
for data in evals:
rows = data["rows"]
by_route = {}
for route in sorted({r["route"] for r in rows}):
sub = [r for r in rows if r["route"] == route]
by_route[route] = dict(
n=len(sub), accuracy=sum(r["correct"] for r in sub) / len(sub)
)
table.append(
dict(
name=data["name"],
split=data["split"],
arm=data["arm"],
n=len(rows),
correct=sum(r["correct"] for r in rows),
complete=data["complete"],
routes=by_route,
compute=data.get("compute", {}),
)
)
panels = {e["name"]: e for e in evals if e["split"] == "test" and e["complete"]}
contrasts = {}
for suffix in ("", "-tokens"):
left, right, seeds = [], [], []
labels = None
for seed in range(1, 9):
keys = [
run_name("confirm", seed, a) + suffix for a in ("editable", "native")
]
if not all(k in panels for k in keys):
continue
a, b = [sorted(panels[k]["rows"], key=lambda x: x["id"]) for k in keys]
if [(r["id"], r["label"]) for r in a] != [(r["id"], r["label"]) for r in b]:
raise ValueError("Mismatched paired papers")
labels = [r["label"] for r in a]
left.append([r["correct"] for r in a])
right.append([r["correct"] for r in b])
seeds.append(seed)
if seeds:
outcome = crossed_interval(left, right, labels)
outcome["seeds"] = seeds
outcome["interpretation"] = (
"incomplete, descriptive"
if len(seeds) < 8
else (
"supports superiority"
if outcome["low"] > 0
else "evidence against superiority"
if outcome["high"] <= 0
else "inconclusive"
)
)
contrasts["matched_tokens" if suffix else "primary"] = outcome
return runs, evals, table, contrasts