BurnyCoder's picture
Publish the verified paper-review protocol before model inference
387a4cf verified
Raw History Blame Contribute Delete
6.21 kB
"""Freeze reviewer identities independently of the sealed paper outcomes."""
from __future__ import annotations
import time
from .data import canonical, digest, dump, load, read_split
from .inference import now
from .search import run_name
from .workspace import ROOT
def roster():
"""Order primary pairs before secondary controls; fork is attached to seed one."""
return (
[(s, a) for s in range(1, 9) for a in ("editable", "native")]
+ [(s, a) for s in range(1, 6) for a in ("uniform", "handcrafted")]
+ [(1, "fork-native")]
)
def frozen_entry(run, endpoint=None):
"""The selected source and validation checkpoint fully identify a reviewer."""
ep = endpoint or run["endpoints"][-1]
workspace = next(n["workspace"] for n in run["nodes"] if n["id"] == ep["node"])
if digest(workspace["reviewer"]) != ep["key"]:
raise ValueError("Endpoint program identity failed verification")
return dict(
name=run["name"],
seed=run["seed"],
arm=run["arm"],
source=workspace["reviewer"],
key=ep["key"],
n=ep["n"],
compute=ep["compute"],
)
def freeze(project, allow_incomplete=False):
"""Build the entire endpoint manifest before opening any test manuscript."""
path = project / "results" / "freeze.json"
if path.exists():
return load(path)
budget = load(project / "results" / "budget.json")
stop_file = project / "results" / "time-stop.json"
if allow_incomplete and time.time() < budget["deadline"] and not stop_file.exists():
raise ValueError(
"An incomplete cohort requires the registered deadline or timing-reserve stopping record"
)
entries = {
"root": dict(
name="root",
seed=0,
arm="root",
source=ROOT["reviewer"],
key=digest(ROOT["reviewer"]),
n=0,
)
}
runs = {}
missing = []
for seed, arm in roster():
name = run_name("confirm", seed, arm)
source = project / "results" / "runs" / (name + ".json")
data = load(source) if source.exists() else {}
if data.get("complete") and data["budget"] == budget["budget"]:
runs[name] = data
entries[name] = frozen_entry(data)
else:
missing.append(name)
if missing and not allow_incomplete:
raise ValueError("Pending confirmatory endpoints: " + ", ".join(missing))
# Token sensitivity uses a cutoff fixed without consulting any test predictions.
for seed in range(1, 9):
pair = [run_name("confirm", seed, a) for a in ("editable", "native")]
if not all(name in runs for name in pair):
continue
def tokens(ep):
return ep["compute"]["input"] + ep["compute"]["output"]
cutoff = min(tokens(runs[name]["endpoints"][-1]) for name in pair)
for name in pair:
eligible = [ep for ep in runs[name]["endpoints"] if tokens(ep) <= cutoff]
entry = (
frozen_entry(runs[name], eligible[-1])
if eligible
else dict(entries["root"], seed=seed, arm=runs[name]["arm"])
)
entries[name + "-tokens"] = dict(entry, cutoff=cutoff)
manifest = dict(
frozen_at=now(),
split_sha=load(project / "results" / "split-manifest.json")["splits"]["test"][
"sha256"
],
budget=budget["budget"],
missing=missing,
entries=entries,
entries_sha=digest(canonical(entries)),
)
dump(path, manifest)
return manifest
def score(project, domain, split, name, entry=None):
"""Score only frozen test programs; interrupted item scoring resumes idempotently."""
if split == "test":
manifest = load(project / "results" / "freeze.json")
if (
digest(canonical(manifest["entries"])) != manifest["entries_sha"]
or name not in manifest["entries"]
):
raise ValueError("Missing or corrupt frozen endpoint")
entry = manifest["entries"][name]
elif split == "pilot":
if entry is None and name == "root":
entry = dict(
name=name,
seed=0,
arm="root",
key=digest(ROOT["reviewer"]),
source=ROOT["reviewer"],
n=0,
)
elif entry is None:
run = load(project / "results" / "runs" / (name + ".json"))
if not run["complete"] or run["stage"] != "pilot":
raise ValueError(
"Only completed pilot searches may access the pilot evaluation"
)
entry = frozen_entry(run)
else:
raise ValueError("Unknown evaluation split")
if digest(entry["source"]) != entry["key"]:
raise ValueError("Frozen source hash mismatch")
papers = read_split(project, split)
path = project / "results" / "evaluations" / f"{split}-{name}.json"
out = (
load(path)
if path.exists()
else dict(
split=split,
name=name,
seed=entry["seed"],
arm=entry["arm"],
key=entry["key"],
checkpoint=entry["n"],
rows=[],
elapsed=0.0,
started=now(),
complete=False,
)
)
if out["key"] != entry["key"]:
raise ValueError("Endpoint changed during evaluation")
done = {r["id"] for r in out["rows"]}
for paper in papers:
if paper["id"] in done:
continue
start = time.monotonic()
result = domain.review(
{"reviewer": entry["source"]}, paper, f"endpoint/{paper['id']}"
)
out["rows"].append(
dict(
id=paper["id"],
label=paper["label"],
correct=int(result["decision"] == paper["label"]),
**result,
)
)
out["elapsed"] += time.monotonic() - start
out["compute"] = domain.broker.metrics("endpoint")
out["complete"] = len(out["rows"]) == len(papers)
dump(path, out)
return out