File size: 2,631 Bytes
d61821a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | #!/usr/bin/env python3
"""Fail if Study 5 derived evidence and manuscript claims drift apart."""
from __future__ import annotations
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def require(condition: bool, message: str) -> None:
if not condition:
raise RuntimeError(message)
def audit(root: Path = ROOT) -> dict[str, object]:
derived = root / "results" / "derived" / "study5"
audit = json.loads((derived / "artifact_audit.json").read_text())
stats = json.loads((derived / "statistical_results.json").read_text())
paper = (root / "paper" / "main.tex").read_text()
expected = {
"E13": (1440, 624, 610, 148),
"E14": (540, 143, 140, 0),
"E15": (540, 206, 199, 0),
"E16": (306, 126, 124, 1),
}
require(audit["passed"] and audit["total_cells"] == 2826, "Study 5 audit/count drift")
for experiment, values in expected.items():
row = audit["experiments"][experiment]
observed = (
row["run_count"], row["accepted_edit_count"],
row["applicable_patch_count"], row["resolved_count"],
)
require(observed == values, f"{experiment} funnel drift: {observed}")
rank = stats["e16_held_out"]["rank_stability"]
require(abs(rank["accepted"]["spearman_rank_correlation"] - 0.04411764705882353) < 1e-12,
"E16 accepted rank-correlation drift")
require(abs(rank["applicable"]["spearman_rank_correlation"] + 0.2125118592516207) < 1e-12,
"E16 applicable rank-correlation drift")
require(abs(rank["tokens"]["spearman_rank_correlation"] - 0.9428571428571428) < 1e-12,
"E16 token rank-correlation drift")
markers = (
"5,453 scored experimental cells", "2,826 new cells",
"624 executor-accepted", "143 accepted edits", "206 accepted",
"126 accepted edits", "$\\rho=.04$ for accepted edits", "$-.21$ for applicable patches",
"mean-token rank is highly stable ($\\rho=.94$)",
"Only one held-out cell resolves",
)
missing = [marker for marker in markers if marker not in paper]
require(not missing, f"manuscript is missing Study 5 evidence markers: {missing}")
require("When Better Retrieval Does Not Produce Better Patches" not in paper,
"legacy caveat title remains in manuscript")
return {
"status": "pass", "study5_cells": audit["total_cells"],
"audit_sha256": audit["audit_sha256"], "markers_checked": len(markers),
}
def main() -> None:
print(json.dumps(audit(), indent=2, sort_keys=True))
if __name__ == "__main__":
main()
|