File size: 6,295 Bytes
d61821a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 | """Fail closed when Study 2--4 evidence and manuscript claims drift apart."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
def _require(condition: bool, message: str) -> None:
if not condition:
raise AssertionError(message)
def audit(root: Path = ROOT) -> dict[str, Any]:
analysis_path = root / "results" / "derived" / "study2" / "e08_analysis.json"
paper_path = root / "paper" / "main.tex"
report = json.loads(analysis_path.read_text(encoding="utf-8"))
study3 = json.loads(
(root / "results/derived/study3/e09_analysis.json").read_text(encoding="utf-8")
)
study4 = json.loads(
(root / "results/derived/study4/e10_analysis.json").read_text(encoding="utf-8")
)
ancillary = json.loads(
(root / "results/derived/study4_ancillary/e11_e12_analysis.json").read_text(
encoding="utf-8"
)
)
paper = paper_path.read_text(encoding="utf-8")
manifest = report["analysis_manifest"]
run_audit = report["audit"]
contrasts = report["contrasts"]
primary = next(row for row in contrasts if row["family"] == "confirmatory_primary")
secondary = [row for row in contrasts if row["family"] != "confirmatory_primary"]
failures = report["failure_analysis"]["failure_stages"]
reliability = report["reliability"]
diagnostic = report["exploratory_tool_protocol_diagnostic"]["by_model"]
_require(manifest["input_cells"] == 912, "analysis must consume 912 E08 cells")
_require(run_audit["main_cells"] == 840, "main cell count drift")
_require(run_audit["reliability_cells"] == 72, "reliability cell count drift")
_require(run_audit["unique_run_ids"] == 912, "run IDs are not unique/complete")
_require(run_audit["model_keys"] == {
"openai/gpt-oss-20b": 456,
"qwen/qwen3.6-35b-a3b": 456,
}, "model assignment is not balanced")
_require(run_audit["execution_revision"] ==
"58933d6fa8af09fcc5a832fb3b523ffb4182bc50", "execution revision drift")
_require(primary["contrast"] == "P1_H007_vs_H000_M002", "primary contrast drift")
_require(primary["tasks"] == 60, "primary task count drift")
_require(primary["left_resolved"] == primary["right_resolved"] == 2,
"primary resolution counts drift")
_require(primary["paired_risk_difference"] == 0.0, "primary effect drift")
_require(primary["risk_difference_ci_low"] == -0.05 and
primary["risk_difference_ci_high"] == 0.05, "primary CI drift")
_require(primary["mcnemar_p"] == 1.0, "primary exact p-value drift")
_require(len(secondary) == 13, "secondary contrast-family size drift")
_require(all(row["mcnemar_p_holm"] == 1.0 for row in secondary),
"a secondary Holm-adjusted p-value changed")
_require(failures == {"empty_patch": 804, "resolved": 16, "tests": 20},
"main endpoint accounting drift")
_require(reliability["observations"] == 72 and reliability["successes"] == 0,
"reliability outcome drift")
_require(reliability["fleiss_kappa"] is None and
reliability["krippendorff_alpha_nominal"] is None and
reliability["icc_1_1"] is None, "degenerate reliability must remain undefined")
_require(diagnostic["M003"]["patch_attempts"] == 887 and
diagnostic["M003"]["patch_protocol_errors"] == 877,
"exploratory protocol diagnostic drift")
study3_primary = next(
row for row in study3["contrasts"] if row["family"] == "confirmatory_primary"
)
_require(study3["audit"]["cells"] == 540, "E09 cell count drift")
_require(
study3_primary["left_count"] == 27
and study3_primary["right_count"] == 0
and study3_primary["paired_risk_difference"] == 0.45,
"E09 primary result drift",
)
_require(
{model: value["selected_interface"] for model, value in
study3["compatibility_gate"]["models"].items()}
== {"M002": "P002", "M003": "P003", "M004": "P003"},
"E09 compatibility gate drift",
)
_require(study4["audit"]["input_cells"] == 180, "E10 cell count drift")
_require(
study4["primary"]["h007_count"] == 1
and study4["primary"]["h000_count"] == 0
and study4["primary"]["exact_task_cluster_sign_flip_p"] == 1.0,
"E10 primary result drift",
)
_require(
all(row["mcnemar_p_holm"] == 1.0 for row in study4["secondary_contrasts"]),
"E10 secondary family drift",
)
_require(ancillary["input_cells"] == 30, "E11/E12 cell count drift")
_require(
ancillary["reliability"]["unanimous_resolution_rate"] == 1.0
and ancillary["reliability"]["unanimous_trajectory_rate"] == 0.0,
"E11 reliability result drift",
)
_require(
ancillary["context_scarcity"]["resolution_by_context"]
== {"16384": 0, "65536": 0},
"E12 context result drift",
)
manuscript_markers = [
"5,453 scored cells",
"resolve 2/60 tasks",
"13 secondary exact tests",
"Only 36/840 main cells",
"877/887",
"912/912 finalized identities",
"exact replacement produces accepted edits on 27/60",
"one H007 resolution and zero H000",
"trajectories match in 0/6",
manifest["analysis_code_revision"][:12],
run_audit["execution_revision"][:12],
]
missing = [marker for marker in manuscript_markers if marker not in paper]
_require(not missing, f"manuscript is missing generated-evidence markers: {missing}")
return {
"status": "pass",
"analysis_revision": manifest["analysis_code_revision"],
"execution_revision": run_audit["execution_revision"],
"cells": manifest["input_cells"],
"primary": primary["contrast"],
"secondary_tests": len(secondary),
"study3_cells": study3["audit"]["cells"],
"study4_main_cells": study4["audit"]["input_cells"],
"study4_ancillary_cells": ancillary["input_cells"],
"paper_markers_checked": len(manuscript_markers),
}
def main() -> None:
print(json.dumps(audit(), indent=2, sort_keys=True))
if __name__ == "__main__":
main()
|