Spaces:
Running
Running
File size: 3,151 Bytes
ecf3aeb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | #!/usr/bin/env python3
"""Independent, standard-library-only checker for baseline artifact files."""
from __future__ import annotations
import csv
import json
import sys
from pathlib import Path
def read_csv(path: Path) -> list[dict[str, str]]:
with path.open(newline="") as handle:
return list(csv.DictReader(handle))
def main() -> None:
if len(sys.argv) != 2:
raise SystemExit("usage: check_baseline_artifacts.py ARTIFACT_DIR")
root = Path(sys.argv[1])
failures: list[str] = []
c1 = read_csv(root / "claim-1" / "exhaustive_formulas.csv")
c1_rules = read_csv(root / "claim-1" / "primitive_rules.csv")
c1_controls = read_csv(root / "claim-1" / "negative_controls.csv")
if len(c1) != 25 or sum(int(row["formulas"]) for row in c1) != 6350:
failures.append("claim_1_counts")
if max(float(row["max_probability_error"]) for row in c1) >= 1e-14:
failures.append("claim_1_probability")
if max(float(row["max_score_error"]) for row in c1) >= 1e-14:
failures.append("claim_1_score")
if len(c1_rules) != 100:
failures.append("claim_1_rules")
if len(c1_controls) != 3 or not all(row["rejected"] == "True" for row in c1_controls):
failures.append("claim_1_controls")
c2 = read_csv(root / "claim-2" / "table_2_cells.csv")
c2_control = json.loads((root / "claim-2" / "negative_control.json").read_text())
logdiff = [row for row in c2 if row["method"] == "LOGDIFF"]
constant = [row for row in c2 if row["method"] == "constant"]
if len(logdiff) != 8 or sum(row["inside_claimed_range"] == "True" for row in logdiff) != 2:
failures.append("claim_2_logdiff_cells")
if len(constant) != 8 or sum(row["inside_claimed_range"] == "True" for row in constant) != 5:
failures.append("claim_2_constant_cells")
if not c2_control["verifier_rejected_false_falsification"]:
failures.append("claim_2_control")
c5_independent = read_csv(root / "claim-5" / "independent_groups.csv")
c5_taxonomy = read_csv(root / "claim-5" / "taxonomy.csv")
c5_control = json.loads((root / "claim-5" / "negative_control.json").read_text())
if len(c5_independent) != 25 or sum(int(row["events"]) for row in c5_independent) != 20650:
failures.append("claim_5_independent_counts")
if len(c5_taxonomy) != 25 or sum(int(row["events"]) for row in c5_taxonomy) != 6350:
failures.append("claim_5_taxonomy_counts")
if max(float(row["max_probability_error"]) for row in c5_independent + c5_taxonomy) >= 1e-14:
failures.append("claim_5_probability")
if max(float(row["max_score_error"]) for row in c5_independent + c5_taxonomy) >= 1e-14:
failures.append("claim_5_score")
if not c5_control["rejected"]:
failures.append("claim_5_control")
result = {
"checker": "standard-library artifact checker",
"status": "PASS" if not failures else "FAIL",
"failures": failures,
"checked_claims": [1, 2, 5],
}
print(json.dumps(result, indent=2, sort_keys=True))
if failures:
raise SystemExit(1)
if __name__ == "__main__":
main()
|