Spaces:
Running
Running
| #!/usr/bin/env python3 | |
| """Independent, standard-library-only checker for baseline artifact files.""" | |
| from __future__ import annotations | |
| import csv | |
| import json | |
| import sys | |
| from pathlib import Path | |
| def read_csv(path: Path) -> list[dict[str, str]]: | |
| with path.open(newline="") as handle: | |
| return list(csv.DictReader(handle)) | |
| def main() -> None: | |
| if len(sys.argv) != 2: | |
| raise SystemExit("usage: check_baseline_artifacts.py ARTIFACT_DIR") | |
| root = Path(sys.argv[1]) | |
| failures: list[str] = [] | |
| c1 = read_csv(root / "claim-1" / "exhaustive_formulas.csv") | |
| c1_rules = read_csv(root / "claim-1" / "primitive_rules.csv") | |
| c1_controls = read_csv(root / "claim-1" / "negative_controls.csv") | |
| if len(c1) != 25 or sum(int(row["formulas"]) for row in c1) != 6350: | |
| failures.append("claim_1_counts") | |
| if max(float(row["max_probability_error"]) for row in c1) >= 1e-14: | |
| failures.append("claim_1_probability") | |
| if max(float(row["max_score_error"]) for row in c1) >= 1e-14: | |
| failures.append("claim_1_score") | |
| if len(c1_rules) != 100: | |
| failures.append("claim_1_rules") | |
| if len(c1_controls) != 3 or not all(row["rejected"] == "True" for row in c1_controls): | |
| failures.append("claim_1_controls") | |
| c2 = read_csv(root / "claim-2" / "table_2_cells.csv") | |
| c2_control = json.loads((root / "claim-2" / "negative_control.json").read_text()) | |
| logdiff = [row for row in c2 if row["method"] == "LOGDIFF"] | |
| constant = [row for row in c2 if row["method"] == "constant"] | |
| if len(logdiff) != 8 or sum(row["inside_claimed_range"] == "True" for row in logdiff) != 2: | |
| failures.append("claim_2_logdiff_cells") | |
| if len(constant) != 8 or sum(row["inside_claimed_range"] == "True" for row in constant) != 5: | |
| failures.append("claim_2_constant_cells") | |
| if not c2_control["verifier_rejected_false_falsification"]: | |
| failures.append("claim_2_control") | |
| c5_independent = read_csv(root / "claim-5" / "independent_groups.csv") | |
| c5_taxonomy = read_csv(root / "claim-5" / "taxonomy.csv") | |
| c5_control = json.loads((root / "claim-5" / "negative_control.json").read_text()) | |
| if len(c5_independent) != 25 or sum(int(row["events"]) for row in c5_independent) != 20650: | |
| failures.append("claim_5_independent_counts") | |
| if len(c5_taxonomy) != 25 or sum(int(row["events"]) for row in c5_taxonomy) != 6350: | |
| failures.append("claim_5_taxonomy_counts") | |
| if max(float(row["max_probability_error"]) for row in c5_independent + c5_taxonomy) >= 1e-14: | |
| failures.append("claim_5_probability") | |
| if max(float(row["max_score_error"]) for row in c5_independent + c5_taxonomy) >= 1e-14: | |
| failures.append("claim_5_score") | |
| if not c5_control["rejected"]: | |
| failures.append("claim_5_control") | |
| result = { | |
| "checker": "standard-library artifact checker", | |
| "status": "PASS" if not failures else "FAIL", | |
| "failures": failures, | |
| "checked_claims": [1, 2, 5], | |
| } | |
| print(json.dumps(result, indent=2, sort_keys=True)) | |
| if failures: | |
| raise SystemExit(1) | |
| if __name__ == "__main__": | |
| main() | |