| |
| from __future__ import annotations |
| import hashlib,json,os,subprocess,sys,tempfile |
| from pathlib import Path |
| r=Path(__file__).resolve().parent |
| load=lambda p:json.loads((r/p).read_text(encoding="utf-8")) |
| claims=load("CLAIMS.json");official=load("official_claims.json");matrix=load("EVIDENCE_MATRIX.json") |
| out=load("outputs/results.json");a=load("replay_a/results.json");b=load("replay_b/results.json");p=load("packaged_replay/results.json") |
| assert claims==official and len(claims)==6 |
| assert out==a==b==p |
| assert out["summary"]=={"all_pass":True,"challenge_points":10,"conservative_points":8,"falsified":2,"seeded_pairs":300000,"toy":2,"verified":2} |
| assert matrix["release_quality_gate"]["status"]=="pass_6_of_6_decisive_with_two_literal_toys" |
| assert matrix["release_quality_gate"]["expected_verified_points"]==8 |
| expected_assessments=["verified","verified","falsified","falsified","toy","toy"] |
| for i,(literal,result,row) in enumerate(zip(claims,out["claims"],matrix["claims"]),1): |
| assert result["claim"]==row["claim"]==i and result["literal_claim"]==row["literal_claim"]==literal |
| assert result["assessment"]==row["assessment"]==expected_assessments[i-1] and result["pass"] is True |
| assert row["paper_or_released_scale"] is (i <= 4) |
| if i<=2: |
| assert row["registered_system_executed"] is True and row["actual_model_or_dataset_used"] is True |
| else: |
| assert row["registered_system_executed"] is False |
| assert load(f"outputs/claim{i}.json")==load(f"replay_a/claim{i}.json")==load(f"replay_b/claim{i}.json")==load(f"packaged_replay/claim{i}.json")==result |
| assert (r/f"pages/claim-{i}/page.md").is_file() |
| for rel,digest in out["source"]["tex_sha256"].items(): |
| assert hashlib.sha256((r/"source/tex"/rel).read_bytes()).hexdigest()==digest,rel |
| env=dict(os.environ,PYTHONDONTWRITEBYTECODE="1",PYTHONHASHSEED="0",PYTHONWARNINGS="error") |
| with tempfile.TemporaryDirectory(prefix="tablelong-validator-") as td: |
| subprocess.run([sys.executable,"-W","error",str(r/"reproduce.py"),"--out",td],check=True,env=env,stdout=subprocess.DEVNULL) |
| assert (Path(td)/"results.json").read_bytes()==(r/"outputs/results.json").read_bytes() |
| print(json.dumps({"status":"pass","claims":6,"conservative_points":8,"paired_replay":"byte-identical","seeded_pairs":300000},sort_keys=True)) |
|
|
