File size: 2,276 Bytes
5114c4e 988e5c8 44541cd 988e5c8 5114c4e 44541cd 988e5c8 44541cd 5114c4e 44541cd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | #!/usr/bin/env python3
from __future__ import annotations
import hashlib,json,os,subprocess,sys,tempfile
from pathlib import Path
r=Path(__file__).resolve().parent
load=lambda p:json.loads((r/p).read_text(encoding="utf-8"))
claims=load("CLAIMS.json");official=load("official_claims.json");matrix=load("EVIDENCE_MATRIX.json")
out=load("outputs/results.json");a=load("replay_a/results.json");b=load("replay_b/results.json");p=load("packaged_replay/results.json")
assert claims==official and len(claims)==6
assert out==a==b==p
assert out["summary"]=={"all_pass":True,"challenge_points":10,"conservative_points":8,"falsified":2,"seeded_pairs":300000,"toy":2,"verified":2}
assert matrix["release_quality_gate"]["status"]=="pass_6_of_6_decisive_with_two_literal_toys"
assert matrix["release_quality_gate"]["expected_verified_points"]==8
expected_assessments=["verified","verified","falsified","falsified","toy","toy"]
for i,(literal,result,row) in enumerate(zip(claims,out["claims"],matrix["claims"]),1):
assert result["claim"]==row["claim"]==i and result["literal_claim"]==row["literal_claim"]==literal
assert result["assessment"]==row["assessment"]==expected_assessments[i-1] and result["pass"] is True
assert row["paper_or_released_scale"] is (i <= 4)
if i<=2:
assert row["registered_system_executed"] is True and row["actual_model_or_dataset_used"] is True
else:
assert row["registered_system_executed"] is False
assert load(f"outputs/claim{i}.json")==load(f"replay_a/claim{i}.json")==load(f"replay_b/claim{i}.json")==load(f"packaged_replay/claim{i}.json")==result
assert (r/f"pages/claim-{i}/page.md").is_file()
for rel,digest in out["source"]["tex_sha256"].items():
assert hashlib.sha256((r/"source/tex"/rel).read_bytes()).hexdigest()==digest,rel
env=dict(os.environ,PYTHONDONTWRITEBYTECODE="1",PYTHONHASHSEED="0",PYTHONWARNINGS="error")
with tempfile.TemporaryDirectory(prefix="tablelong-validator-") as td:
subprocess.run([sys.executable,"-W","error",str(r/"reproduce.py"),"--out",td],check=True,env=env,stdout=subprocess.DEVNULL)
assert (Path(td)/"results.json").read_bytes()==(r/"outputs/results.json").read_bytes()
print(json.dumps({"status":"pass","claims":6,"conservative_points":8,"paired_replay":"byte-identical","seeded_pairs":300000},sort_keys=True))
|