from __future__ import annotations import json import importlib.util from pathlib import Path _SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "run_public_table100_claim.py" _SPEC = importlib.util.spec_from_file_location("run_public_table100_claim", _SCRIPT_PATH) assert _SPEC is not None and _SPEC.loader is not None _MODULE = importlib.util.module_from_spec(_SPEC) _SPEC.loader.exec_module(_MODULE) _load_existing_report = _MODULE._load_existing_report _write_interim_status = _MODULE._write_interim_status def test_load_existing_report_deduplicates_by_baseline_and_input_path(tmp_path: Path) -> None: report = tmp_path / "report.json" report.write_text( json.dumps( [ { "baseline": "candidate", "input_path": "page.png", "status": "error", "latency_seconds": 300.0, "sample_id": "s1", "slices": ["public"], "metrics": {}, "error": "TimeoutExpired", }, { "baseline": "candidate", "input_path": "page.png", "status": "ok", "latency_seconds": 0.0, "sample_id": "s1", "slices": ["public"], "metrics": {"cer": 0.1}, "error": None, }, { "baseline": "surya", "input_path": "page.png", "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["public"], "metrics": {"cer": 0.2}, "error": None, }, ] ), encoding="utf-8", ) rows = _load_existing_report(report) assert len(rows) == 2 by_key = {(row.baseline, row.input_path): row for row in rows} assert by_key[("candidate", "page.png")].status == "ok" assert by_key[("candidate", "page.png")].metrics["cer"] == 0.1 assert by_key[("surya", "page.png")].status == "ok" def test_write_interim_status_marks_incomplete_until_all_baselines_and_artifacts_exist(tmp_path: Path) -> None: image = tmp_path / "page.png" image.write_bytes(b"image") entry = _MODULE.ManifestEntry( sample_id="s1", dataset="fixture", split="eval", image_path=str(image), text="hello", ) output_root = tmp_path / "run" benchmark_dir = output_root / "benchmark" benchmark_dir.mkdir(parents=True) report = benchmark_dir / "report.json" report.write_text( json.dumps( [ { "baseline": "candidate", "input_path": str(image), "status": "error", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["public"], "metrics": {}, "error": "TimeoutExpired", } ] ), encoding="utf-8", ) status = _write_interim_status(output_root, report, [entry], ["candidate", "surya"]) assert status["status"] == "incomplete" assert status["evidence_package_ready"] is False assert status["paper_claim_ready"] is False assert status["validation"] == {"present": False, "claim_status": None, "passed": None} assert status["rows"] == 1 assert status["expected_rows"] == 2 assert status["baselines"] == {"candidate": 1} assert status["baseline_progress"] == { "candidate": {"expected": 1, "completed": 1, "ok": 0, "error": 1, "missing": 0}, "surya": {"expected": 1, "completed": 0, "ok": 0, "error": 0, "missing": 1}, } assert status["current_phase"] == "surya" assert status["row_statuses"] == {"error": 1} assert status["last_result"] == { "baseline": "candidate", "sample_id": "s1", "input_path": str(image), "status": "error", "error": "TimeoutExpired", } assert status["error_count"] == 1 assert status["recent_errors"] == [ { "baseline": "candidate", "sample_id": "s1", "input_path": str(image), "error": "TimeoutExpired", } ] assert status["next"] == {"baseline": "surya", "sample_id": "s1", "input_path": str(image)} assert status["final_artifacts"] == {"summary": False, "validation": False, "table_cell_analysis": False} assert (output_root / "run-status.json").exists() status_md = output_root / "run-status.md" assert status_md.exists() status_markdown = status_md.read_text(encoding="utf-8") assert "Evidence package ready: `no`" in status_markdown assert "Paper claim ready: `no`" in status_markdown assert "| candidate | 1 | 1 | 0 | 1 | 0 |" in status_markdown assert "- `candidate` / `s1`: TimeoutExpired" in status_markdown def test_write_interim_status_distinguishes_evidence_package_from_validated_claim(tmp_path: Path) -> None: image = tmp_path / "page.png" image.write_bytes(b"image") entry = _MODULE.ManifestEntry( sample_id="s1", dataset="fixture", split="eval", image_path=str(image), text="hello", ) output_root = tmp_path / "run" benchmark_dir = output_root / "benchmark" benchmark_dir.mkdir(parents=True) report = benchmark_dir / "report.json" report.write_text( json.dumps( [ { "baseline": "candidate", "input_path": str(image), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["public"], "metrics": {"cer": 0.2}, "error": None, }, { "baseline": "surya", "input_path": str(image), "status": "ok", "latency_seconds": 1.0, "sample_id": "s1", "slices": ["public"], "metrics": {"cer": 0.1}, "error": None, }, ] ), encoding="utf-8", ) (output_root / "summary").mkdir() (output_root / "summary" / "summary.json").write_text("{}", encoding="utf-8") (output_root / "table-cell-analysis").mkdir() (output_root / "table-cell-analysis" / "table-cell-analysis.json").write_text("{}", encoding="utf-8") (output_root / "validation").mkdir() (output_root / "validation" / "validation.json").write_text( json.dumps( { "claim_status": "not_validated", "passed": False, "missing_requirements": ["absolute improvement below threshold"], } ), encoding="utf-8", ) status = _write_interim_status(output_root, report, [entry], ["candidate", "surya"]) assert status["status"] == "complete" assert status["evidence_package_ready"] is True assert status["paper_claim_ready"] is False assert status["validation"] == { "present": True, "claim_status": "not_validated", "passed": False, "missing_requirements": ["absolute improvement below threshold"], } status_markdown = (output_root / "run-status.md").read_text(encoding="utf-8") assert "Evidence package ready: `yes`" in status_markdown assert "Paper claim ready: `no`" in status_markdown assert "- claim_status: `not_validated`" in status_markdown assert "absolute improvement below threshold" in status_markdown