ocr-tech-k2 / tests /test_public_table100_runner.py
prosperious's picture
Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified
Raw History Blame Contribute Delete
7.83 kB
from __future__ import annotations
import json
import importlib.util
from pathlib import Path
_SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "run_public_table100_claim.py"
_SPEC = importlib.util.spec_from_file_location("run_public_table100_claim", _SCRIPT_PATH)
assert _SPEC is not None and _SPEC.loader is not None
_MODULE = importlib.util.module_from_spec(_SPEC)
_SPEC.loader.exec_module(_MODULE)
_load_existing_report = _MODULE._load_existing_report
_write_interim_status = _MODULE._write_interim_status
def test_load_existing_report_deduplicates_by_baseline_and_input_path(tmp_path: Path) -> None:
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
{
"baseline": "candidate",
"input_path": "page.png",
"status": "error",
"latency_seconds": 300.0,
"sample_id": "s1",
"slices": ["public"],
"metrics": {},
"error": "TimeoutExpired",
},
{
"baseline": "candidate",
"input_path": "page.png",
"status": "ok",
"latency_seconds": 0.0,
"sample_id": "s1",
"slices": ["public"],
"metrics": {"cer": 0.1},
"error": None,
},
{
"baseline": "surya",
"input_path": "page.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["public"],
"metrics": {"cer": 0.2},
"error": None,
},
]
),
encoding="utf-8",
)
rows = _load_existing_report(report)
assert len(rows) == 2
by_key = {(row.baseline, row.input_path): row for row in rows}
assert by_key[("candidate", "page.png")].status == "ok"
assert by_key[("candidate", "page.png")].metrics["cer"] == 0.1
assert by_key[("surya", "page.png")].status == "ok"
def test_write_interim_status_marks_incomplete_until_all_baselines_and_artifacts_exist(tmp_path: Path) -> None:
image = tmp_path / "page.png"
image.write_bytes(b"image")
entry = _MODULE.ManifestEntry(
sample_id="s1",
dataset="fixture",
split="eval",
image_path=str(image),
text="hello",
)
output_root = tmp_path / "run"
benchmark_dir = output_root / "benchmark"
benchmark_dir.mkdir(parents=True)
report = benchmark_dir / "report.json"
report.write_text(
json.dumps(
[
{
"baseline": "candidate",
"input_path": str(image),
"status": "error",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["public"],
"metrics": {},
"error": "TimeoutExpired",
}
]
),
encoding="utf-8",
)
status = _write_interim_status(output_root, report, [entry], ["candidate", "surya"])
assert status["status"] == "incomplete"
assert status["evidence_package_ready"] is False
assert status["paper_claim_ready"] is False
assert status["validation"] == {"present": False, "claim_status": None, "passed": None}
assert status["rows"] == 1
assert status["expected_rows"] == 2
assert status["baselines"] == {"candidate": 1}
assert status["baseline_progress"] == {
"candidate": {"expected": 1, "completed": 1, "ok": 0, "error": 1, "missing": 0},
"surya": {"expected": 1, "completed": 0, "ok": 0, "error": 0, "missing": 1},
}
assert status["current_phase"] == "surya"
assert status["row_statuses"] == {"error": 1}
assert status["last_result"] == {
"baseline": "candidate",
"sample_id": "s1",
"input_path": str(image),
"status": "error",
"error": "TimeoutExpired",
}
assert status["error_count"] == 1
assert status["recent_errors"] == [
{
"baseline": "candidate",
"sample_id": "s1",
"input_path": str(image),
"error": "TimeoutExpired",
}
]
assert status["next"] == {"baseline": "surya", "sample_id": "s1", "input_path": str(image)}
assert status["final_artifacts"] == {"summary": False, "validation": False, "table_cell_analysis": False}
assert (output_root / "run-status.json").exists()
status_md = output_root / "run-status.md"
assert status_md.exists()
status_markdown = status_md.read_text(encoding="utf-8")
assert "Evidence package ready: `no`" in status_markdown
assert "Paper claim ready: `no`" in status_markdown
assert "| candidate | 1 | 1 | 0 | 1 | 0 |" in status_markdown
assert "- `candidate` / `s1`: TimeoutExpired" in status_markdown
def test_write_interim_status_distinguishes_evidence_package_from_validated_claim(tmp_path: Path) -> None:
image = tmp_path / "page.png"
image.write_bytes(b"image")
entry = _MODULE.ManifestEntry(
sample_id="s1",
dataset="fixture",
split="eval",
image_path=str(image),
text="hello",
)
output_root = tmp_path / "run"
benchmark_dir = output_root / "benchmark"
benchmark_dir.mkdir(parents=True)
report = benchmark_dir / "report.json"
report.write_text(
json.dumps(
[
{
"baseline": "candidate",
"input_path": str(image),
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["public"],
"metrics": {"cer": 0.2},
"error": None,
},
{
"baseline": "surya",
"input_path": str(image),
"status": "ok",
"latency_seconds": 1.0,
"sample_id": "s1",
"slices": ["public"],
"metrics": {"cer": 0.1},
"error": None,
},
]
),
encoding="utf-8",
)
(output_root / "summary").mkdir()
(output_root / "summary" / "summary.json").write_text("{}", encoding="utf-8")
(output_root / "table-cell-analysis").mkdir()
(output_root / "table-cell-analysis" / "table-cell-analysis.json").write_text("{}", encoding="utf-8")
(output_root / "validation").mkdir()
(output_root / "validation" / "validation.json").write_text(
json.dumps(
{
"claim_status": "not_validated",
"passed": False,
"missing_requirements": ["absolute improvement below threshold"],
}
),
encoding="utf-8",
)
status = _write_interim_status(output_root, report, [entry], ["candidate", "surya"])
assert status["status"] == "complete"
assert status["evidence_package_ready"] is True
assert status["paper_claim_ready"] is False
assert status["validation"] == {
"present": True,
"claim_status": "not_validated",
"passed": False,
"missing_requirements": ["absolute improvement below threshold"],
}
status_markdown = (output_root / "run-status.md").read_text(encoding="utf-8")
assert "Evidence package ready: `yes`" in status_markdown
assert "Paper claim ready: `no`" in status_markdown
assert "- claim_status: `not_validated`" in status_markdown
assert "absolute improvement below threshold" in status_markdown