Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_public_table100_runner.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 7.83 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_public_table100_runner.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_public_table100_runner.py
-
curl -L -o test_public_table100_runner.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_public_table100_runner.py
7.83 kB
| from __future__ import annotations | |
| import json | |
| import importlib.util | |
| from pathlib import Path | |
| _SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "run_public_table100_claim.py" | |
| _SPEC = importlib.util.spec_from_file_location("run_public_table100_claim", _SCRIPT_PATH) | |
| assert _SPEC is not None and _SPEC.loader is not None | |
| _MODULE = importlib.util.module_from_spec(_SPEC) | |
| _SPEC.loader.exec_module(_MODULE) | |
| _load_existing_report = _MODULE._load_existing_report | |
| _write_interim_status = _MODULE._write_interim_status | |
| def test_load_existing_report_deduplicates_by_baseline_and_input_path(tmp_path: Path) -> None: | |
| report = tmp_path / "report.json" | |
| report.write_text( | |
| json.dumps( | |
| [ | |
| { | |
| "baseline": "candidate", | |
| "input_path": "page.png", | |
| "status": "error", | |
| "latency_seconds": 300.0, | |
| "sample_id": "s1", | |
| "slices": ["public"], | |
| "metrics": {}, | |
| "error": "TimeoutExpired", | |
| }, | |
| { | |
| "baseline": "candidate", | |
| "input_path": "page.png", | |
| "status": "ok", | |
| "latency_seconds": 0.0, | |
| "sample_id": "s1", | |
| "slices": ["public"], | |
| "metrics": {"cer": 0.1}, | |
| "error": None, | |
| }, | |
| { | |
| "baseline": "surya", | |
| "input_path": "page.png", | |
| "status": "ok", | |
| "latency_seconds": 1.0, | |
| "sample_id": "s1", | |
| "slices": ["public"], | |
| "metrics": {"cer": 0.2}, | |
| "error": None, | |
| }, | |
| ] | |
| ), | |
| encoding="utf-8", | |
| ) | |
| rows = _load_existing_report(report) | |
| assert len(rows) == 2 | |
| by_key = {(row.baseline, row.input_path): row for row in rows} | |
| assert by_key[("candidate", "page.png")].status == "ok" | |
| assert by_key[("candidate", "page.png")].metrics["cer"] == 0.1 | |
| assert by_key[("surya", "page.png")].status == "ok" | |
| def test_write_interim_status_marks_incomplete_until_all_baselines_and_artifacts_exist(tmp_path: Path) -> None: | |
| image = tmp_path / "page.png" | |
| image.write_bytes(b"image") | |
| entry = _MODULE.ManifestEntry( | |
| sample_id="s1", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(image), | |
| text="hello", | |
| ) | |
| output_root = tmp_path / "run" | |
| benchmark_dir = output_root / "benchmark" | |
| benchmark_dir.mkdir(parents=True) | |
| report = benchmark_dir / "report.json" | |
| report.write_text( | |
| json.dumps( | |
| [ | |
| { | |
| "baseline": "candidate", | |
| "input_path": str(image), | |
| "status": "error", | |
| "latency_seconds": 1.0, | |
| "sample_id": "s1", | |
| "slices": ["public"], | |
| "metrics": {}, | |
| "error": "TimeoutExpired", | |
| } | |
| ] | |
| ), | |
| encoding="utf-8", | |
| ) | |
| status = _write_interim_status(output_root, report, [entry], ["candidate", "surya"]) | |
| assert status["status"] == "incomplete" | |
| assert status["evidence_package_ready"] is False | |
| assert status["paper_claim_ready"] is False | |
| assert status["validation"] == {"present": False, "claim_status": None, "passed": None} | |
| assert status["rows"] == 1 | |
| assert status["expected_rows"] == 2 | |
| assert status["baselines"] == {"candidate": 1} | |
| assert status["baseline_progress"] == { | |
| "candidate": {"expected": 1, "completed": 1, "ok": 0, "error": 1, "missing": 0}, | |
| "surya": {"expected": 1, "completed": 0, "ok": 0, "error": 0, "missing": 1}, | |
| } | |
| assert status["current_phase"] == "surya" | |
| assert status["row_statuses"] == {"error": 1} | |
| assert status["last_result"] == { | |
| "baseline": "candidate", | |
| "sample_id": "s1", | |
| "input_path": str(image), | |
| "status": "error", | |
| "error": "TimeoutExpired", | |
| } | |
| assert status["error_count"] == 1 | |
| assert status["recent_errors"] == [ | |
| { | |
| "baseline": "candidate", | |
| "sample_id": "s1", | |
| "input_path": str(image), | |
| "error": "TimeoutExpired", | |
| } | |
| ] | |
| assert status["next"] == {"baseline": "surya", "sample_id": "s1", "input_path": str(image)} | |
| assert status["final_artifacts"] == {"summary": False, "validation": False, "table_cell_analysis": False} | |
| assert (output_root / "run-status.json").exists() | |
| status_md = output_root / "run-status.md" | |
| assert status_md.exists() | |
| status_markdown = status_md.read_text(encoding="utf-8") | |
| assert "Evidence package ready: `no`" in status_markdown | |
| assert "Paper claim ready: `no`" in status_markdown | |
| assert "| candidate | 1 | 1 | 0 | 1 | 0 |" in status_markdown | |
| assert "- `candidate` / `s1`: TimeoutExpired" in status_markdown | |
| def test_write_interim_status_distinguishes_evidence_package_from_validated_claim(tmp_path: Path) -> None: | |
| image = tmp_path / "page.png" | |
| image.write_bytes(b"image") | |
| entry = _MODULE.ManifestEntry( | |
| sample_id="s1", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(image), | |
| text="hello", | |
| ) | |
| output_root = tmp_path / "run" | |
| benchmark_dir = output_root / "benchmark" | |
| benchmark_dir.mkdir(parents=True) | |
| report = benchmark_dir / "report.json" | |
| report.write_text( | |
| json.dumps( | |
| [ | |
| { | |
| "baseline": "candidate", | |
| "input_path": str(image), | |
| "status": "ok", | |
| "latency_seconds": 1.0, | |
| "sample_id": "s1", | |
| "slices": ["public"], | |
| "metrics": {"cer": 0.2}, | |
| "error": None, | |
| }, | |
| { | |
| "baseline": "surya", | |
| "input_path": str(image), | |
| "status": "ok", | |
| "latency_seconds": 1.0, | |
| "sample_id": "s1", | |
| "slices": ["public"], | |
| "metrics": {"cer": 0.1}, | |
| "error": None, | |
| }, | |
| ] | |
| ), | |
| encoding="utf-8", | |
| ) | |
| (output_root / "summary").mkdir() | |
| (output_root / "summary" / "summary.json").write_text("{}", encoding="utf-8") | |
| (output_root / "table-cell-analysis").mkdir() | |
| (output_root / "table-cell-analysis" / "table-cell-analysis.json").write_text("{}", encoding="utf-8") | |
| (output_root / "validation").mkdir() | |
| (output_root / "validation" / "validation.json").write_text( | |
| json.dumps( | |
| { | |
| "claim_status": "not_validated", | |
| "passed": False, | |
| "missing_requirements": ["absolute improvement below threshold"], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| status = _write_interim_status(output_root, report, [entry], ["candidate", "surya"]) | |
| assert status["status"] == "complete" | |
| assert status["evidence_package_ready"] is True | |
| assert status["paper_claim_ready"] is False | |
| assert status["validation"] == { | |
| "present": True, | |
| "claim_status": "not_validated", | |
| "passed": False, | |
| "missing_requirements": ["absolute improvement below threshold"], | |
| } | |
| status_markdown = (output_root / "run-status.md").read_text(encoding="utf-8") | |
| assert "Evidence package ready: `yes`" in status_markdown | |
| assert "Paper claim ready: `no`" in status_markdown | |
| assert "- claim_status: `not_validated`" in status_markdown | |
| assert "absolute improvement below threshold" in status_markdown | |