Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/tests/test_audit_recognizer_eval_pack.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 11.3 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/tests/test_audit_recognizer_eval_pack.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/tests/test_audit_recognizer_eval_pack.py
-
curl -L -o test_audit_recognizer_eval_pack.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/tests/test_audit_recognizer_eval_pack.py
11.3 kB
| from __future__ import annotations | |
| import importlib.util | |
| import json | |
| import sys | |
| from argparse import Namespace | |
| from pathlib import Path | |
| def _load_tool(): | |
| path = Path(__file__).resolve().parents[1] / "tools" / "audit_recognizer_eval_pack.py" | |
| spec = importlib.util.spec_from_file_location("audit_recognizer_eval_pack", path) | |
| assert spec is not None | |
| assert spec.loader is not None | |
| module = importlib.util.module_from_spec(spec) | |
| sys.modules[spec.name] = module | |
| spec.loader.exec_module(module) | |
| return module | |
| def _args(tmp_path: Path) -> Namespace: | |
| return Namespace( | |
| label_file=tmp_path / "pack" / "eval.txt", | |
| data_dir=tmp_path / "pack", | |
| manifest_file=tmp_path / "pack" / "manifest.jsonl", | |
| summary_file=tmp_path / "pack" / "summary.json", | |
| out=tmp_path / "audit", | |
| language="newa", | |
| script="Newa", | |
| domain="real_handwritten", | |
| gt_status="human_verified", | |
| split="eval", | |
| eval_pack_version="fixture-v1", | |
| frozen_eval_pack=True, | |
| unseen_for_selection=True, | |
| claim_eligible=True, | |
| require_claim_ready=False, | |
| ) | |
| def _write_ready_pack(tmp_path: Path, *, rows: int = 30) -> None: | |
| pack = tmp_path / "pack" | |
| images = pack / "images" | |
| images.mkdir(parents=True) | |
| label = "𑐣𑐾𑐥𑐵𑐮" | |
| tool = _load_tool() | |
| label_lines: list[str] = [] | |
| manifest_lines: list[str] = [] | |
| content_rows: list[dict[str, object]] = [] | |
| for index in range(1, rows + 1): | |
| image = images / f"line-{index}.png" | |
| image.write_bytes(f"line image {index}".encode("utf-8")) | |
| rel_path = f"images/line-{index}.png" | |
| image_sha256 = tool._hash_file(image) | |
| text_sha256 = tool._hash_text(label) | |
| label_lines.append(f"{rel_path}\t{label}") | |
| manifest_lines.append( | |
| json.dumps( | |
| { | |
| "sample_id": f"sample-{index}", | |
| "image": rel_path, | |
| "text": label, | |
| "sha256": image_sha256, | |
| "text_sha256": text_sha256, | |
| "sample_sha256": tool._sample_hash(image_sha256, text_sha256), | |
| "source_image": f"source-page-{index}.jpg", | |
| "page": index, | |
| "bbox": [10, 20, 100, 40], | |
| "metadata": { | |
| "language": "newa", | |
| "script": "Newa", | |
| "domain": "real_handwritten", | |
| "gt_status": "human_verified", | |
| "split": "eval", | |
| "claim_evidence_eligible": True, | |
| "reviewed_by": "fixture-reviewer", | |
| }, | |
| }, | |
| ensure_ascii=False, | |
| ) | |
| ) | |
| content_rows.append( | |
| { | |
| "sample_id": f"sample-{index}", | |
| "image_sha256": image_sha256, | |
| "gt_text_sha256_nfc": text_sha256, | |
| "language": "newa", | |
| "script": "Newa", | |
| "domain": "real_handwritten", | |
| "gt_status": "human_verified", | |
| "split": "eval", | |
| "claim_eligible": True, | |
| } | |
| ) | |
| (pack / "eval.txt").write_text("\n".join(label_lines) + "\n", encoding="utf-8") | |
| (pack / "manifest.jsonl").write_text("\n".join(manifest_lines) + "\n", encoding="utf-8") | |
| (pack / "summary.json").write_text( | |
| json.dumps( | |
| { | |
| "dataset": "fixture", | |
| "eval_pack_version": "fixture-v1", | |
| "pack_content_sha256": tool._pack_content_digest(content_rows), | |
| "eval_pack": {"line_crops": rows}, | |
| } | |
| ) | |
| + "\n", | |
| encoding="utf-8", | |
| ) | |
| def test_audit_recognizer_eval_pack_accepts_claim_ready_pack(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| _write_ready_pack(tmp_path) | |
| audit = tool.audit_pack(_args(tmp_path)) | |
| assert audit["passed"] is True | |
| assert audit["claim_ready"] is True | |
| assert audit["warnings"] == [] | |
| assert audit["counts"]["manifest_matched"] == 30 | |
| assert audit["counts"]["claim_eligible"] == 30 | |
| def test_audit_recognizer_eval_pack_warns_on_incomplete_provenance(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| _write_ready_pack(tmp_path, rows=1) | |
| manifest = tmp_path / "pack" / "manifest.jsonl" | |
| payload = json.loads(manifest.read_text(encoding="utf-8")) | |
| payload.pop("page") | |
| payload.pop("bbox") | |
| payload["metadata"]["claim_evidence_eligible"] = False | |
| manifest.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") | |
| summary = tmp_path / "pack" / "summary.json" | |
| summary_payload = json.loads(summary.read_text(encoding="utf-8")) | |
| summary_payload["pack_content_sha256"] = tool._pack_content_digest( | |
| [ | |
| { | |
| "sample_id": payload["sample_id"], | |
| "image_sha256": payload["sha256"], | |
| "gt_text_sha256_nfc": payload["text_sha256"], | |
| "language": "newa", | |
| "script": "Newa", | |
| "domain": "real_handwritten", | |
| "gt_status": "human_verified", | |
| "split": "eval", | |
| "claim_eligible": False, | |
| } | |
| ] | |
| ) | |
| summary.write_text(json.dumps(summary_payload) + "\n", encoding="utf-8") | |
| audit = tool.audit_pack(_args(tmp_path)) | |
| assert audit["passed"] is True | |
| assert audit["claim_ready"] is False | |
| assert "missing_page: 1/1" in audit["warnings"] | |
| assert "missing_bbox: 1/1" in audit["warnings"] | |
| assert "not all rows are claim eligible: 0/1" in audit["warnings"] | |
| def test_audit_recognizer_eval_pack_accepts_pdf_native_provenance_keys(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| _write_ready_pack(tmp_path, rows=30) | |
| pack = tmp_path / "pack" | |
| manifest_rows = [] | |
| for line in (pack / "manifest.jsonl").read_text(encoding="utf-8").splitlines(): | |
| payload = json.loads(line) | |
| payload.pop("source_image") | |
| payload.pop("page") | |
| payload.pop("bbox") | |
| payload["pdf_url"] = "https://example.test/source.pdf" | |
| payload["pdf_page_number"] = 6 | |
| payload["bbox_rendered_pixels"] = [10, 20, 100, 40] | |
| manifest_rows.append(json.dumps(payload, ensure_ascii=False)) | |
| (pack / "manifest.jsonl").write_text("\n".join(manifest_rows) + "\n", encoding="utf-8") | |
| audit = tool.audit_pack(_args(tmp_path)) | |
| assert audit["passed"] is True | |
| assert audit["counts"]["missing_source_document"] == 0 | |
| assert audit["counts"]["missing_page"] == 0 | |
| assert audit["counts"]["missing_bbox"] == 0 | |
| assert not any(warning.startswith("missing_source_document") for warning in audit["warnings"]) | |
| assert not any(warning.startswith("missing_page") for warning in audit["warnings"]) | |
| assert not any(warning.startswith("missing_bbox") for warning in audit["warnings"]) | |
| def test_audit_recognizer_eval_pack_warns_on_human_verified_without_review_evidence(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| _write_ready_pack(tmp_path, rows=1) | |
| manifest = tmp_path / "pack" / "manifest.jsonl" | |
| payload = json.loads(manifest.read_text(encoding="utf-8")) | |
| payload["metadata"].pop("reviewed_by") | |
| manifest.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") | |
| audit = tool.audit_pack(_args(tmp_path)) | |
| assert audit["passed"] is True | |
| assert audit["claim_ready"] is False | |
| assert "missing_human_review_evidence: 1/1" in audit["warnings"] | |
| def test_audit_recognizer_eval_pack_surfaces_normalizer_defaults(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| _write_ready_pack(tmp_path, rows=1) | |
| summary = tmp_path / "pack" / "summary.json" | |
| payload = json.loads(summary.read_text(encoding="utf-8")) | |
| payload["counts"] = { | |
| "defaulted_domain": 1, | |
| "defaulted_gt_status": 1, | |
| "defaulted_split": 1, | |
| } | |
| summary.write_text(json.dumps(payload) + "\n", encoding="utf-8") | |
| audit = tool.audit_pack(_args(tmp_path)) | |
| assert audit["passed"] is True | |
| assert audit["claim_ready"] is False | |
| assert audit["normalizer_counts"]["defaulted_domain"] == 1 | |
| assert "normalizer_defaulted_domain: 1/1" in audit["warnings"] | |
| assert "normalizer_defaulted_gt_status: 1/1" in audit["warnings"] | |
| assert "normalizer_defaulted_split: 1/1" in audit["warnings"] | |
| def test_audit_recognizer_eval_pack_warns_on_tiny_claim_pack(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| _write_ready_pack(tmp_path, rows=1) | |
| audit = tool.audit_pack(_args(tmp_path)) | |
| assert audit["passed"] is True | |
| assert audit["claim_ready"] is False | |
| assert "claim-facing sample size too small: 1 scoring units, minimum 30" in audit["warnings"] | |
| assert ( | |
| "slice sample size too small: newa/Newa/real_handwritten/human_verified/eval has 1 scoring units, minimum 30" | |
| in audit["warnings"] | |
| ) | |
| def test_audit_recognizer_eval_pack_fails_on_missing_image(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| pack = tmp_path / "pack" | |
| pack.mkdir() | |
| (pack / "eval.txt").write_text("images/missing.png\ttext\n", encoding="utf-8") | |
| (pack / "manifest.jsonl").write_text('{"image":"images/missing.png","text":"text"}\n', encoding="utf-8") | |
| (pack / "summary.json").write_text('{"dataset":"fixture"}\n', encoding="utf-8") | |
| audit = tool.audit_pack(_args(tmp_path)) | |
| assert audit["passed"] is False | |
| assert audit["claim_ready"] is False | |
| assert any("missing image" in issue for issue in audit["issues"]) | |
| def test_audit_recognizer_eval_pack_fails_on_duplicate_manifest_image_basename(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| _write_ready_pack(tmp_path) | |
| manifest = tmp_path / "pack" / "manifest.jsonl" | |
| rows = [json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()] | |
| duplicate = dict(rows[0]) | |
| duplicate["sample_id"] = "sample-2" | |
| duplicate["image"] = "other/line-1.png" | |
| rows.append(duplicate) | |
| manifest.write_text("\n".join(json.dumps(row, ensure_ascii=False) for row in rows) + "\n", encoding="utf-8") | |
| audit = tool.audit_pack(_args(tmp_path)) | |
| assert audit["passed"] is False | |
| assert audit["claim_ready"] is False | |
| assert "duplicate manifest image basename: line-1.png" in audit["issues"] | |
| def test_audit_recognizer_eval_pack_fails_on_stale_text_and_sample_hashes(tmp_path: Path) -> None: | |
| tool = _load_tool() | |
| _write_ready_pack(tmp_path) | |
| manifest = tmp_path / "pack" / "manifest.jsonl" | |
| rows = [json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()] | |
| rows[0]["text_sha256"] = "0" * 64 | |
| rows[0]["sample_sha256"] = "1" * 64 | |
| manifest.write_text("\n".join(json.dumps(row, ensure_ascii=False) for row in rows) + "\n", encoding="utf-8") | |
| audit = tool.audit_pack(_args(tmp_path)) | |
| assert audit["passed"] is False | |
| assert audit["claim_ready"] is False | |
| assert audit["counts"]["text_sha256_mismatch"] == 1 | |
| assert audit["counts"]["sample_sha256_mismatch"] == 1 | |
| assert any("text_sha256 mismatch for images/line-1.png" in issue for issue in audit["issues"]) | |
| assert any("sample_sha256 mismatch for images/line-1.png" in issue for issue in audit["issues"]) | |