ocr-projects-archive / ocr-tech-mps-integration /tests /test_audit_recognizer_eval_pack.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
11.3 kB
from __future__ import annotations
import importlib.util
import json
import sys
from argparse import Namespace
from pathlib import Path
def _load_tool():
path = Path(__file__).resolve().parents[1] / "tools" / "audit_recognizer_eval_pack.py"
spec = importlib.util.spec_from_file_location("audit_recognizer_eval_pack", path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _args(tmp_path: Path) -> Namespace:
return Namespace(
label_file=tmp_path / "pack" / "eval.txt",
data_dir=tmp_path / "pack",
manifest_file=tmp_path / "pack" / "manifest.jsonl",
summary_file=tmp_path / "pack" / "summary.json",
out=tmp_path / "audit",
language="newa",
script="Newa",
domain="real_handwritten",
gt_status="human_verified",
split="eval",
eval_pack_version="fixture-v1",
frozen_eval_pack=True,
unseen_for_selection=True,
claim_eligible=True,
require_claim_ready=False,
)
def _write_ready_pack(tmp_path: Path, *, rows: int = 30) -> None:
pack = tmp_path / "pack"
images = pack / "images"
images.mkdir(parents=True)
label = "𑐣𑐾𑐥𑐵𑐮"
tool = _load_tool()
label_lines: list[str] = []
manifest_lines: list[str] = []
content_rows: list[dict[str, object]] = []
for index in range(1, rows + 1):
image = images / f"line-{index}.png"
image.write_bytes(f"line image {index}".encode("utf-8"))
rel_path = f"images/line-{index}.png"
image_sha256 = tool._hash_file(image)
text_sha256 = tool._hash_text(label)
label_lines.append(f"{rel_path}\t{label}")
manifest_lines.append(
json.dumps(
{
"sample_id": f"sample-{index}",
"image": rel_path,
"text": label,
"sha256": image_sha256,
"text_sha256": text_sha256,
"sample_sha256": tool._sample_hash(image_sha256, text_sha256),
"source_image": f"source-page-{index}.jpg",
"page": index,
"bbox": [10, 20, 100, 40],
"metadata": {
"language": "newa",
"script": "Newa",
"domain": "real_handwritten",
"gt_status": "human_verified",
"split": "eval",
"claim_evidence_eligible": True,
"reviewed_by": "fixture-reviewer",
},
},
ensure_ascii=False,
)
)
content_rows.append(
{
"sample_id": f"sample-{index}",
"image_sha256": image_sha256,
"gt_text_sha256_nfc": text_sha256,
"language": "newa",
"script": "Newa",
"domain": "real_handwritten",
"gt_status": "human_verified",
"split": "eval",
"claim_eligible": True,
}
)
(pack / "eval.txt").write_text("\n".join(label_lines) + "\n", encoding="utf-8")
(pack / "manifest.jsonl").write_text("\n".join(manifest_lines) + "\n", encoding="utf-8")
(pack / "summary.json").write_text(
json.dumps(
{
"dataset": "fixture",
"eval_pack_version": "fixture-v1",
"pack_content_sha256": tool._pack_content_digest(content_rows),
"eval_pack": {"line_crops": rows},
}
)
+ "\n",
encoding="utf-8",
)
def test_audit_recognizer_eval_pack_accepts_claim_ready_pack(tmp_path: Path) -> None:
tool = _load_tool()
_write_ready_pack(tmp_path)
audit = tool.audit_pack(_args(tmp_path))
assert audit["passed"] is True
assert audit["claim_ready"] is True
assert audit["warnings"] == []
assert audit["counts"]["manifest_matched"] == 30
assert audit["counts"]["claim_eligible"] == 30
def test_audit_recognizer_eval_pack_warns_on_incomplete_provenance(tmp_path: Path) -> None:
tool = _load_tool()
_write_ready_pack(tmp_path, rows=1)
manifest = tmp_path / "pack" / "manifest.jsonl"
payload = json.loads(manifest.read_text(encoding="utf-8"))
payload.pop("page")
payload.pop("bbox")
payload["metadata"]["claim_evidence_eligible"] = False
manifest.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8")
summary = tmp_path / "pack" / "summary.json"
summary_payload = json.loads(summary.read_text(encoding="utf-8"))
summary_payload["pack_content_sha256"] = tool._pack_content_digest(
[
{
"sample_id": payload["sample_id"],
"image_sha256": payload["sha256"],
"gt_text_sha256_nfc": payload["text_sha256"],
"language": "newa",
"script": "Newa",
"domain": "real_handwritten",
"gt_status": "human_verified",
"split": "eval",
"claim_eligible": False,
}
]
)
summary.write_text(json.dumps(summary_payload) + "\n", encoding="utf-8")
audit = tool.audit_pack(_args(tmp_path))
assert audit["passed"] is True
assert audit["claim_ready"] is False
assert "missing_page: 1/1" in audit["warnings"]
assert "missing_bbox: 1/1" in audit["warnings"]
assert "not all rows are claim eligible: 0/1" in audit["warnings"]
def test_audit_recognizer_eval_pack_accepts_pdf_native_provenance_keys(tmp_path: Path) -> None:
tool = _load_tool()
_write_ready_pack(tmp_path, rows=30)
pack = tmp_path / "pack"
manifest_rows = []
for line in (pack / "manifest.jsonl").read_text(encoding="utf-8").splitlines():
payload = json.loads(line)
payload.pop("source_image")
payload.pop("page")
payload.pop("bbox")
payload["pdf_url"] = "https://example.test/source.pdf"
payload["pdf_page_number"] = 6
payload["bbox_rendered_pixels"] = [10, 20, 100, 40]
manifest_rows.append(json.dumps(payload, ensure_ascii=False))
(pack / "manifest.jsonl").write_text("\n".join(manifest_rows) + "\n", encoding="utf-8")
audit = tool.audit_pack(_args(tmp_path))
assert audit["passed"] is True
assert audit["counts"]["missing_source_document"] == 0
assert audit["counts"]["missing_page"] == 0
assert audit["counts"]["missing_bbox"] == 0
assert not any(warning.startswith("missing_source_document") for warning in audit["warnings"])
assert not any(warning.startswith("missing_page") for warning in audit["warnings"])
assert not any(warning.startswith("missing_bbox") for warning in audit["warnings"])
def test_audit_recognizer_eval_pack_warns_on_human_verified_without_review_evidence(tmp_path: Path) -> None:
tool = _load_tool()
_write_ready_pack(tmp_path, rows=1)
manifest = tmp_path / "pack" / "manifest.jsonl"
payload = json.loads(manifest.read_text(encoding="utf-8"))
payload["metadata"].pop("reviewed_by")
manifest.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8")
audit = tool.audit_pack(_args(tmp_path))
assert audit["passed"] is True
assert audit["claim_ready"] is False
assert "missing_human_review_evidence: 1/1" in audit["warnings"]
def test_audit_recognizer_eval_pack_surfaces_normalizer_defaults(tmp_path: Path) -> None:
tool = _load_tool()
_write_ready_pack(tmp_path, rows=1)
summary = tmp_path / "pack" / "summary.json"
payload = json.loads(summary.read_text(encoding="utf-8"))
payload["counts"] = {
"defaulted_domain": 1,
"defaulted_gt_status": 1,
"defaulted_split": 1,
}
summary.write_text(json.dumps(payload) + "\n", encoding="utf-8")
audit = tool.audit_pack(_args(tmp_path))
assert audit["passed"] is True
assert audit["claim_ready"] is False
assert audit["normalizer_counts"]["defaulted_domain"] == 1
assert "normalizer_defaulted_domain: 1/1" in audit["warnings"]
assert "normalizer_defaulted_gt_status: 1/1" in audit["warnings"]
assert "normalizer_defaulted_split: 1/1" in audit["warnings"]
def test_audit_recognizer_eval_pack_warns_on_tiny_claim_pack(tmp_path: Path) -> None:
tool = _load_tool()
_write_ready_pack(tmp_path, rows=1)
audit = tool.audit_pack(_args(tmp_path))
assert audit["passed"] is True
assert audit["claim_ready"] is False
assert "claim-facing sample size too small: 1 scoring units, minimum 30" in audit["warnings"]
assert (
"slice sample size too small: newa/Newa/real_handwritten/human_verified/eval has 1 scoring units, minimum 30"
in audit["warnings"]
)
def test_audit_recognizer_eval_pack_fails_on_missing_image(tmp_path: Path) -> None:
tool = _load_tool()
pack = tmp_path / "pack"
pack.mkdir()
(pack / "eval.txt").write_text("images/missing.png\ttext\n", encoding="utf-8")
(pack / "manifest.jsonl").write_text('{"image":"images/missing.png","text":"text"}\n', encoding="utf-8")
(pack / "summary.json").write_text('{"dataset":"fixture"}\n', encoding="utf-8")
audit = tool.audit_pack(_args(tmp_path))
assert audit["passed"] is False
assert audit["claim_ready"] is False
assert any("missing image" in issue for issue in audit["issues"])
def test_audit_recognizer_eval_pack_fails_on_duplicate_manifest_image_basename(tmp_path: Path) -> None:
tool = _load_tool()
_write_ready_pack(tmp_path)
manifest = tmp_path / "pack" / "manifest.jsonl"
rows = [json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()]
duplicate = dict(rows[0])
duplicate["sample_id"] = "sample-2"
duplicate["image"] = "other/line-1.png"
rows.append(duplicate)
manifest.write_text("\n".join(json.dumps(row, ensure_ascii=False) for row in rows) + "\n", encoding="utf-8")
audit = tool.audit_pack(_args(tmp_path))
assert audit["passed"] is False
assert audit["claim_ready"] is False
assert "duplicate manifest image basename: line-1.png" in audit["issues"]
def test_audit_recognizer_eval_pack_fails_on_stale_text_and_sample_hashes(tmp_path: Path) -> None:
tool = _load_tool()
_write_ready_pack(tmp_path)
manifest = tmp_path / "pack" / "manifest.jsonl"
rows = [json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()]
rows[0]["text_sha256"] = "0" * 64
rows[0]["sample_sha256"] = "1" * 64
manifest.write_text("\n".join(json.dumps(row, ensure_ascii=False) for row in rows) + "\n", encoding="utf-8")
audit = tool.audit_pack(_args(tmp_path))
assert audit["passed"] is False
assert audit["claim_ready"] is False
assert audit["counts"]["text_sha256_mismatch"] == 1
assert audit["counts"]["sample_sha256_mismatch"] == 1
assert any("text_sha256 mismatch for images/line-1.png" in issue for issue in audit["issues"])
assert any("sample_sha256 mismatch for images/line-1.png" in issue for issue in audit["issues"])