from __future__ import annotations import importlib.util import json import sys from argparse import Namespace from pathlib import Path def _load_tool(): path = Path(__file__).resolve().parents[1] / "tools" / "audit_recognizer_eval_pack.py" spec = importlib.util.spec_from_file_location("audit_recognizer_eval_pack", path) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _args(tmp_path: Path) -> Namespace: return Namespace( label_file=tmp_path / "pack" / "eval.txt", data_dir=tmp_path / "pack", manifest_file=tmp_path / "pack" / "manifest.jsonl", summary_file=tmp_path / "pack" / "summary.json", out=tmp_path / "audit", language="newa", script="Newa", domain="real_handwritten", gt_status="human_verified", split="eval", eval_pack_version="fixture-v1", frozen_eval_pack=True, unseen_for_selection=True, claim_eligible=True, require_claim_ready=False, ) def _write_ready_pack(tmp_path: Path, *, rows: int = 30) -> None: pack = tmp_path / "pack" images = pack / "images" images.mkdir(parents=True) label = "𑐣𑐾𑐥𑐵𑐮" tool = _load_tool() label_lines: list[str] = [] manifest_lines: list[str] = [] content_rows: list[dict[str, object]] = [] for index in range(1, rows + 1): image = images / f"line-{index}.png" image.write_bytes(f"line image {index}".encode("utf-8")) rel_path = f"images/line-{index}.png" image_sha256 = tool._hash_file(image) text_sha256 = tool._hash_text(label) label_lines.append(f"{rel_path}\t{label}") manifest_lines.append( json.dumps( { "sample_id": f"sample-{index}", "image": rel_path, "text": label, "sha256": image_sha256, "text_sha256": text_sha256, "sample_sha256": tool._sample_hash(image_sha256, text_sha256), "source_image": f"source-page-{index}.jpg", "page": index, "bbox": [10, 20, 100, 40], "metadata": { "language": "newa", "script": "Newa", "domain": "real_handwritten", "gt_status": "human_verified", "split": "eval", "claim_evidence_eligible": True, "reviewed_by": "fixture-reviewer", }, }, ensure_ascii=False, ) ) content_rows.append( { "sample_id": f"sample-{index}", "image_sha256": image_sha256, "gt_text_sha256_nfc": text_sha256, "language": "newa", "script": "Newa", "domain": "real_handwritten", "gt_status": "human_verified", "split": "eval", "claim_eligible": True, } ) (pack / "eval.txt").write_text("\n".join(label_lines) + "\n", encoding="utf-8") (pack / "manifest.jsonl").write_text("\n".join(manifest_lines) + "\n", encoding="utf-8") (pack / "summary.json").write_text( json.dumps( { "dataset": "fixture", "eval_pack_version": "fixture-v1", "pack_content_sha256": tool._pack_content_digest(content_rows), "eval_pack": {"line_crops": rows}, } ) + "\n", encoding="utf-8", ) def test_audit_recognizer_eval_pack_accepts_claim_ready_pack(tmp_path: Path) -> None: tool = _load_tool() _write_ready_pack(tmp_path) audit = tool.audit_pack(_args(tmp_path)) assert audit["passed"] is True assert audit["claim_ready"] is True assert audit["warnings"] == [] assert audit["counts"]["manifest_matched"] == 30 assert audit["counts"]["claim_eligible"] == 30 def test_audit_recognizer_eval_pack_warns_on_incomplete_provenance(tmp_path: Path) -> None: tool = _load_tool() _write_ready_pack(tmp_path, rows=1) manifest = tmp_path / "pack" / "manifest.jsonl" payload = json.loads(manifest.read_text(encoding="utf-8")) payload.pop("page") payload.pop("bbox") payload["metadata"]["claim_evidence_eligible"] = False manifest.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") summary = tmp_path / "pack" / "summary.json" summary_payload = json.loads(summary.read_text(encoding="utf-8")) summary_payload["pack_content_sha256"] = tool._pack_content_digest( [ { "sample_id": payload["sample_id"], "image_sha256": payload["sha256"], "gt_text_sha256_nfc": payload["text_sha256"], "language": "newa", "script": "Newa", "domain": "real_handwritten", "gt_status": "human_verified", "split": "eval", "claim_eligible": False, } ] ) summary.write_text(json.dumps(summary_payload) + "\n", encoding="utf-8") audit = tool.audit_pack(_args(tmp_path)) assert audit["passed"] is True assert audit["claim_ready"] is False assert "missing_page: 1/1" in audit["warnings"] assert "missing_bbox: 1/1" in audit["warnings"] assert "not all rows are claim eligible: 0/1" in audit["warnings"] def test_audit_recognizer_eval_pack_accepts_pdf_native_provenance_keys(tmp_path: Path) -> None: tool = _load_tool() _write_ready_pack(tmp_path, rows=30) pack = tmp_path / "pack" manifest_rows = [] for line in (pack / "manifest.jsonl").read_text(encoding="utf-8").splitlines(): payload = json.loads(line) payload.pop("source_image") payload.pop("page") payload.pop("bbox") payload["pdf_url"] = "https://example.test/source.pdf" payload["pdf_page_number"] = 6 payload["bbox_rendered_pixels"] = [10, 20, 100, 40] manifest_rows.append(json.dumps(payload, ensure_ascii=False)) (pack / "manifest.jsonl").write_text("\n".join(manifest_rows) + "\n", encoding="utf-8") audit = tool.audit_pack(_args(tmp_path)) assert audit["passed"] is True assert audit["counts"]["missing_source_document"] == 0 assert audit["counts"]["missing_page"] == 0 assert audit["counts"]["missing_bbox"] == 0 assert not any(warning.startswith("missing_source_document") for warning in audit["warnings"]) assert not any(warning.startswith("missing_page") for warning in audit["warnings"]) assert not any(warning.startswith("missing_bbox") for warning in audit["warnings"]) def test_audit_recognizer_eval_pack_warns_on_human_verified_without_review_evidence(tmp_path: Path) -> None: tool = _load_tool() _write_ready_pack(tmp_path, rows=1) manifest = tmp_path / "pack" / "manifest.jsonl" payload = json.loads(manifest.read_text(encoding="utf-8")) payload["metadata"].pop("reviewed_by") manifest.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") audit = tool.audit_pack(_args(tmp_path)) assert audit["passed"] is True assert audit["claim_ready"] is False assert "missing_human_review_evidence: 1/1" in audit["warnings"] def test_audit_recognizer_eval_pack_surfaces_normalizer_defaults(tmp_path: Path) -> None: tool = _load_tool() _write_ready_pack(tmp_path, rows=1) summary = tmp_path / "pack" / "summary.json" payload = json.loads(summary.read_text(encoding="utf-8")) payload["counts"] = { "defaulted_domain": 1, "defaulted_gt_status": 1, "defaulted_split": 1, } summary.write_text(json.dumps(payload) + "\n", encoding="utf-8") audit = tool.audit_pack(_args(tmp_path)) assert audit["passed"] is True assert audit["claim_ready"] is False assert audit["normalizer_counts"]["defaulted_domain"] == 1 assert "normalizer_defaulted_domain: 1/1" in audit["warnings"] assert "normalizer_defaulted_gt_status: 1/1" in audit["warnings"] assert "normalizer_defaulted_split: 1/1" in audit["warnings"] def test_audit_recognizer_eval_pack_warns_on_tiny_claim_pack(tmp_path: Path) -> None: tool = _load_tool() _write_ready_pack(tmp_path, rows=1) audit = tool.audit_pack(_args(tmp_path)) assert audit["passed"] is True assert audit["claim_ready"] is False assert "claim-facing sample size too small: 1 scoring units, minimum 30" in audit["warnings"] assert ( "slice sample size too small: newa/Newa/real_handwritten/human_verified/eval has 1 scoring units, minimum 30" in audit["warnings"] ) def test_audit_recognizer_eval_pack_fails_on_missing_image(tmp_path: Path) -> None: tool = _load_tool() pack = tmp_path / "pack" pack.mkdir() (pack / "eval.txt").write_text("images/missing.png\ttext\n", encoding="utf-8") (pack / "manifest.jsonl").write_text('{"image":"images/missing.png","text":"text"}\n', encoding="utf-8") (pack / "summary.json").write_text('{"dataset":"fixture"}\n', encoding="utf-8") audit = tool.audit_pack(_args(tmp_path)) assert audit["passed"] is False assert audit["claim_ready"] is False assert any("missing image" in issue for issue in audit["issues"]) def test_audit_recognizer_eval_pack_fails_on_duplicate_manifest_image_basename(tmp_path: Path) -> None: tool = _load_tool() _write_ready_pack(tmp_path) manifest = tmp_path / "pack" / "manifest.jsonl" rows = [json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()] duplicate = dict(rows[0]) duplicate["sample_id"] = "sample-2" duplicate["image"] = "other/line-1.png" rows.append(duplicate) manifest.write_text("\n".join(json.dumps(row, ensure_ascii=False) for row in rows) + "\n", encoding="utf-8") audit = tool.audit_pack(_args(tmp_path)) assert audit["passed"] is False assert audit["claim_ready"] is False assert "duplicate manifest image basename: line-1.png" in audit["issues"] def test_audit_recognizer_eval_pack_fails_on_stale_text_and_sample_hashes(tmp_path: Path) -> None: tool = _load_tool() _write_ready_pack(tmp_path) manifest = tmp_path / "pack" / "manifest.jsonl" rows = [json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()] rows[0]["text_sha256"] = "0" * 64 rows[0]["sample_sha256"] = "1" * 64 manifest.write_text("\n".join(json.dumps(row, ensure_ascii=False) for row in rows) + "\n", encoding="utf-8") audit = tool.audit_pack(_args(tmp_path)) assert audit["passed"] is False assert audit["claim_ready"] is False assert audit["counts"]["text_sha256_mismatch"] == 1 assert audit["counts"]["sample_sha256_mismatch"] == 1 assert any("text_sha256 mismatch for images/line-1.png" in issue for issue in audit["issues"]) assert any("sample_sha256 mismatch for images/line-1.png" in issue for issue in audit["issues"])