from __future__ import annotations import csv import hashlib import importlib.util import json import sys from argparse import Namespace from pathlib import Path def _load_tool(): path = Path(__file__).resolve().parents[1] / "tools" / "audit_admitted_page_ocr_candidate.py" spec = importlib.util.spec_from_file_location("audit_admitted_page_ocr_candidate", path) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def _write_json(path: Path, payload: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") def _write_alignment(path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) rows = [ { "sample_id": "page-1", "kind": "matched", "reference_line_id": "ref-1", "predicted_line_id": "pred-1", "reference_bbox": "[0,0,10,10]", "predicted_bbox": "[0,0,10,10]", "page_index": "0", "iou": "1.0", "reference_text": "abc", "predicted_text": "abc", "text_codepoint_edits": "0", "text_codepoint_ref_len": "3", "text_cer_codepoint": "0.0", "text_grapheme_edits": "0", "text_grapheme_ref_len": "3", "text_cer_grapheme": "0.0", }, { "sample_id": "page-1", "kind": "reference_unmatched", "reference_line_id": "ref-2", "predicted_line_id": "", "reference_bbox": "[0,20,10,10]", "predicted_bbox": "", "page_index": "0", "iou": "0.6", "reference_text": "missing", "predicted_text": "", "text_codepoint_edits": "7", "text_codepoint_ref_len": "7", "text_cer_codepoint": "1.0", "text_grapheme_edits": "7", "text_grapheme_ref_len": "7", "text_cer_grapheme": "1.0", }, { "sample_id": "page-1", "kind": "prediction_unmatched", "reference_line_id": "", "predicted_line_id": "pred-2", "reference_bbox": "", "predicted_bbox": "[50,50,10,10]", "page_index": "0", "iou": "0.0", "reference_text": "", "predicted_text": "extra", "text_codepoint_edits": "5", "text_codepoint_ref_len": "0", "text_cer_codepoint": "0.0", "text_grapheme_edits": "5", "text_grapheme_ref_len": "0", "text_cer_grapheme": "0.0", }, ] with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=list(rows[0].keys())) writer.writeheader() writer.writerows(rows) def test_audit_admitted_candidate_summarizes_residuals(tmp_path: Path) -> None: tool = _load_tool() score_dir = tmp_path / "score" _write_alignment(score_dir / "line-alignment.csv") _write_json(score_dir / "page-eval-run.json", {"claim_safe": False}) _write_json(score_dir / "metrics.json", {"text_cer_codepoint_micro": 0.5}) admission = { "selected_threshold": 0.7, "selected_threshold_id": "conf070", "selected_candidate": { "per_page": [ { "label": "fixture-page", "score_dir": str(score_dir), "page_eval_run_sha256": _sha256(score_dir / "page-eval-run.json"), "metrics_sha256": _sha256(score_dir / "metrics.json"), } ] }, } admission_path = tmp_path / "admission.json" _write_json(admission_path, admission) report = tool.audit_candidate( Namespace( admission_report=admission_path, out=tmp_path / "out", high_iou_unmatched_threshold=0.5, near_miss_iou_threshold=0.3, bad_codepoint_cer=0.25, bad_grapheme_cer=0.35, top_n=5, ) ) page = report["pages"][0] assert report["selected_threshold"] == 0.7 assert report["totals"]["matched"] == 1 assert report["totals"]["reference_unmatched"] == 1 assert report["totals"]["prediction_unmatched"] == 1 assert report["totals"]["high_iou_unmatched_references"] == 1 assert report["artifact_integrity_passed"] is True assert report["artifact_integrity_issues"] == [] assert page["rates"]["reference_recall"] == 0.5 assert page["line_alignment_sha256"] == _sha256(score_dir / "line-alignment.csv") assert page["artifact_integrity"]["verified"] is True assert [check["label"] for check in page["artifact_integrity"]["checks"]] == ["page_eval_run", "metrics"] assert page["page_extent"] == { "min_x": 0.0, "min_y": 0.0, "max_x": 60.0, "max_y": 60.0, "width": 60.0, "height": 60.0, } assert page["geometry_summary"]["unmatched_references"]["by_vertical_zone"] == {"middle": 1} assert page["geometry_summary"]["unmatched_references"]["by_width_bin"] == {"short": 1} assert page["geometry_summary"]["unmatched_predictions"]["by_horizontal_zone"] == {"right": 1} assert page["top_high_iou_unmatched_references"][0]["reference_line_id"] == "ref-2" assert page["top_high_iou_unmatched_references"][0]["reference_geometry"]["vertical_zone"] == "middle" assert page["top_unmatched_predictions"][0]["predicted_geometry"]["horizontal_zone"] == "right" def test_audit_admitted_candidate_rejects_stale_score_hashes(tmp_path: Path) -> None: tool = _load_tool() score_dir = tmp_path / "score" _write_alignment(score_dir / "line-alignment.csv") page_run = score_dir / "page-eval-run.json" metrics = score_dir / "metrics.json" _write_json(page_run, {"claim_safe": False}) _write_json(metrics, {"text_cer_codepoint_micro": 0.5}) recorded_page_hash = _sha256(page_run) _write_json(page_run, {"claim_safe": True}) admission = { "selected_threshold": 0.7, "selected_threshold_id": "conf070", "selected_candidate": { "per_page": [ { "label": "fixture-page", "score_dir": str(score_dir), "page_eval_run_sha256": recorded_page_hash, "metrics_sha256": _sha256(metrics), } ] }, } admission_path = tmp_path / "admission.json" _write_json(admission_path, admission) report = tool.audit_candidate( Namespace( admission_report=admission_path, out=tmp_path / "out", high_iou_unmatched_threshold=0.5, near_miss_iou_threshold=0.3, bad_codepoint_cer=0.25, bad_grapheme_cer=0.35, top_n=5, ) ) assert report["artifact_integrity_passed"] is False assert any("page_eval_run sha256 mismatch" in issue for issue in report["artifact_integrity_issues"]) assert report["pages"][0]["artifact_integrity"]["verified"] is False