#!/usr/bin/env python3 """Audit residual page OCR failures for an admitted pruning candidate.""" from __future__ import annotations import argparse import csv import json from datetime import datetime, timezone from pathlib import Path from typing import Any from ocrtech.manifest import sha256_file REPORT_JSON = "admitted-page-ocr-candidate-audit.json" REPORT_MD = "admitted-page-ocr-candidate-audit.md" def _read_json(path: Path, label: str) -> dict[str, Any]: if not path.exists(): raise FileNotFoundError(f"{label} does not exist: {path}") try: payload = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc if not isinstance(payload, dict): raise ValueError(f"{label} must contain a JSON object: {path}") return payload def _write_json(path: Path, payload: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def _read_csv(path: Path) -> list[dict[str, str]]: if not path.exists(): raise FileNotFoundError(f"missing CSV: {path}") with path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) if not rows: raise ValueError(f"CSV has no rows: {path}") return rows def _float(value: Any, default: float = 0.0) -> float: try: return float(value) except (TypeError, ValueError): return default def _int(value: Any, default: int = 0) -> int: try: return int(value) except (TypeError, ValueError): return default def _ratio(numerator: int | float, denominator: int | float) -> float | None: if denominator == 0: return None return float(numerator) / float(denominator) def _parse_bbox(value: str, *, row_label: str, field_name: str) -> tuple[float, float, float, float] | None: if not value: return None try: parsed = json.loads(value) except json.JSONDecodeError as exc: raise ValueError(f"invalid {field_name} JSON for {row_label}: {value}") from exc if not isinstance(parsed, list) or len(parsed) != 4: raise ValueError(f"{field_name} for {row_label} must be [x, y, width, height]: {value}") try: x, y, width, height = (float(item) for item in parsed) except (TypeError, ValueError) as exc: raise ValueError(f"{field_name} for {row_label} contains a non-numeric value: {value}") from exc if width < 0 or height < 0: raise ValueError(f"{field_name} for {row_label} has negative dimensions: {value}") return (x, y, width, height) def _row_label(row: dict[str, str]) -> str: return ( row.get("reference_line_id") or row.get("predicted_line_id") or row.get("sample_id") or "unknown-row" ) def _all_bboxes(rows: list[dict[str, str]]) -> list[tuple[float, float, float, float]]: bboxes = [] for row in rows: label = _row_label(row) for field_name in ["reference_bbox", "predicted_bbox"]: bbox = _parse_bbox(row.get(field_name, ""), row_label=label, field_name=field_name) if bbox is not None: bboxes.append(bbox) return bboxes def _page_extent(rows: list[dict[str, str]]) -> dict[str, float] | None: bboxes = _all_bboxes(rows) if not bboxes: return None min_x = min(bbox[0] for bbox in bboxes) min_y = min(bbox[1] for bbox in bboxes) max_x = max(bbox[0] + bbox[2] for bbox in bboxes) max_y = max(bbox[1] + bbox[3] for bbox in bboxes) return { "min_x": min_x, "min_y": min_y, "max_x": max_x, "max_y": max_y, "width": max_x - min_x, "height": max_y - min_y, } def _third_zone(value: float, start: float, span: float, labels: tuple[str, str, str]) -> str: if span <= 0: return "unknown" ratio = (value - start) / span if ratio < 1 / 3: return labels[0] if ratio < 2 / 3: return labels[1] return labels[2] def _width_bin(width_ratio: float | None) -> str: if width_ratio is None: return "unknown" if width_ratio >= 0.45: return "long" if width_ratio >= 0.18: return "medium" return "short" def _height_bin(height_ratio: float | None) -> str: if height_ratio is None: return "unknown" if height_ratio >= 0.06: return "tall" if height_ratio >= 0.012: return "line" return "small" def _geometry_record( bbox: tuple[float, float, float, float] | None, extent: dict[str, float] | None, ) -> dict[str, Any] | None: if bbox is None: return None x, y, width, height = bbox center_x = x + width / 2 center_y = y + height / 2 page_width = extent["width"] if extent else 0.0 page_height = extent["height"] if extent else 0.0 width_ratio = _ratio(width, page_width) height_ratio = _ratio(height, page_height) return { "x": x, "y": y, "width": width, "height": height, "center_x": center_x, "center_y": center_y, "area": width * height, "width_ratio": width_ratio, "height_ratio": height_ratio, "horizontal_zone": ( _third_zone(center_x, extent["min_x"], page_width, ("left", "center", "right")) if extent else "unknown" ), "vertical_zone": ( _third_zone(center_y, extent["min_y"], page_height, ("top", "middle", "bottom")) if extent else "unknown" ), "width_bin": _width_bin(width_ratio), "height_bin": _height_bin(height_ratio), } def _increment(mapping: dict[str, int], key: str) -> None: mapping[key] = mapping.get(key, 0) + 1 def _mean(values: list[float]) -> float | None: if not values: return None return sum(values) / len(values) def _artifact_check(path: Path, expected_sha256: Any, label: str) -> dict[str, Any]: check = { "label": label, "path": str(path), "expected_sha256": expected_sha256 if isinstance(expected_sha256, str) else None, "actual_sha256": None, "verified": False, "issue": None, } if not isinstance(expected_sha256, str) or not expected_sha256: check["issue"] = f"{label} has no recorded sha256 in admission report" return check if not path.exists(): check["issue"] = f"{label} artifact does not exist: {path}" return check actual = sha256_file(path) check["actual_sha256"] = actual if actual != expected_sha256: check["issue"] = f"{label} sha256 mismatch: recorded={expected_sha256} actual={actual}" return check check["verified"] = True return check def _artifact_integrity(page: dict[str, Any], score_dir: Path) -> dict[str, Any]: documents_dir_value = page.get("documents_dir") comparison_dir_value = page.get("comparison_dir") documents_dir = Path(str(documents_dir_value)) if documents_dir_value not in (None, "") else None comparison_dir = Path(str(comparison_dir_value)) if comparison_dir_value not in (None, "") else None requested_checks: list[tuple[Path | None, str, str]] = [ (score_dir / "page-eval-run.json", "page_eval_run_sha256", "page_eval_run"), (score_dir / "metrics.json", "metrics_sha256", "metrics"), ] if comparison_dir is not None: requested_checks.append((comparison_dir / "comparison.json", "comparison_sha256", "comparison")) if documents_dir is not None: requested_checks.extend( [ (documents_dir / "ocr-line-filter-run.json", "documents_filter_run_sha256", "documents_filter_run"), (documents_dir / "ocr-line-filter-lines.jsonl", "documents_line_report_sha256", "documents_line_report"), (documents_dir / "ocr-line-filter-audit.json", "documents_audit_sha256", "documents_audit"), ] ) checks = [_artifact_check(path, page.get(hash_field), label) for path, hash_field, label in requested_checks if path is not None] issues = [str(check["issue"]) for check in checks if check.get("issue")] return { "verified": not issues, "issues": issues, "checks": checks, } def _geometry_summary( rows: list[dict[str, str]], *, bbox_field: str, extent: dict[str, float] | None, ) -> dict[str, Any]: by_horizontal_zone: dict[str, int] = {} by_vertical_zone: dict[str, int] = {} by_width_bin: dict[str, int] = {} by_height_bin: dict[str, int] = {} width_ratios: list[float] = [] height_ratios: list[float] = [] missing_bbox_count = 0 for row in rows: bbox = _parse_bbox(row.get(bbox_field, ""), row_label=_row_label(row), field_name=bbox_field) geometry = _geometry_record(bbox, extent) if geometry is None: missing_bbox_count += 1 continue _increment(by_horizontal_zone, str(geometry["horizontal_zone"])) _increment(by_vertical_zone, str(geometry["vertical_zone"])) _increment(by_width_bin, str(geometry["width_bin"])) _increment(by_height_bin, str(geometry["height_bin"])) if geometry["width_ratio"] is not None: width_ratios.append(float(geometry["width_ratio"])) if geometry["height_ratio"] is not None: height_ratios.append(float(geometry["height_ratio"])) return { "count": len(rows), "bbox_field": bbox_field, "missing_bbox_count": missing_bbox_count, "by_horizontal_zone": by_horizontal_zone, "by_vertical_zone": by_vertical_zone, "by_width_bin": by_width_bin, "by_height_bin": by_height_bin, "mean_width_ratio": _mean(width_ratios), "mean_height_ratio": _mean(height_ratios), } def _top_rows( rows: list[dict[str, str]], limit: int, sort_key: str, *, extent: dict[str, float] | None, ) -> list[dict[str, Any]]: def key(row: dict[str, str]) -> tuple[float, int]: return (_float(row.get(sort_key)), _int(row.get("text_codepoint_ref_len"))) selected = sorted(rows, key=key, reverse=True)[:limit] enriched = [] for row in selected: label = _row_label(row) reference_bbox = _parse_bbox(row.get("reference_bbox", ""), row_label=label, field_name="reference_bbox") predicted_bbox = _parse_bbox(row.get("predicted_bbox", ""), row_label=label, field_name="predicted_bbox") enriched.append( { "sample_id": row.get("sample_id"), "kind": row.get("kind"), "reference_line_id": row.get("reference_line_id"), "predicted_line_id": row.get("predicted_line_id"), "iou": _float(row.get("iou")), "reference_bbox": row.get("reference_bbox"), "predicted_bbox": row.get("predicted_bbox"), "reference_geometry": _geometry_record(reference_bbox, extent), "predicted_geometry": _geometry_record(predicted_bbox, extent), "reference_text": row.get("reference_text"), "predicted_text": row.get("predicted_text"), "text_codepoint_ref_len": _int(row.get("text_codepoint_ref_len")), "text_cer_codepoint": _float(row.get("text_cer_codepoint")), "text_cer_grapheme": _float(row.get("text_cer_grapheme")), } ) return enriched def _audit_page(page: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]: label = str(page.get("label") or "") score_dir = Path(str(page.get("score_dir") or "")) if not score_dir.exists(): raise FileNotFoundError(f"selected candidate score_dir does not exist for {label}: {score_dir}") alignment_path = score_dir / "line-alignment.csv" rows = _read_csv(alignment_path) matched = [row for row in rows if row.get("kind") == "matched"] ref_unmatched = [row for row in rows if row.get("kind") == "reference_unmatched"] pred_unmatched = [row for row in rows if row.get("kind") == "prediction_unmatched"] high_iou_ref = [row for row in ref_unmatched if _float(row.get("iou")) >= args.high_iou_unmatched_threshold] possible_near_miss_ref = [row for row in ref_unmatched if _float(row.get("iou")) >= args.near_miss_iou_threshold] high_text_error = [ row for row in matched if _float(row.get("text_cer_codepoint")) >= args.bad_codepoint_cer or _float(row.get("text_cer_grapheme")) >= args.bad_grapheme_cer ] reference_count = len(matched) + len(ref_unmatched) prediction_count = len(matched) + len(pred_unmatched) extent = _page_extent(rows) artifact_integrity = _artifact_integrity(page, score_dir) return { "label": label, "score_dir": str(score_dir), "line_alignment": str(alignment_path), "line_alignment_sha256": sha256_file(alignment_path), "artifact_integrity": artifact_integrity, "page_extent": extent, "kind_counts": { "matched": len(matched), "reference_unmatched": len(ref_unmatched), "prediction_unmatched": len(pred_unmatched), "total_rows": len(rows), }, "rates": { "reference_recall": _ratio(len(matched), reference_count), "prediction_precision": _ratio(len(matched), prediction_count), "reference_unmatched_rate": _ratio(len(ref_unmatched), reference_count), "prediction_unmatched_rate": _ratio(len(pred_unmatched), prediction_count), }, "residual_counts": { "high_iou_unmatched_references": len(high_iou_ref), "near_miss_unmatched_references": len(possible_near_miss_ref), "matched_high_text_error": len(high_text_error), }, "geometry_summary": { "unmatched_references": _geometry_summary(ref_unmatched, bbox_field="reference_bbox", extent=extent), "unmatched_predictions": _geometry_summary(pred_unmatched, bbox_field="predicted_bbox", extent=extent), "near_miss_unmatched_references": _geometry_summary( possible_near_miss_ref, bbox_field="reference_bbox", extent=extent, ), "matched_high_text_error_references": _geometry_summary( high_text_error, bbox_field="reference_bbox", extent=extent, ), "matched_high_text_error_predictions": _geometry_summary( high_text_error, bbox_field="predicted_bbox", extent=extent, ), }, "top_unmatched_references": _top_rows(ref_unmatched, args.top_n, "text_codepoint_ref_len", extent=extent), "top_unmatched_predictions": _top_rows(pred_unmatched, args.top_n, "iou", extent=extent), "top_high_iou_unmatched_references": _top_rows(high_iou_ref, args.top_n, "iou", extent=extent), "top_matched_high_text_error": _top_rows(high_text_error, args.top_n, "text_cer_codepoint", extent=extent), } def _write_markdown(path: Path, report: dict[str, Any]) -> None: lines = [ "# Admitted Page OCR Candidate Audit", "", f"- selected threshold: {report['selected_threshold']}", f"- page count: {report['page_count']}", f"- total matched: {report['totals']['matched']}", f"- total unmatched references: {report['totals']['reference_unmatched']}", f"- total unmatched predictions: {report['totals']['prediction_unmatched']}", f"- artifact integrity passed: {report['artifact_integrity_passed']}", f"- claim safe: {report['claim_safe']}", "", "## Pages", "", ] for page in report["pages"]: rates = page["rates"] geometry = page["geometry_summary"]["unmatched_references"] lines.append( "- {label}: matched={matched} ref_unmatched={ref_unmatched} pred_unmatched={pred_unmatched} " "recall={recall} precision={precision} artifact_integrity={integrity} ref_miss_vertical={vertical}".format( label=page["label"], matched=page["kind_counts"]["matched"], ref_unmatched=page["kind_counts"]["reference_unmatched"], pred_unmatched=page["kind_counts"]["prediction_unmatched"], recall=rates["reference_recall"], precision=rates["prediction_precision"], integrity=page["artifact_integrity"]["verified"], vertical=geometry["by_vertical_zone"], ) ) path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") def audit_candidate(args: argparse.Namespace) -> dict[str, Any]: out = Path(args.out) out.mkdir(parents=True, exist_ok=True) admission_path = Path(args.admission_report) admission = _read_json(admission_path, "threshold admission report") selected = admission.get("selected_candidate") if not isinstance(selected, dict): raise ValueError(f"admission report has no selected_candidate object: {admission_path}") per_page = selected.get("per_page") if not isinstance(per_page, list) or not per_page: raise ValueError(f"selected_candidate has no per_page rows: {admission_path}") pages = [_audit_page(page, args) for page in per_page if isinstance(page, dict)] totals = { "matched": sum(page["kind_counts"]["matched"] for page in pages), "reference_unmatched": sum(page["kind_counts"]["reference_unmatched"] for page in pages), "prediction_unmatched": sum(page["kind_counts"]["prediction_unmatched"] for page in pages), "high_iou_unmatched_references": sum(page["residual_counts"]["high_iou_unmatched_references"] for page in pages), "near_miss_unmatched_references": sum(page["residual_counts"]["near_miss_unmatched_references"] for page in pages), "matched_high_text_error": sum(page["residual_counts"]["matched_high_text_error"] for page in pages), } artifact_integrity_issues = [ f"{page['label']}: {issue}" for page in pages for issue in page["artifact_integrity"]["issues"] ] report = { "created_at": datetime.now(timezone.utc).isoformat(), "tool": "audit_admitted_page_ocr_candidate.py", "claim_safe": False, "artifact_integrity_passed": not artifact_integrity_issues, "artifact_integrity_issues": artifact_integrity_issues, "claim_boundary": ( "Residual operational audit for an admitted pruning threshold. Inputs are draft references and " "non-claim per-page scores; this guides layout work but does not certify final OCR quality." ), "admission_report": str(admission_path), "admission_report_sha256": sha256_file(admission_path), "selected_threshold": admission.get("selected_threshold"), "selected_threshold_id": admission.get("selected_threshold_id"), "page_count": len(pages), "thresholds": { "high_iou_unmatched_threshold": args.high_iou_unmatched_threshold, "near_miss_iou_threshold": args.near_miss_iou_threshold, "bad_codepoint_cer": args.bad_codepoint_cer, "bad_grapheme_cer": args.bad_grapheme_cer, }, "totals": totals, "pages": pages, } _write_json(out / REPORT_JSON, report) _write_markdown(out / REPORT_MD, report) return report def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--admission-report", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--high-iou-unmatched-threshold", type=float, default=0.5) parser.add_argument("--near-miss-iou-threshold", type=float, default=0.3) parser.add_argument("--bad-codepoint-cer", type=float, default=0.25) parser.add_argument("--bad-grapheme-cer", type=float, default=0.35) parser.add_argument("--top-n", type=int, default=20) args = parser.parse_args(argv) report = audit_candidate(args) print(Path(args.out) / REPORT_JSON) print( f"threshold={report['selected_threshold']} pages={report['page_count']} " f"matched={report['totals']['matched']} ref_unmatched={report['totals']['reference_unmatched']} " f"pred_unmatched={report['totals']['prediction_unmatched']} claim_safe={report['claim_safe']}" ) return 0 if __name__ == "__main__": raise SystemExit(main())