#!/usr/bin/env python3 """Audit page candidate-line insertion effects against a baseline page score.""" from __future__ import annotations import argparse import csv import json from datetime import UTC, datetime from pathlib import Path from typing import Any from ocrtech.manifest import sha256_file REPORT_JSON = "candidate-page-line-application-effect-audit.json" INSERTED_TSV = "candidate-page-line-application-inserted-lines.tsv" OBJECTIVE_METRICS = ( "text_cer_codepoint_micro", "text_cer_grapheme_micro", "line_detection_f1_micro", "line_detection_precision_micro", "line_detection_recall_micro", "line_unmatched_reference_count", "line_unmatched_prediction_count", "line_blank_prediction_count", "matched_line_reading_order_pair_accuracy", "matched_line_text_cer_codepoint_micro", ) def _read_json(path: Path, label: str) -> dict[str, Any]: try: payload = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError as exc: raise ValueError(f"{label} is missing: {path}") from exc except json.JSONDecodeError as exc: raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc if not isinstance(payload, dict): raise ValueError(f"{label} must contain a JSON object: {path}") return payload def _float(value: Any) -> float | None: if value in ("", None): return None try: return float(value) except (TypeError, ValueError): return None def _metric_delta(candidate: Any, baseline: Any) -> float | None: candidate_float = _float(candidate) baseline_float = _float(baseline) if candidate_float is None or baseline_float is None: return None return candidate_float - baseline_float def _read_alignment_by_prediction(path: Path) -> dict[str, dict[str, Any]]: try: handle = path.open("r", encoding="utf-8", newline="") except FileNotFoundError as exc: raise ValueError(f"line alignment CSV is missing: {path}") from exc rows: dict[str, dict[str, Any]] = {} with handle: reader = csv.DictReader(handle) if reader.fieldnames is None: raise ValueError(f"line alignment CSV has no header: {path}") for row_number, row in enumerate(reader, start=2): predicted_line_id = str(row.get("predicted_line_id") or "").strip() if not predicted_line_id: continue if predicted_line_id in rows: raise ValueError(f"duplicate predicted_line_id in line alignment at {path}:{row_number}: {predicted_line_id}") rows[predicted_line_id] = dict(row) return rows def _inserted_rows(application: dict[str, Any], alignment_by_prediction: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: applied_rows = application.get("applied_rows") if not isinstance(applied_rows, list): raise ValueError("application report missing applied_rows list") inserted: list[dict[str, Any]] = [] for row in applied_rows: if not isinstance(row, dict): raise ValueError("application applied_rows entries must be objects") inserted_line_id = str(row.get("inserted_line_id") or "").strip() if not inserted_line_id: raise ValueError(f"applied row missing inserted_line_id: {row.get('row_id')!r}") alignment = alignment_by_prediction.get(inserted_line_id) if alignment is None: inserted.append( { "row_id": row.get("row_id") or "", "sample_id": row.get("sample_id") or "", "candidate_line_id": row.get("candidate_line_id") or "", "inserted_line_id": inserted_line_id, "alignment_kind": "alignment_missing", "reference_line_id": "", "iou": None, "text_codepoint_edits": None, "text_codepoint_ref_len": None, "text_cer_codepoint": None, "inserted_text": row.get("inserted_text") or "", "selection_policy": row.get("selection_policy") or "", } ) continue inserted.append( { "row_id": row.get("row_id") or "", "sample_id": row.get("sample_id") or alignment.get("sample_id") or "", "candidate_line_id": row.get("candidate_line_id") or "", "inserted_line_id": inserted_line_id, "alignment_kind": alignment.get("kind") or "", "reference_line_id": alignment.get("reference_line_id") or "", "iou": _float(alignment.get("iou")), "text_codepoint_edits": _float(alignment.get("text_codepoint_edits")), "text_codepoint_ref_len": _float(alignment.get("text_codepoint_ref_len")), "text_cer_codepoint": _float(alignment.get("text_cer_codepoint")), "inserted_text": row.get("inserted_text") or "", "selection_policy": row.get("selection_policy") or "", } ) return inserted def _write_inserted_tsv(path: Path, rows: list[dict[str, Any]]) -> None: fieldnames = [ "row_id", "sample_id", "candidate_line_id", "inserted_line_id", "alignment_kind", "reference_line_id", "iou", "text_codepoint_edits", "text_codepoint_ref_len", "text_cer_codepoint", "selection_policy", ] with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, delimiter="\t", fieldnames=fieldnames) writer.writeheader() for row in rows: writer.writerow({field: row.get(field, "") for field in fieldnames}) def _metric_summary(baseline: dict[str, Any], candidate: dict[str, Any]) -> dict[str, dict[str, Any]]: summary: dict[str, dict[str, Any]] = {} for metric in OBJECTIVE_METRICS: baseline_value = baseline.get(metric) candidate_value = candidate.get(metric) summary[metric] = { "baseline": baseline_value, "candidate": candidate_value, "delta_candidate_minus_baseline": _metric_delta(candidate_value, baseline_value), } return summary def _book_ocr_issues(metrics: dict[str, dict[str, Any]], inserted_counts: dict[str, int]) -> list[str]: issues: list[str] = [] cp_delta = metrics["text_cer_codepoint_micro"]["delta_candidate_minus_baseline"] if cp_delta is None or cp_delta > 0: issues.append(f"text_cer_codepoint_micro does not improve or tie: delta={cp_delta}") graph_delta = metrics["text_cer_grapheme_micro"]["delta_candidate_minus_baseline"] if graph_delta is None or graph_delta > 0: issues.append(f"text_cer_grapheme_micro does not improve or tie: delta={graph_delta}") line_f1_delta = metrics["line_detection_f1_micro"]["delta_candidate_minus_baseline"] if line_f1_delta is None or line_f1_delta < 0: issues.append(f"line_detection_f1_micro regresses: delta={line_f1_delta}") extra_delta = metrics["line_unmatched_prediction_count"]["delta_candidate_minus_baseline"] if extra_delta is None or extra_delta > 0: issues.append(f"line_unmatched_prediction_count increases: delta={extra_delta}") missed_delta = metrics["line_unmatched_reference_count"]["delta_candidate_minus_baseline"] if missed_delta is None or missed_delta > 0: issues.append(f"line_unmatched_reference_count increases: delta={missed_delta}") order_delta = metrics["matched_line_reading_order_pair_accuracy"]["delta_candidate_minus_baseline"] if order_delta is None or order_delta < 0: issues.append(f"matched_line_reading_order_pair_accuracy regresses: delta={order_delta}") if inserted_counts.get("prediction_unmatched", 0) > 0: issues.append(f"inserted candidate rows include unmatched extra predictions: {inserted_counts['prediction_unmatched']}") if inserted_counts.get("alignment_missing", 0) > 0: issues.append(f"inserted candidate rows missing from line alignment: {inserted_counts['alignment_missing']}") return issues def audit_application_effect(args: argparse.Namespace) -> dict[str, Any]: application_path = Path(args.application_report) baseline_metrics_path = Path(args.baseline_score_dir) / "metrics.json" candidate_metrics_path = Path(args.candidate_score_dir) / "metrics.json" alignment_path = Path(args.candidate_score_dir) / "line-alignment.csv" out = Path(args.out) out.mkdir(parents=True, exist_ok=True) application = _read_json(application_path, "candidate page-line application report") baseline_metrics = _read_json(baseline_metrics_path, "baseline metrics") candidate_metrics = _read_json(candidate_metrics_path, "candidate metrics") alignment = _read_alignment_by_prediction(alignment_path) inserted = _inserted_rows(application, alignment) inserted_counts: dict[str, int] = {} for row in inserted: kind = str(row.get("alignment_kind") or "unknown") inserted_counts[kind] = inserted_counts.get(kind, 0) + 1 metrics = _metric_summary(baseline_metrics, candidate_metrics) issues = _book_ocr_issues(metrics, inserted_counts) inserted_tsv = out / INSERTED_TSV _write_inserted_tsv(inserted_tsv, inserted) report_path = out / REPORT_JSON report = { "tool": "tools/audit_candidate_page_line_application_effect.py", "created_at_utc": datetime.now(UTC).isoformat(), "application_report": str(application_path), "application_report_sha256": sha256_file(application_path), "baseline_score_dir": str(args.baseline_score_dir), "baseline_metrics_sha256": sha256_file(baseline_metrics_path), "candidate_score_dir": str(args.candidate_score_dir), "candidate_metrics_sha256": sha256_file(candidate_metrics_path), "line_alignment_sha256": sha256_file(alignment_path), "inserted_line_count": len(inserted), "inserted_alignment_counts": inserted_counts, "objective_metric_comparison": metrics, "book_ocr_objective_safe": not issues, "issues": issues, "inserted_lines_tsv": str(inserted_tsv), "inserted_lines_tsv_sha256": sha256_file(inserted_tsv), "claim_boundary": ( "Page-candidate application objective audit. It compares a candidate projection to a baseline " "on page OCR objective metrics and inserted-line alignment; it does not certify claim-grade OCR." ), } report["report_json"] = str(report_path) report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") return report def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--application-report", required=True, type=Path) parser.add_argument("--baseline-score-dir", required=True, type=Path) parser.add_argument("--candidate-score-dir", required=True, type=Path) parser.add_argument("--out", required=True, type=Path) args = parser.parse_args() report = audit_application_effect(args) print(report["report_json"]) print( f"book_ocr_objective_safe={report['book_ocr_objective_safe']} " f"inserted={report['inserted_line_count']} issues={len(report['issues'])}" ) if __name__ == "__main__": main()