#!/usr/bin/env python3 """Summarize cross-page admission for OCR-line pruning thresholds.""" from __future__ import annotations import argparse import json import platform from datetime import datetime, timezone from pathlib import Path from typing import Any from ocrtech.manifest import sha256_file REPORT_JSON = "ocr-line-pruning-threshold-admission.json" REPORT_MD = "ocr-line-pruning-threshold-admission.md" def _read_json(path: Path, label: str) -> dict[str, Any]: if not path.exists(): raise FileNotFoundError(f"{label} does not exist: {path}") try: payload = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc if not isinstance(payload, dict): raise ValueError(f"{label} must contain a JSON object: {path}") return payload def _write_json(path: Path, payload: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def _float(value: Any, default: float = 0.0) -> float: try: return float(value) except (TypeError, ValueError): return default def _threshold_key(value: Any) -> str: threshold = _float(value) return f"{threshold:.6f}" def _sweep_label(path: Path, payload: dict[str, Any]) -> str: source = str(payload.get("source_documents_dir") or "") if source: return source.rstrip("/").split("/")[-1] return path.parent.name def _load_sweep(path: Path) -> dict[str, Any]: payload = _read_json(path, "pruning sweep") candidates = payload.get("candidates") if not isinstance(candidates, list) or not candidates: raise ValueError(f"pruning sweep has no candidates: {path}") rows: dict[str, dict[str, Any]] = {} for candidate in candidates: if not isinstance(candidate, dict): raise ValueError(f"candidate row must be an object in {path}") key = _threshold_key(candidate.get("threshold")) rows[key] = candidate return { "path": str(path), "sha256": sha256_file(path), "label": _sweep_label(path, payload), "claim_safe": payload.get("claim_safe"), "candidate_count": len(candidates), "baseline_metrics": payload.get("baseline_metrics") or {}, "rows": rows, } def _candidate_summary(threshold_key: str, sweeps: list[dict[str, Any]], args: argparse.Namespace) -> dict[str, Any]: per_page: list[dict[str, Any]] = [] missing_sweeps: list[str] = [] for sweep in sweeps: row = sweep["rows"].get(threshold_key) if row is None: missing_sweeps.append(str(sweep["label"])) continue cp_improvement = _float(row.get("codepoint_absolute_improvement")) gr_improvement = _float(row.get("grapheme_absolute_improvement")) removed = _float(row.get("removed_line_count")) kept = _float(row.get("kept_line_count")) total = kept + removed per_page.append( { "label": sweep["label"], "threshold_id": row.get("threshold_id"), "documents_dir": row.get("documents_dir"), "score_dir": row.get("score_dir"), "comparison_dir": row.get("comparison_dir"), "documents_audit_passed": row.get("documents_audit_passed") is True, "comparison_claim_safe": row.get("comparison_claim_safe") is True, "codepoint_absolute_improvement": cp_improvement, "grapheme_absolute_improvement": gr_improvement, "text_cer_codepoint_micro": row.get("text_cer_codepoint_micro"), "text_cer_grapheme_micro": row.get("text_cer_grapheme_micro"), "kept_line_count": int(kept), "removed_line_count": int(removed), "removed_fraction": (removed / total) if total > 0 else None, "documents_filter_run_sha256": row.get("documents_filter_run_sha256"), "documents_line_report_sha256": row.get("documents_line_report_sha256"), "documents_audit_sha256": row.get("documents_audit_sha256"), "page_eval_run_sha256": row.get("page_eval_run_sha256"), "metrics_sha256": row.get("metrics_sha256"), "comparison_sha256": row.get("comparison_sha256"), "warnings": row.get("comparison_warnings") or [], } ) issues: list[str] = [] if missing_sweeps: issues.append(f"threshold missing from sweeps: {missing_sweeps}") if len(per_page) < int(args.min_sweeps): issues.append(f"threshold appears in {len(per_page)} sweeps, below required {args.min_sweeps}") failed_audits = [row["label"] for row in per_page if not row["documents_audit_passed"]] if failed_audits: issues.append(f"document filter audits failed for: {failed_audits}") regressions = [ row["label"] for row in per_page if row["codepoint_absolute_improvement"] < -float(args.max_codepoint_regression) or row["grapheme_absolute_improvement"] < -float(args.max_grapheme_regression) ] if regressions: issues.append(f"threshold regresses included pages beyond tolerance: {regressions}") weak = [ row["label"] for row in per_page if row["codepoint_absolute_improvement"] < float(args.min_codepoint_improvement) ] if weak: issues.append(f"threshold fails minimum codepoint improvement on: {weak}") codepoint_values = [row["codepoint_absolute_improvement"] for row in per_page] grapheme_values = [row["grapheme_absolute_improvement"] for row in per_page] removed_fractions = [row["removed_fraction"] for row in per_page if row["removed_fraction"] is not None] return { "threshold": float(threshold_key), "threshold_key": threshold_key, "included_sweep_count": len(per_page), "missing_sweeps": missing_sweeps, "admitted_operational": not issues, "issues": issues, "mean_codepoint_absolute_improvement": sum(codepoint_values) / len(codepoint_values) if codepoint_values else None, "worst_codepoint_absolute_improvement": min(codepoint_values) if codepoint_values else None, "mean_grapheme_absolute_improvement": sum(grapheme_values) / len(grapheme_values) if grapheme_values else None, "worst_grapheme_absolute_improvement": min(grapheme_values) if grapheme_values else None, "mean_removed_fraction": sum(removed_fractions) / len(removed_fractions) if removed_fractions else None, "per_page": per_page, } def _rank_key(row: dict[str, Any]) -> tuple[int, float, float, float]: admitted = 1 if row["admitted_operational"] else 0 worst = float(row["worst_codepoint_absolute_improvement"] or -999.0) mean = float(row["mean_codepoint_absolute_improvement"] or -999.0) removed = float(row["mean_removed_fraction"] or 1.0) return admitted, worst, mean, -removed def _write_markdown(path: Path, report: dict[str, Any]) -> None: selected = report.get("selected_threshold") lines = [ "# OCR Line Pruning Threshold Admission", "", f"- selected threshold: {selected if selected is not None else 'none'}", f"- selected threshold id: {report.get('selected_threshold_id') or 'none'}", f"- admitted candidate count: {report['admitted_candidate_count']}", f"- claim safe: {report['claim_safe']}", "", "## Candidates", "", ] for row in report["candidates"]: lines.append( "- threshold {threshold:.2f}: admitted={admitted} worst_cp={worst} mean_cp={mean} issues={issues}".format( threshold=row["threshold"], admitted=row["admitted_operational"], worst=row["worst_codepoint_absolute_improvement"], mean=row["mean_codepoint_absolute_improvement"], issues="; ".join(row["issues"]) or "none", ) ) path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") def admit_threshold(args: argparse.Namespace) -> dict[str, Any]: out = Path(args.out) out.mkdir(parents=True, exist_ok=True) sweeps = [_load_sweep(Path(path)) for path in args.sweep] threshold_sets = [set(sweep["rows"].keys()) for sweep in sweeps] common_thresholds = sorted(set.intersection(*threshold_sets)) if threshold_sets else [] if args.require_common_thresholds and not common_thresholds: raise ValueError("no common thresholds are present across all sweeps") candidate_keys = common_thresholds if args.require_common_thresholds else sorted(set.union(*threshold_sets)) candidates = [_candidate_summary(key, sweeps, args) for key in candidate_keys] candidates.sort(key=_rank_key, reverse=True) admitted = [row for row in candidates if row["admitted_operational"]] selected = admitted[0] if admitted else None claim_safe = False report = { "created_at": datetime.now(timezone.utc).isoformat(), "tool": "admit_page_ocr_line_pruning_threshold.py", "machine": { "platform": platform.platform(), "python": platform.python_version(), }, "claim_safe": claim_safe, "claim_boundary": ( "Cross-page operational threshold admission only. Inputs are draft page references and " "per-page comparisons are not claim-safe, so this cannot certify final OCR quality." ), "sweeps": [ { "path": sweep["path"], "sha256": sweep["sha256"], "label": sweep["label"], "claim_safe": sweep["claim_safe"], "candidate_count": sweep["candidate_count"], } for sweep in sweeps ], "requirements": { "min_sweeps": args.min_sweeps, "require_common_thresholds": args.require_common_thresholds, "min_codepoint_improvement": args.min_codepoint_improvement, "max_codepoint_regression": args.max_codepoint_regression, "max_grapheme_regression": args.max_grapheme_regression, }, "common_thresholds": [float(key) for key in common_thresholds], "candidate_count": len(candidates), "admitted_candidate_count": len(admitted), "selected_threshold": selected["threshold"] if selected else None, "selected_threshold_id": f"conf{int(round(selected['threshold'] * 100)):03d}" if selected else None, "selected_candidate": selected, "candidates": candidates, } _write_json(out / REPORT_JSON, report) _write_markdown(out / REPORT_MD, report) return report def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--sweep", type=Path, action="append", required=True) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--min-sweeps", type=int, default=2) parser.add_argument("--require-common-thresholds", action="store_true") parser.add_argument("--min-codepoint-improvement", type=float, default=0.0) parser.add_argument("--max-codepoint-regression", type=float, default=0.0) parser.add_argument("--max-grapheme-regression", type=float, default=0.0) args = parser.parse_args(argv) report = admit_threshold(args) print(Path(args.out) / REPORT_JSON) print( f"selected_threshold={report['selected_threshold']} " f"admitted={report['admitted_candidate_count']} candidates={report['candidate_count']} " f"claim_safe={report['claim_safe']}" ) return 0 if report["admitted_candidate_count"] else 1 if __name__ == "__main__": raise SystemExit(main())