ocr-projects-archive / ocr-tech /tools /admit_page_ocr_line_pruning_threshold.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
12 kB
#!/usr/bin/env python3
"""Summarize cross-page admission for OCR-line pruning thresholds."""
from __future__ import annotations
import argparse
import json
import platform
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from ocrtech.manifest import sha256_file
REPORT_JSON = "ocr-line-pruning-threshold-admission.json"
REPORT_MD = "ocr-line-pruning-threshold-admission.md"
def _read_json(path: Path, label: str) -> dict[str, Any]:
if not path.exists():
raise FileNotFoundError(f"{label} does not exist: {path}")
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc
if not isinstance(payload, dict):
raise ValueError(f"{label} must contain a JSON object: {path}")
return payload
def _write_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def _float(value: Any, default: float = 0.0) -> float:
try:
return float(value)
except (TypeError, ValueError):
return default
def _threshold_key(value: Any) -> str:
threshold = _float(value)
return f"{threshold:.6f}"
def _sweep_label(path: Path, payload: dict[str, Any]) -> str:
source = str(payload.get("source_documents_dir") or "")
if source:
return source.rstrip("/").split("/")[-1]
return path.parent.name
def _load_sweep(path: Path) -> dict[str, Any]:
payload = _read_json(path, "pruning sweep")
candidates = payload.get("candidates")
if not isinstance(candidates, list) or not candidates:
raise ValueError(f"pruning sweep has no candidates: {path}")
rows: dict[str, dict[str, Any]] = {}
for candidate in candidates:
if not isinstance(candidate, dict):
raise ValueError(f"candidate row must be an object in {path}")
key = _threshold_key(candidate.get("threshold"))
rows[key] = candidate
return {
"path": str(path),
"sha256": sha256_file(path),
"label": _sweep_label(path, payload),
"claim_safe": payload.get("claim_safe"),
"candidate_count": len(candidates),
"baseline_metrics": payload.get("baseline_metrics") or {},
"rows": rows,
}
def _candidate_summary(threshold_key: str, sweeps: list[dict[str, Any]], args: argparse.Namespace) -> dict[str, Any]:
per_page: list[dict[str, Any]] = []
missing_sweeps: list[str] = []
for sweep in sweeps:
row = sweep["rows"].get(threshold_key)
if row is None:
missing_sweeps.append(str(sweep["label"]))
continue
cp_improvement = _float(row.get("codepoint_absolute_improvement"))
gr_improvement = _float(row.get("grapheme_absolute_improvement"))
removed = _float(row.get("removed_line_count"))
kept = _float(row.get("kept_line_count"))
total = kept + removed
per_page.append(
{
"label": sweep["label"],
"threshold_id": row.get("threshold_id"),
"documents_dir": row.get("documents_dir"),
"score_dir": row.get("score_dir"),
"comparison_dir": row.get("comparison_dir"),
"documents_audit_passed": row.get("documents_audit_passed") is True,
"comparison_claim_safe": row.get("comparison_claim_safe") is True,
"codepoint_absolute_improvement": cp_improvement,
"grapheme_absolute_improvement": gr_improvement,
"text_cer_codepoint_micro": row.get("text_cer_codepoint_micro"),
"text_cer_grapheme_micro": row.get("text_cer_grapheme_micro"),
"kept_line_count": int(kept),
"removed_line_count": int(removed),
"removed_fraction": (removed / total) if total > 0 else None,
"documents_filter_run_sha256": row.get("documents_filter_run_sha256"),
"documents_line_report_sha256": row.get("documents_line_report_sha256"),
"documents_audit_sha256": row.get("documents_audit_sha256"),
"page_eval_run_sha256": row.get("page_eval_run_sha256"),
"metrics_sha256": row.get("metrics_sha256"),
"comparison_sha256": row.get("comparison_sha256"),
"warnings": row.get("comparison_warnings") or [],
}
)
issues: list[str] = []
if missing_sweeps:
issues.append(f"threshold missing from sweeps: {missing_sweeps}")
if len(per_page) < int(args.min_sweeps):
issues.append(f"threshold appears in {len(per_page)} sweeps, below required {args.min_sweeps}")
failed_audits = [row["label"] for row in per_page if not row["documents_audit_passed"]]
if failed_audits:
issues.append(f"document filter audits failed for: {failed_audits}")
regressions = [
row["label"]
for row in per_page
if row["codepoint_absolute_improvement"] < -float(args.max_codepoint_regression)
or row["grapheme_absolute_improvement"] < -float(args.max_grapheme_regression)
]
if regressions:
issues.append(f"threshold regresses included pages beyond tolerance: {regressions}")
weak = [
row["label"]
for row in per_page
if row["codepoint_absolute_improvement"] < float(args.min_codepoint_improvement)
]
if weak:
issues.append(f"threshold fails minimum codepoint improvement on: {weak}")
codepoint_values = [row["codepoint_absolute_improvement"] for row in per_page]
grapheme_values = [row["grapheme_absolute_improvement"] for row in per_page]
removed_fractions = [row["removed_fraction"] for row in per_page if row["removed_fraction"] is not None]
return {
"threshold": float(threshold_key),
"threshold_key": threshold_key,
"included_sweep_count": len(per_page),
"missing_sweeps": missing_sweeps,
"admitted_operational": not issues,
"issues": issues,
"mean_codepoint_absolute_improvement": sum(codepoint_values) / len(codepoint_values) if codepoint_values else None,
"worst_codepoint_absolute_improvement": min(codepoint_values) if codepoint_values else None,
"mean_grapheme_absolute_improvement": sum(grapheme_values) / len(grapheme_values) if grapheme_values else None,
"worst_grapheme_absolute_improvement": min(grapheme_values) if grapheme_values else None,
"mean_removed_fraction": sum(removed_fractions) / len(removed_fractions) if removed_fractions else None,
"per_page": per_page,
}
def _rank_key(row: dict[str, Any]) -> tuple[int, float, float, float]:
admitted = 1 if row["admitted_operational"] else 0
worst = float(row["worst_codepoint_absolute_improvement"] or -999.0)
mean = float(row["mean_codepoint_absolute_improvement"] or -999.0)
removed = float(row["mean_removed_fraction"] or 1.0)
return admitted, worst, mean, -removed
def _write_markdown(path: Path, report: dict[str, Any]) -> None:
selected = report.get("selected_threshold")
lines = [
"# OCR Line Pruning Threshold Admission",
"",
f"- selected threshold: {selected if selected is not None else 'none'}",
f"- selected threshold id: {report.get('selected_threshold_id') or 'none'}",
f"- admitted candidate count: {report['admitted_candidate_count']}",
f"- claim safe: {report['claim_safe']}",
"",
"## Candidates",
"",
]
for row in report["candidates"]:
lines.append(
"- threshold {threshold:.2f}: admitted={admitted} worst_cp={worst} mean_cp={mean} issues={issues}".format(
threshold=row["threshold"],
admitted=row["admitted_operational"],
worst=row["worst_codepoint_absolute_improvement"],
mean=row["mean_codepoint_absolute_improvement"],
issues="; ".join(row["issues"]) or "none",
)
)
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
def admit_threshold(args: argparse.Namespace) -> dict[str, Any]:
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
sweeps = [_load_sweep(Path(path)) for path in args.sweep]
threshold_sets = [set(sweep["rows"].keys()) for sweep in sweeps]
common_thresholds = sorted(set.intersection(*threshold_sets)) if threshold_sets else []
if args.require_common_thresholds and not common_thresholds:
raise ValueError("no common thresholds are present across all sweeps")
candidate_keys = common_thresholds if args.require_common_thresholds else sorted(set.union(*threshold_sets))
candidates = [_candidate_summary(key, sweeps, args) for key in candidate_keys]
candidates.sort(key=_rank_key, reverse=True)
admitted = [row for row in candidates if row["admitted_operational"]]
selected = admitted[0] if admitted else None
claim_safe = False
report = {
"created_at": datetime.now(timezone.utc).isoformat(),
"tool": "admit_page_ocr_line_pruning_threshold.py",
"machine": {
"platform": platform.platform(),
"python": platform.python_version(),
},
"claim_safe": claim_safe,
"claim_boundary": (
"Cross-page operational threshold admission only. Inputs are draft page references and "
"per-page comparisons are not claim-safe, so this cannot certify final OCR quality."
),
"sweeps": [
{
"path": sweep["path"],
"sha256": sweep["sha256"],
"label": sweep["label"],
"claim_safe": sweep["claim_safe"],
"candidate_count": sweep["candidate_count"],
}
for sweep in sweeps
],
"requirements": {
"min_sweeps": args.min_sweeps,
"require_common_thresholds": args.require_common_thresholds,
"min_codepoint_improvement": args.min_codepoint_improvement,
"max_codepoint_regression": args.max_codepoint_regression,
"max_grapheme_regression": args.max_grapheme_regression,
},
"common_thresholds": [float(key) for key in common_thresholds],
"candidate_count": len(candidates),
"admitted_candidate_count": len(admitted),
"selected_threshold": selected["threshold"] if selected else None,
"selected_threshold_id": f"conf{int(round(selected['threshold'] * 100)):03d}" if selected else None,
"selected_candidate": selected,
"candidates": candidates,
}
_write_json(out / REPORT_JSON, report)
_write_markdown(out / REPORT_MD, report)
return report
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--sweep", type=Path, action="append", required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--min-sweeps", type=int, default=2)
parser.add_argument("--require-common-thresholds", action="store_true")
parser.add_argument("--min-codepoint-improvement", type=float, default=0.0)
parser.add_argument("--max-codepoint-regression", type=float, default=0.0)
parser.add_argument("--max-grapheme-regression", type=float, default=0.0)
args = parser.parse_args(argv)
report = admit_threshold(args)
print(Path(args.out) / REPORT_JSON)
print(
f"selected_threshold={report['selected_threshold']} "
f"admitted={report['admitted_candidate_count']} candidates={report['candidate_count']} "
f"claim_safe={report['claim_safe']}"
)
return 0 if report["admitted_candidate_count"] else 1
if __name__ == "__main__":
raise SystemExit(main())