Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/admit_page_ocr_line_pruning_threshold.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 12 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/admit_page_ocr_line_pruning_threshold.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/admit_page_ocr_line_pruning_threshold.py
-
curl -L -o admit_page_ocr_line_pruning_threshold.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/admit_page_ocr_line_pruning_threshold.py
12 kB
| #!/usr/bin/env python3 | |
| """Summarize cross-page admission for OCR-line pruning thresholds.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import platform | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from typing import Any | |
| from ocrtech.manifest import sha256_file | |
| REPORT_JSON = "ocr-line-pruning-threshold-admission.json" | |
| REPORT_MD = "ocr-line-pruning-threshold-admission.md" | |
| def _read_json(path: Path, label: str) -> dict[str, Any]: | |
| if not path.exists(): | |
| raise FileNotFoundError(f"{label} does not exist: {path}") | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise ValueError(f"{label} must contain a JSON object: {path}") | |
| return payload | |
| def _write_json(path: Path, payload: Any) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| def _float(value: Any, default: float = 0.0) -> float: | |
| try: | |
| return float(value) | |
| except (TypeError, ValueError): | |
| return default | |
| def _threshold_key(value: Any) -> str: | |
| threshold = _float(value) | |
| return f"{threshold:.6f}" | |
| def _sweep_label(path: Path, payload: dict[str, Any]) -> str: | |
| source = str(payload.get("source_documents_dir") or "") | |
| if source: | |
| return source.rstrip("/").split("/")[-1] | |
| return path.parent.name | |
| def _load_sweep(path: Path) -> dict[str, Any]: | |
| payload = _read_json(path, "pruning sweep") | |
| candidates = payload.get("candidates") | |
| if not isinstance(candidates, list) or not candidates: | |
| raise ValueError(f"pruning sweep has no candidates: {path}") | |
| rows: dict[str, dict[str, Any]] = {} | |
| for candidate in candidates: | |
| if not isinstance(candidate, dict): | |
| raise ValueError(f"candidate row must be an object in {path}") | |
| key = _threshold_key(candidate.get("threshold")) | |
| rows[key] = candidate | |
| return { | |
| "path": str(path), | |
| "sha256": sha256_file(path), | |
| "label": _sweep_label(path, payload), | |
| "claim_safe": payload.get("claim_safe"), | |
| "candidate_count": len(candidates), | |
| "baseline_metrics": payload.get("baseline_metrics") or {}, | |
| "rows": rows, | |
| } | |
| def _candidate_summary(threshold_key: str, sweeps: list[dict[str, Any]], args: argparse.Namespace) -> dict[str, Any]: | |
| per_page: list[dict[str, Any]] = [] | |
| missing_sweeps: list[str] = [] | |
| for sweep in sweeps: | |
| row = sweep["rows"].get(threshold_key) | |
| if row is None: | |
| missing_sweeps.append(str(sweep["label"])) | |
| continue | |
| cp_improvement = _float(row.get("codepoint_absolute_improvement")) | |
| gr_improvement = _float(row.get("grapheme_absolute_improvement")) | |
| removed = _float(row.get("removed_line_count")) | |
| kept = _float(row.get("kept_line_count")) | |
| total = kept + removed | |
| per_page.append( | |
| { | |
| "label": sweep["label"], | |
| "threshold_id": row.get("threshold_id"), | |
| "documents_dir": row.get("documents_dir"), | |
| "score_dir": row.get("score_dir"), | |
| "comparison_dir": row.get("comparison_dir"), | |
| "documents_audit_passed": row.get("documents_audit_passed") is True, | |
| "comparison_claim_safe": row.get("comparison_claim_safe") is True, | |
| "codepoint_absolute_improvement": cp_improvement, | |
| "grapheme_absolute_improvement": gr_improvement, | |
| "text_cer_codepoint_micro": row.get("text_cer_codepoint_micro"), | |
| "text_cer_grapheme_micro": row.get("text_cer_grapheme_micro"), | |
| "kept_line_count": int(kept), | |
| "removed_line_count": int(removed), | |
| "removed_fraction": (removed / total) if total > 0 else None, | |
| "documents_filter_run_sha256": row.get("documents_filter_run_sha256"), | |
| "documents_line_report_sha256": row.get("documents_line_report_sha256"), | |
| "documents_audit_sha256": row.get("documents_audit_sha256"), | |
| "page_eval_run_sha256": row.get("page_eval_run_sha256"), | |
| "metrics_sha256": row.get("metrics_sha256"), | |
| "comparison_sha256": row.get("comparison_sha256"), | |
| "warnings": row.get("comparison_warnings") or [], | |
| } | |
| ) | |
| issues: list[str] = [] | |
| if missing_sweeps: | |
| issues.append(f"threshold missing from sweeps: {missing_sweeps}") | |
| if len(per_page) < int(args.min_sweeps): | |
| issues.append(f"threshold appears in {len(per_page)} sweeps, below required {args.min_sweeps}") | |
| failed_audits = [row["label"] for row in per_page if not row["documents_audit_passed"]] | |
| if failed_audits: | |
| issues.append(f"document filter audits failed for: {failed_audits}") | |
| regressions = [ | |
| row["label"] | |
| for row in per_page | |
| if row["codepoint_absolute_improvement"] < -float(args.max_codepoint_regression) | |
| or row["grapheme_absolute_improvement"] < -float(args.max_grapheme_regression) | |
| ] | |
| if regressions: | |
| issues.append(f"threshold regresses included pages beyond tolerance: {regressions}") | |
| weak = [ | |
| row["label"] | |
| for row in per_page | |
| if row["codepoint_absolute_improvement"] < float(args.min_codepoint_improvement) | |
| ] | |
| if weak: | |
| issues.append(f"threshold fails minimum codepoint improvement on: {weak}") | |
| codepoint_values = [row["codepoint_absolute_improvement"] for row in per_page] | |
| grapheme_values = [row["grapheme_absolute_improvement"] for row in per_page] | |
| removed_fractions = [row["removed_fraction"] for row in per_page if row["removed_fraction"] is not None] | |
| return { | |
| "threshold": float(threshold_key), | |
| "threshold_key": threshold_key, | |
| "included_sweep_count": len(per_page), | |
| "missing_sweeps": missing_sweeps, | |
| "admitted_operational": not issues, | |
| "issues": issues, | |
| "mean_codepoint_absolute_improvement": sum(codepoint_values) / len(codepoint_values) if codepoint_values else None, | |
| "worst_codepoint_absolute_improvement": min(codepoint_values) if codepoint_values else None, | |
| "mean_grapheme_absolute_improvement": sum(grapheme_values) / len(grapheme_values) if grapheme_values else None, | |
| "worst_grapheme_absolute_improvement": min(grapheme_values) if grapheme_values else None, | |
| "mean_removed_fraction": sum(removed_fractions) / len(removed_fractions) if removed_fractions else None, | |
| "per_page": per_page, | |
| } | |
| def _rank_key(row: dict[str, Any]) -> tuple[int, float, float, float]: | |
| admitted = 1 if row["admitted_operational"] else 0 | |
| worst = float(row["worst_codepoint_absolute_improvement"] or -999.0) | |
| mean = float(row["mean_codepoint_absolute_improvement"] or -999.0) | |
| removed = float(row["mean_removed_fraction"] or 1.0) | |
| return admitted, worst, mean, -removed | |
| def _write_markdown(path: Path, report: dict[str, Any]) -> None: | |
| selected = report.get("selected_threshold") | |
| lines = [ | |
| "# OCR Line Pruning Threshold Admission", | |
| "", | |
| f"- selected threshold: {selected if selected is not None else 'none'}", | |
| f"- selected threshold id: {report.get('selected_threshold_id') or 'none'}", | |
| f"- admitted candidate count: {report['admitted_candidate_count']}", | |
| f"- claim safe: {report['claim_safe']}", | |
| "", | |
| "## Candidates", | |
| "", | |
| ] | |
| for row in report["candidates"]: | |
| lines.append( | |
| "- threshold {threshold:.2f}: admitted={admitted} worst_cp={worst} mean_cp={mean} issues={issues}".format( | |
| threshold=row["threshold"], | |
| admitted=row["admitted_operational"], | |
| worst=row["worst_codepoint_absolute_improvement"], | |
| mean=row["mean_codepoint_absolute_improvement"], | |
| issues="; ".join(row["issues"]) or "none", | |
| ) | |
| ) | |
| path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") | |
| def admit_threshold(args: argparse.Namespace) -> dict[str, Any]: | |
| out = Path(args.out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| sweeps = [_load_sweep(Path(path)) for path in args.sweep] | |
| threshold_sets = [set(sweep["rows"].keys()) for sweep in sweeps] | |
| common_thresholds = sorted(set.intersection(*threshold_sets)) if threshold_sets else [] | |
| if args.require_common_thresholds and not common_thresholds: | |
| raise ValueError("no common thresholds are present across all sweeps") | |
| candidate_keys = common_thresholds if args.require_common_thresholds else sorted(set.union(*threshold_sets)) | |
| candidates = [_candidate_summary(key, sweeps, args) for key in candidate_keys] | |
| candidates.sort(key=_rank_key, reverse=True) | |
| admitted = [row for row in candidates if row["admitted_operational"]] | |
| selected = admitted[0] if admitted else None | |
| claim_safe = False | |
| report = { | |
| "created_at": datetime.now(timezone.utc).isoformat(), | |
| "tool": "admit_page_ocr_line_pruning_threshold.py", | |
| "machine": { | |
| "platform": platform.platform(), | |
| "python": platform.python_version(), | |
| }, | |
| "claim_safe": claim_safe, | |
| "claim_boundary": ( | |
| "Cross-page operational threshold admission only. Inputs are draft page references and " | |
| "per-page comparisons are not claim-safe, so this cannot certify final OCR quality." | |
| ), | |
| "sweeps": [ | |
| { | |
| "path": sweep["path"], | |
| "sha256": sweep["sha256"], | |
| "label": sweep["label"], | |
| "claim_safe": sweep["claim_safe"], | |
| "candidate_count": sweep["candidate_count"], | |
| } | |
| for sweep in sweeps | |
| ], | |
| "requirements": { | |
| "min_sweeps": args.min_sweeps, | |
| "require_common_thresholds": args.require_common_thresholds, | |
| "min_codepoint_improvement": args.min_codepoint_improvement, | |
| "max_codepoint_regression": args.max_codepoint_regression, | |
| "max_grapheme_regression": args.max_grapheme_regression, | |
| }, | |
| "common_thresholds": [float(key) for key in common_thresholds], | |
| "candidate_count": len(candidates), | |
| "admitted_candidate_count": len(admitted), | |
| "selected_threshold": selected["threshold"] if selected else None, | |
| "selected_threshold_id": f"conf{int(round(selected['threshold'] * 100)):03d}" if selected else None, | |
| "selected_candidate": selected, | |
| "candidates": candidates, | |
| } | |
| _write_json(out / REPORT_JSON, report) | |
| _write_markdown(out / REPORT_MD, report) | |
| return report | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--sweep", type=Path, action="append", required=True) | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument("--min-sweeps", type=int, default=2) | |
| parser.add_argument("--require-common-thresholds", action="store_true") | |
| parser.add_argument("--min-codepoint-improvement", type=float, default=0.0) | |
| parser.add_argument("--max-codepoint-regression", type=float, default=0.0) | |
| parser.add_argument("--max-grapheme-regression", type=float, default=0.0) | |
| args = parser.parse_args(argv) | |
| report = admit_threshold(args) | |
| print(Path(args.out) / REPORT_JSON) | |
| print( | |
| f"selected_threshold={report['selected_threshold']} " | |
| f"admitted={report['admitted_candidate_count']} candidates={report['candidate_count']} " | |
| f"claim_safe={report['claim_safe']}" | |
| ) | |
| return 0 if report["admitted_candidate_count"] else 1 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |