Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/audit_admitted_page_ocr_candidate.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 20.8 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/audit_admitted_page_ocr_candidate.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/audit_admitted_page_ocr_candidate.py
-
curl -L -o audit_admitted_page_ocr_candidate.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/audit_admitted_page_ocr_candidate.py
20.8 kB
| #!/usr/bin/env python3 | |
| """Audit residual page OCR failures for an admitted pruning candidate.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from typing import Any | |
| from ocrtech.manifest import sha256_file | |
| REPORT_JSON = "admitted-page-ocr-candidate-audit.json" | |
| REPORT_MD = "admitted-page-ocr-candidate-audit.md" | |
| def _read_json(path: Path, label: str) -> dict[str, Any]: | |
| if not path.exists(): | |
| raise FileNotFoundError(f"{label} does not exist: {path}") | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise ValueError(f"{label} must contain a JSON object: {path}") | |
| return payload | |
| def _write_json(path: Path, payload: Any) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| def _read_csv(path: Path) -> list[dict[str, str]]: | |
| if not path.exists(): | |
| raise FileNotFoundError(f"missing CSV: {path}") | |
| with path.open("r", encoding="utf-8", newline="") as handle: | |
| rows = list(csv.DictReader(handle)) | |
| if not rows: | |
| raise ValueError(f"CSV has no rows: {path}") | |
| return rows | |
| def _float(value: Any, default: float = 0.0) -> float: | |
| try: | |
| return float(value) | |
| except (TypeError, ValueError): | |
| return default | |
| def _int(value: Any, default: int = 0) -> int: | |
| try: | |
| return int(value) | |
| except (TypeError, ValueError): | |
| return default | |
| def _ratio(numerator: int | float, denominator: int | float) -> float | None: | |
| if denominator == 0: | |
| return None | |
| return float(numerator) / float(denominator) | |
| def _parse_bbox(value: str, *, row_label: str, field_name: str) -> tuple[float, float, float, float] | None: | |
| if not value: | |
| return None | |
| try: | |
| parsed = json.loads(value) | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"invalid {field_name} JSON for {row_label}: {value}") from exc | |
| if not isinstance(parsed, list) or len(parsed) != 4: | |
| raise ValueError(f"{field_name} for {row_label} must be [x, y, width, height]: {value}") | |
| try: | |
| x, y, width, height = (float(item) for item in parsed) | |
| except (TypeError, ValueError) as exc: | |
| raise ValueError(f"{field_name} for {row_label} contains a non-numeric value: {value}") from exc | |
| if width < 0 or height < 0: | |
| raise ValueError(f"{field_name} for {row_label} has negative dimensions: {value}") | |
| return (x, y, width, height) | |
| def _row_label(row: dict[str, str]) -> str: | |
| return ( | |
| row.get("reference_line_id") | |
| or row.get("predicted_line_id") | |
| or row.get("sample_id") | |
| or "unknown-row" | |
| ) | |
| def _all_bboxes(rows: list[dict[str, str]]) -> list[tuple[float, float, float, float]]: | |
| bboxes = [] | |
| for row in rows: | |
| label = _row_label(row) | |
| for field_name in ["reference_bbox", "predicted_bbox"]: | |
| bbox = _parse_bbox(row.get(field_name, ""), row_label=label, field_name=field_name) | |
| if bbox is not None: | |
| bboxes.append(bbox) | |
| return bboxes | |
| def _page_extent(rows: list[dict[str, str]]) -> dict[str, float] | None: | |
| bboxes = _all_bboxes(rows) | |
| if not bboxes: | |
| return None | |
| min_x = min(bbox[0] for bbox in bboxes) | |
| min_y = min(bbox[1] for bbox in bboxes) | |
| max_x = max(bbox[0] + bbox[2] for bbox in bboxes) | |
| max_y = max(bbox[1] + bbox[3] for bbox in bboxes) | |
| return { | |
| "min_x": min_x, | |
| "min_y": min_y, | |
| "max_x": max_x, | |
| "max_y": max_y, | |
| "width": max_x - min_x, | |
| "height": max_y - min_y, | |
| } | |
| def _third_zone(value: float, start: float, span: float, labels: tuple[str, str, str]) -> str: | |
| if span <= 0: | |
| return "unknown" | |
| ratio = (value - start) / span | |
| if ratio < 1 / 3: | |
| return labels[0] | |
| if ratio < 2 / 3: | |
| return labels[1] | |
| return labels[2] | |
| def _width_bin(width_ratio: float | None) -> str: | |
| if width_ratio is None: | |
| return "unknown" | |
| if width_ratio >= 0.45: | |
| return "long" | |
| if width_ratio >= 0.18: | |
| return "medium" | |
| return "short" | |
| def _height_bin(height_ratio: float | None) -> str: | |
| if height_ratio is None: | |
| return "unknown" | |
| if height_ratio >= 0.06: | |
| return "tall" | |
| if height_ratio >= 0.012: | |
| return "line" | |
| return "small" | |
| def _geometry_record( | |
| bbox: tuple[float, float, float, float] | None, | |
| extent: dict[str, float] | None, | |
| ) -> dict[str, Any] | None: | |
| if bbox is None: | |
| return None | |
| x, y, width, height = bbox | |
| center_x = x + width / 2 | |
| center_y = y + height / 2 | |
| page_width = extent["width"] if extent else 0.0 | |
| page_height = extent["height"] if extent else 0.0 | |
| width_ratio = _ratio(width, page_width) | |
| height_ratio = _ratio(height, page_height) | |
| return { | |
| "x": x, | |
| "y": y, | |
| "width": width, | |
| "height": height, | |
| "center_x": center_x, | |
| "center_y": center_y, | |
| "area": width * height, | |
| "width_ratio": width_ratio, | |
| "height_ratio": height_ratio, | |
| "horizontal_zone": ( | |
| _third_zone(center_x, extent["min_x"], page_width, ("left", "center", "right")) | |
| if extent | |
| else "unknown" | |
| ), | |
| "vertical_zone": ( | |
| _third_zone(center_y, extent["min_y"], page_height, ("top", "middle", "bottom")) | |
| if extent | |
| else "unknown" | |
| ), | |
| "width_bin": _width_bin(width_ratio), | |
| "height_bin": _height_bin(height_ratio), | |
| } | |
| def _increment(mapping: dict[str, int], key: str) -> None: | |
| mapping[key] = mapping.get(key, 0) + 1 | |
| def _mean(values: list[float]) -> float | None: | |
| if not values: | |
| return None | |
| return sum(values) / len(values) | |
| def _artifact_check(path: Path, expected_sha256: Any, label: str) -> dict[str, Any]: | |
| check = { | |
| "label": label, | |
| "path": str(path), | |
| "expected_sha256": expected_sha256 if isinstance(expected_sha256, str) else None, | |
| "actual_sha256": None, | |
| "verified": False, | |
| "issue": None, | |
| } | |
| if not isinstance(expected_sha256, str) or not expected_sha256: | |
| check["issue"] = f"{label} has no recorded sha256 in admission report" | |
| return check | |
| if not path.exists(): | |
| check["issue"] = f"{label} artifact does not exist: {path}" | |
| return check | |
| actual = sha256_file(path) | |
| check["actual_sha256"] = actual | |
| if actual != expected_sha256: | |
| check["issue"] = f"{label} sha256 mismatch: recorded={expected_sha256} actual={actual}" | |
| return check | |
| check["verified"] = True | |
| return check | |
| def _artifact_integrity(page: dict[str, Any], score_dir: Path) -> dict[str, Any]: | |
| documents_dir_value = page.get("documents_dir") | |
| comparison_dir_value = page.get("comparison_dir") | |
| documents_dir = Path(str(documents_dir_value)) if documents_dir_value not in (None, "") else None | |
| comparison_dir = Path(str(comparison_dir_value)) if comparison_dir_value not in (None, "") else None | |
| requested_checks: list[tuple[Path | None, str, str]] = [ | |
| (score_dir / "page-eval-run.json", "page_eval_run_sha256", "page_eval_run"), | |
| (score_dir / "metrics.json", "metrics_sha256", "metrics"), | |
| ] | |
| if comparison_dir is not None: | |
| requested_checks.append((comparison_dir / "comparison.json", "comparison_sha256", "comparison")) | |
| if documents_dir is not None: | |
| requested_checks.extend( | |
| [ | |
| (documents_dir / "ocr-line-filter-run.json", "documents_filter_run_sha256", "documents_filter_run"), | |
| (documents_dir / "ocr-line-filter-lines.jsonl", "documents_line_report_sha256", "documents_line_report"), | |
| (documents_dir / "ocr-line-filter-audit.json", "documents_audit_sha256", "documents_audit"), | |
| ] | |
| ) | |
| checks = [_artifact_check(path, page.get(hash_field), label) for path, hash_field, label in requested_checks if path is not None] | |
| issues = [str(check["issue"]) for check in checks if check.get("issue")] | |
| return { | |
| "verified": not issues, | |
| "issues": issues, | |
| "checks": checks, | |
| } | |
| def _geometry_summary( | |
| rows: list[dict[str, str]], | |
| *, | |
| bbox_field: str, | |
| extent: dict[str, float] | None, | |
| ) -> dict[str, Any]: | |
| by_horizontal_zone: dict[str, int] = {} | |
| by_vertical_zone: dict[str, int] = {} | |
| by_width_bin: dict[str, int] = {} | |
| by_height_bin: dict[str, int] = {} | |
| width_ratios: list[float] = [] | |
| height_ratios: list[float] = [] | |
| missing_bbox_count = 0 | |
| for row in rows: | |
| bbox = _parse_bbox(row.get(bbox_field, ""), row_label=_row_label(row), field_name=bbox_field) | |
| geometry = _geometry_record(bbox, extent) | |
| if geometry is None: | |
| missing_bbox_count += 1 | |
| continue | |
| _increment(by_horizontal_zone, str(geometry["horizontal_zone"])) | |
| _increment(by_vertical_zone, str(geometry["vertical_zone"])) | |
| _increment(by_width_bin, str(geometry["width_bin"])) | |
| _increment(by_height_bin, str(geometry["height_bin"])) | |
| if geometry["width_ratio"] is not None: | |
| width_ratios.append(float(geometry["width_ratio"])) | |
| if geometry["height_ratio"] is not None: | |
| height_ratios.append(float(geometry["height_ratio"])) | |
| return { | |
| "count": len(rows), | |
| "bbox_field": bbox_field, | |
| "missing_bbox_count": missing_bbox_count, | |
| "by_horizontal_zone": by_horizontal_zone, | |
| "by_vertical_zone": by_vertical_zone, | |
| "by_width_bin": by_width_bin, | |
| "by_height_bin": by_height_bin, | |
| "mean_width_ratio": _mean(width_ratios), | |
| "mean_height_ratio": _mean(height_ratios), | |
| } | |
| def _top_rows( | |
| rows: list[dict[str, str]], | |
| limit: int, | |
| sort_key: str, | |
| *, | |
| extent: dict[str, float] | None, | |
| ) -> list[dict[str, Any]]: | |
| def key(row: dict[str, str]) -> tuple[float, int]: | |
| return (_float(row.get(sort_key)), _int(row.get("text_codepoint_ref_len"))) | |
| selected = sorted(rows, key=key, reverse=True)[:limit] | |
| enriched = [] | |
| for row in selected: | |
| label = _row_label(row) | |
| reference_bbox = _parse_bbox(row.get("reference_bbox", ""), row_label=label, field_name="reference_bbox") | |
| predicted_bbox = _parse_bbox(row.get("predicted_bbox", ""), row_label=label, field_name="predicted_bbox") | |
| enriched.append( | |
| { | |
| "sample_id": row.get("sample_id"), | |
| "kind": row.get("kind"), | |
| "reference_line_id": row.get("reference_line_id"), | |
| "predicted_line_id": row.get("predicted_line_id"), | |
| "iou": _float(row.get("iou")), | |
| "reference_bbox": row.get("reference_bbox"), | |
| "predicted_bbox": row.get("predicted_bbox"), | |
| "reference_geometry": _geometry_record(reference_bbox, extent), | |
| "predicted_geometry": _geometry_record(predicted_bbox, extent), | |
| "reference_text": row.get("reference_text"), | |
| "predicted_text": row.get("predicted_text"), | |
| "text_codepoint_ref_len": _int(row.get("text_codepoint_ref_len")), | |
| "text_cer_codepoint": _float(row.get("text_cer_codepoint")), | |
| "text_cer_grapheme": _float(row.get("text_cer_grapheme")), | |
| } | |
| ) | |
| return enriched | |
| def _audit_page(page: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]: | |
| label = str(page.get("label") or "") | |
| score_dir = Path(str(page.get("score_dir") or "")) | |
| if not score_dir.exists(): | |
| raise FileNotFoundError(f"selected candidate score_dir does not exist for {label}: {score_dir}") | |
| alignment_path = score_dir / "line-alignment.csv" | |
| rows = _read_csv(alignment_path) | |
| matched = [row for row in rows if row.get("kind") == "matched"] | |
| ref_unmatched = [row for row in rows if row.get("kind") == "reference_unmatched"] | |
| pred_unmatched = [row for row in rows if row.get("kind") == "prediction_unmatched"] | |
| high_iou_ref = [row for row in ref_unmatched if _float(row.get("iou")) >= args.high_iou_unmatched_threshold] | |
| possible_near_miss_ref = [row for row in ref_unmatched if _float(row.get("iou")) >= args.near_miss_iou_threshold] | |
| high_text_error = [ | |
| row | |
| for row in matched | |
| if _float(row.get("text_cer_codepoint")) >= args.bad_codepoint_cer | |
| or _float(row.get("text_cer_grapheme")) >= args.bad_grapheme_cer | |
| ] | |
| reference_count = len(matched) + len(ref_unmatched) | |
| prediction_count = len(matched) + len(pred_unmatched) | |
| extent = _page_extent(rows) | |
| artifact_integrity = _artifact_integrity(page, score_dir) | |
| return { | |
| "label": label, | |
| "score_dir": str(score_dir), | |
| "line_alignment": str(alignment_path), | |
| "line_alignment_sha256": sha256_file(alignment_path), | |
| "artifact_integrity": artifact_integrity, | |
| "page_extent": extent, | |
| "kind_counts": { | |
| "matched": len(matched), | |
| "reference_unmatched": len(ref_unmatched), | |
| "prediction_unmatched": len(pred_unmatched), | |
| "total_rows": len(rows), | |
| }, | |
| "rates": { | |
| "reference_recall": _ratio(len(matched), reference_count), | |
| "prediction_precision": _ratio(len(matched), prediction_count), | |
| "reference_unmatched_rate": _ratio(len(ref_unmatched), reference_count), | |
| "prediction_unmatched_rate": _ratio(len(pred_unmatched), prediction_count), | |
| }, | |
| "residual_counts": { | |
| "high_iou_unmatched_references": len(high_iou_ref), | |
| "near_miss_unmatched_references": len(possible_near_miss_ref), | |
| "matched_high_text_error": len(high_text_error), | |
| }, | |
| "geometry_summary": { | |
| "unmatched_references": _geometry_summary(ref_unmatched, bbox_field="reference_bbox", extent=extent), | |
| "unmatched_predictions": _geometry_summary(pred_unmatched, bbox_field="predicted_bbox", extent=extent), | |
| "near_miss_unmatched_references": _geometry_summary( | |
| possible_near_miss_ref, | |
| bbox_field="reference_bbox", | |
| extent=extent, | |
| ), | |
| "matched_high_text_error_references": _geometry_summary( | |
| high_text_error, | |
| bbox_field="reference_bbox", | |
| extent=extent, | |
| ), | |
| "matched_high_text_error_predictions": _geometry_summary( | |
| high_text_error, | |
| bbox_field="predicted_bbox", | |
| extent=extent, | |
| ), | |
| }, | |
| "top_unmatched_references": _top_rows(ref_unmatched, args.top_n, "text_codepoint_ref_len", extent=extent), | |
| "top_unmatched_predictions": _top_rows(pred_unmatched, args.top_n, "iou", extent=extent), | |
| "top_high_iou_unmatched_references": _top_rows(high_iou_ref, args.top_n, "iou", extent=extent), | |
| "top_matched_high_text_error": _top_rows(high_text_error, args.top_n, "text_cer_codepoint", extent=extent), | |
| } | |
| def _write_markdown(path: Path, report: dict[str, Any]) -> None: | |
| lines = [ | |
| "# Admitted Page OCR Candidate Audit", | |
| "", | |
| f"- selected threshold: {report['selected_threshold']}", | |
| f"- page count: {report['page_count']}", | |
| f"- total matched: {report['totals']['matched']}", | |
| f"- total unmatched references: {report['totals']['reference_unmatched']}", | |
| f"- total unmatched predictions: {report['totals']['prediction_unmatched']}", | |
| f"- artifact integrity passed: {report['artifact_integrity_passed']}", | |
| f"- claim safe: {report['claim_safe']}", | |
| "", | |
| "## Pages", | |
| "", | |
| ] | |
| for page in report["pages"]: | |
| rates = page["rates"] | |
| geometry = page["geometry_summary"]["unmatched_references"] | |
| lines.append( | |
| "- {label}: matched={matched} ref_unmatched={ref_unmatched} pred_unmatched={pred_unmatched} " | |
| "recall={recall} precision={precision} artifact_integrity={integrity} ref_miss_vertical={vertical}".format( | |
| label=page["label"], | |
| matched=page["kind_counts"]["matched"], | |
| ref_unmatched=page["kind_counts"]["reference_unmatched"], | |
| pred_unmatched=page["kind_counts"]["prediction_unmatched"], | |
| recall=rates["reference_recall"], | |
| precision=rates["prediction_precision"], | |
| integrity=page["artifact_integrity"]["verified"], | |
| vertical=geometry["by_vertical_zone"], | |
| ) | |
| ) | |
| path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") | |
| def audit_candidate(args: argparse.Namespace) -> dict[str, Any]: | |
| out = Path(args.out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| admission_path = Path(args.admission_report) | |
| admission = _read_json(admission_path, "threshold admission report") | |
| selected = admission.get("selected_candidate") | |
| if not isinstance(selected, dict): | |
| raise ValueError(f"admission report has no selected_candidate object: {admission_path}") | |
| per_page = selected.get("per_page") | |
| if not isinstance(per_page, list) or not per_page: | |
| raise ValueError(f"selected_candidate has no per_page rows: {admission_path}") | |
| pages = [_audit_page(page, args) for page in per_page if isinstance(page, dict)] | |
| totals = { | |
| "matched": sum(page["kind_counts"]["matched"] for page in pages), | |
| "reference_unmatched": sum(page["kind_counts"]["reference_unmatched"] for page in pages), | |
| "prediction_unmatched": sum(page["kind_counts"]["prediction_unmatched"] for page in pages), | |
| "high_iou_unmatched_references": sum(page["residual_counts"]["high_iou_unmatched_references"] for page in pages), | |
| "near_miss_unmatched_references": sum(page["residual_counts"]["near_miss_unmatched_references"] for page in pages), | |
| "matched_high_text_error": sum(page["residual_counts"]["matched_high_text_error"] for page in pages), | |
| } | |
| artifact_integrity_issues = [ | |
| f"{page['label']}: {issue}" | |
| for page in pages | |
| for issue in page["artifact_integrity"]["issues"] | |
| ] | |
| report = { | |
| "created_at": datetime.now(timezone.utc).isoformat(), | |
| "tool": "audit_admitted_page_ocr_candidate.py", | |
| "claim_safe": False, | |
| "artifact_integrity_passed": not artifact_integrity_issues, | |
| "artifact_integrity_issues": artifact_integrity_issues, | |
| "claim_boundary": ( | |
| "Residual operational audit for an admitted pruning threshold. Inputs are draft references and " | |
| "non-claim per-page scores; this guides layout work but does not certify final OCR quality." | |
| ), | |
| "admission_report": str(admission_path), | |
| "admission_report_sha256": sha256_file(admission_path), | |
| "selected_threshold": admission.get("selected_threshold"), | |
| "selected_threshold_id": admission.get("selected_threshold_id"), | |
| "page_count": len(pages), | |
| "thresholds": { | |
| "high_iou_unmatched_threshold": args.high_iou_unmatched_threshold, | |
| "near_miss_iou_threshold": args.near_miss_iou_threshold, | |
| "bad_codepoint_cer": args.bad_codepoint_cer, | |
| "bad_grapheme_cer": args.bad_grapheme_cer, | |
| }, | |
| "totals": totals, | |
| "pages": pages, | |
| } | |
| _write_json(out / REPORT_JSON, report) | |
| _write_markdown(out / REPORT_MD, report) | |
| return report | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--admission-report", type=Path, required=True) | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument("--high-iou-unmatched-threshold", type=float, default=0.5) | |
| parser.add_argument("--near-miss-iou-threshold", type=float, default=0.3) | |
| parser.add_argument("--bad-codepoint-cer", type=float, default=0.25) | |
| parser.add_argument("--bad-grapheme-cer", type=float, default=0.35) | |
| parser.add_argument("--top-n", type=int, default=20) | |
| args = parser.parse_args(argv) | |
| report = audit_candidate(args) | |
| print(Path(args.out) / REPORT_JSON) | |
| print( | |
| f"threshold={report['selected_threshold']} pages={report['page_count']} " | |
| f"matched={report['totals']['matched']} ref_unmatched={report['totals']['reference_unmatched']} " | |
| f"pred_unmatched={report['totals']['prediction_unmatched']} claim_safe={report['claim_safe']}" | |
| ) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |