ocr-projects-archive / ocr-tech /tools /audit_admitted_page_ocr_candidate.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
20.8 kB
#!/usr/bin/env python3
"""Audit residual page OCR failures for an admitted pruning candidate."""
from __future__ import annotations
import argparse
import csv
import json
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from ocrtech.manifest import sha256_file
REPORT_JSON = "admitted-page-ocr-candidate-audit.json"
REPORT_MD = "admitted-page-ocr-candidate-audit.md"
def _read_json(path: Path, label: str) -> dict[str, Any]:
if not path.exists():
raise FileNotFoundError(f"{label} does not exist: {path}")
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc
if not isinstance(payload, dict):
raise ValueError(f"{label} must contain a JSON object: {path}")
return payload
def _write_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def _read_csv(path: Path) -> list[dict[str, str]]:
if not path.exists():
raise FileNotFoundError(f"missing CSV: {path}")
with path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle))
if not rows:
raise ValueError(f"CSV has no rows: {path}")
return rows
def _float(value: Any, default: float = 0.0) -> float:
try:
return float(value)
except (TypeError, ValueError):
return default
def _int(value: Any, default: int = 0) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default
def _ratio(numerator: int | float, denominator: int | float) -> float | None:
if denominator == 0:
return None
return float(numerator) / float(denominator)
def _parse_bbox(value: str, *, row_label: str, field_name: str) -> tuple[float, float, float, float] | None:
if not value:
return None
try:
parsed = json.loads(value)
except json.JSONDecodeError as exc:
raise ValueError(f"invalid {field_name} JSON for {row_label}: {value}") from exc
if not isinstance(parsed, list) or len(parsed) != 4:
raise ValueError(f"{field_name} for {row_label} must be [x, y, width, height]: {value}")
try:
x, y, width, height = (float(item) for item in parsed)
except (TypeError, ValueError) as exc:
raise ValueError(f"{field_name} for {row_label} contains a non-numeric value: {value}") from exc
if width < 0 or height < 0:
raise ValueError(f"{field_name} for {row_label} has negative dimensions: {value}")
return (x, y, width, height)
def _row_label(row: dict[str, str]) -> str:
return (
row.get("reference_line_id")
or row.get("predicted_line_id")
or row.get("sample_id")
or "unknown-row"
)
def _all_bboxes(rows: list[dict[str, str]]) -> list[tuple[float, float, float, float]]:
bboxes = []
for row in rows:
label = _row_label(row)
for field_name in ["reference_bbox", "predicted_bbox"]:
bbox = _parse_bbox(row.get(field_name, ""), row_label=label, field_name=field_name)
if bbox is not None:
bboxes.append(bbox)
return bboxes
def _page_extent(rows: list[dict[str, str]]) -> dict[str, float] | None:
bboxes = _all_bboxes(rows)
if not bboxes:
return None
min_x = min(bbox[0] for bbox in bboxes)
min_y = min(bbox[1] for bbox in bboxes)
max_x = max(bbox[0] + bbox[2] for bbox in bboxes)
max_y = max(bbox[1] + bbox[3] for bbox in bboxes)
return {
"min_x": min_x,
"min_y": min_y,
"max_x": max_x,
"max_y": max_y,
"width": max_x - min_x,
"height": max_y - min_y,
}
def _third_zone(value: float, start: float, span: float, labels: tuple[str, str, str]) -> str:
if span <= 0:
return "unknown"
ratio = (value - start) / span
if ratio < 1 / 3:
return labels[0]
if ratio < 2 / 3:
return labels[1]
return labels[2]
def _width_bin(width_ratio: float | None) -> str:
if width_ratio is None:
return "unknown"
if width_ratio >= 0.45:
return "long"
if width_ratio >= 0.18:
return "medium"
return "short"
def _height_bin(height_ratio: float | None) -> str:
if height_ratio is None:
return "unknown"
if height_ratio >= 0.06:
return "tall"
if height_ratio >= 0.012:
return "line"
return "small"
def _geometry_record(
bbox: tuple[float, float, float, float] | None,
extent: dict[str, float] | None,
) -> dict[str, Any] | None:
if bbox is None:
return None
x, y, width, height = bbox
center_x = x + width / 2
center_y = y + height / 2
page_width = extent["width"] if extent else 0.0
page_height = extent["height"] if extent else 0.0
width_ratio = _ratio(width, page_width)
height_ratio = _ratio(height, page_height)
return {
"x": x,
"y": y,
"width": width,
"height": height,
"center_x": center_x,
"center_y": center_y,
"area": width * height,
"width_ratio": width_ratio,
"height_ratio": height_ratio,
"horizontal_zone": (
_third_zone(center_x, extent["min_x"], page_width, ("left", "center", "right"))
if extent
else "unknown"
),
"vertical_zone": (
_third_zone(center_y, extent["min_y"], page_height, ("top", "middle", "bottom"))
if extent
else "unknown"
),
"width_bin": _width_bin(width_ratio),
"height_bin": _height_bin(height_ratio),
}
def _increment(mapping: dict[str, int], key: str) -> None:
mapping[key] = mapping.get(key, 0) + 1
def _mean(values: list[float]) -> float | None:
if not values:
return None
return sum(values) / len(values)
def _artifact_check(path: Path, expected_sha256: Any, label: str) -> dict[str, Any]:
check = {
"label": label,
"path": str(path),
"expected_sha256": expected_sha256 if isinstance(expected_sha256, str) else None,
"actual_sha256": None,
"verified": False,
"issue": None,
}
if not isinstance(expected_sha256, str) or not expected_sha256:
check["issue"] = f"{label} has no recorded sha256 in admission report"
return check
if not path.exists():
check["issue"] = f"{label} artifact does not exist: {path}"
return check
actual = sha256_file(path)
check["actual_sha256"] = actual
if actual != expected_sha256:
check["issue"] = f"{label} sha256 mismatch: recorded={expected_sha256} actual={actual}"
return check
check["verified"] = True
return check
def _artifact_integrity(page: dict[str, Any], score_dir: Path) -> dict[str, Any]:
documents_dir_value = page.get("documents_dir")
comparison_dir_value = page.get("comparison_dir")
documents_dir = Path(str(documents_dir_value)) if documents_dir_value not in (None, "") else None
comparison_dir = Path(str(comparison_dir_value)) if comparison_dir_value not in (None, "") else None
requested_checks: list[tuple[Path | None, str, str]] = [
(score_dir / "page-eval-run.json", "page_eval_run_sha256", "page_eval_run"),
(score_dir / "metrics.json", "metrics_sha256", "metrics"),
]
if comparison_dir is not None:
requested_checks.append((comparison_dir / "comparison.json", "comparison_sha256", "comparison"))
if documents_dir is not None:
requested_checks.extend(
[
(documents_dir / "ocr-line-filter-run.json", "documents_filter_run_sha256", "documents_filter_run"),
(documents_dir / "ocr-line-filter-lines.jsonl", "documents_line_report_sha256", "documents_line_report"),
(documents_dir / "ocr-line-filter-audit.json", "documents_audit_sha256", "documents_audit"),
]
)
checks = [_artifact_check(path, page.get(hash_field), label) for path, hash_field, label in requested_checks if path is not None]
issues = [str(check["issue"]) for check in checks if check.get("issue")]
return {
"verified": not issues,
"issues": issues,
"checks": checks,
}
def _geometry_summary(
rows: list[dict[str, str]],
*,
bbox_field: str,
extent: dict[str, float] | None,
) -> dict[str, Any]:
by_horizontal_zone: dict[str, int] = {}
by_vertical_zone: dict[str, int] = {}
by_width_bin: dict[str, int] = {}
by_height_bin: dict[str, int] = {}
width_ratios: list[float] = []
height_ratios: list[float] = []
missing_bbox_count = 0
for row in rows:
bbox = _parse_bbox(row.get(bbox_field, ""), row_label=_row_label(row), field_name=bbox_field)
geometry = _geometry_record(bbox, extent)
if geometry is None:
missing_bbox_count += 1
continue
_increment(by_horizontal_zone, str(geometry["horizontal_zone"]))
_increment(by_vertical_zone, str(geometry["vertical_zone"]))
_increment(by_width_bin, str(geometry["width_bin"]))
_increment(by_height_bin, str(geometry["height_bin"]))
if geometry["width_ratio"] is not None:
width_ratios.append(float(geometry["width_ratio"]))
if geometry["height_ratio"] is not None:
height_ratios.append(float(geometry["height_ratio"]))
return {
"count": len(rows),
"bbox_field": bbox_field,
"missing_bbox_count": missing_bbox_count,
"by_horizontal_zone": by_horizontal_zone,
"by_vertical_zone": by_vertical_zone,
"by_width_bin": by_width_bin,
"by_height_bin": by_height_bin,
"mean_width_ratio": _mean(width_ratios),
"mean_height_ratio": _mean(height_ratios),
}
def _top_rows(
rows: list[dict[str, str]],
limit: int,
sort_key: str,
*,
extent: dict[str, float] | None,
) -> list[dict[str, Any]]:
def key(row: dict[str, str]) -> tuple[float, int]:
return (_float(row.get(sort_key)), _int(row.get("text_codepoint_ref_len")))
selected = sorted(rows, key=key, reverse=True)[:limit]
enriched = []
for row in selected:
label = _row_label(row)
reference_bbox = _parse_bbox(row.get("reference_bbox", ""), row_label=label, field_name="reference_bbox")
predicted_bbox = _parse_bbox(row.get("predicted_bbox", ""), row_label=label, field_name="predicted_bbox")
enriched.append(
{
"sample_id": row.get("sample_id"),
"kind": row.get("kind"),
"reference_line_id": row.get("reference_line_id"),
"predicted_line_id": row.get("predicted_line_id"),
"iou": _float(row.get("iou")),
"reference_bbox": row.get("reference_bbox"),
"predicted_bbox": row.get("predicted_bbox"),
"reference_geometry": _geometry_record(reference_bbox, extent),
"predicted_geometry": _geometry_record(predicted_bbox, extent),
"reference_text": row.get("reference_text"),
"predicted_text": row.get("predicted_text"),
"text_codepoint_ref_len": _int(row.get("text_codepoint_ref_len")),
"text_cer_codepoint": _float(row.get("text_cer_codepoint")),
"text_cer_grapheme": _float(row.get("text_cer_grapheme")),
}
)
return enriched
def _audit_page(page: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]:
label = str(page.get("label") or "")
score_dir = Path(str(page.get("score_dir") or ""))
if not score_dir.exists():
raise FileNotFoundError(f"selected candidate score_dir does not exist for {label}: {score_dir}")
alignment_path = score_dir / "line-alignment.csv"
rows = _read_csv(alignment_path)
matched = [row for row in rows if row.get("kind") == "matched"]
ref_unmatched = [row for row in rows if row.get("kind") == "reference_unmatched"]
pred_unmatched = [row for row in rows if row.get("kind") == "prediction_unmatched"]
high_iou_ref = [row for row in ref_unmatched if _float(row.get("iou")) >= args.high_iou_unmatched_threshold]
possible_near_miss_ref = [row for row in ref_unmatched if _float(row.get("iou")) >= args.near_miss_iou_threshold]
high_text_error = [
row
for row in matched
if _float(row.get("text_cer_codepoint")) >= args.bad_codepoint_cer
or _float(row.get("text_cer_grapheme")) >= args.bad_grapheme_cer
]
reference_count = len(matched) + len(ref_unmatched)
prediction_count = len(matched) + len(pred_unmatched)
extent = _page_extent(rows)
artifact_integrity = _artifact_integrity(page, score_dir)
return {
"label": label,
"score_dir": str(score_dir),
"line_alignment": str(alignment_path),
"line_alignment_sha256": sha256_file(alignment_path),
"artifact_integrity": artifact_integrity,
"page_extent": extent,
"kind_counts": {
"matched": len(matched),
"reference_unmatched": len(ref_unmatched),
"prediction_unmatched": len(pred_unmatched),
"total_rows": len(rows),
},
"rates": {
"reference_recall": _ratio(len(matched), reference_count),
"prediction_precision": _ratio(len(matched), prediction_count),
"reference_unmatched_rate": _ratio(len(ref_unmatched), reference_count),
"prediction_unmatched_rate": _ratio(len(pred_unmatched), prediction_count),
},
"residual_counts": {
"high_iou_unmatched_references": len(high_iou_ref),
"near_miss_unmatched_references": len(possible_near_miss_ref),
"matched_high_text_error": len(high_text_error),
},
"geometry_summary": {
"unmatched_references": _geometry_summary(ref_unmatched, bbox_field="reference_bbox", extent=extent),
"unmatched_predictions": _geometry_summary(pred_unmatched, bbox_field="predicted_bbox", extent=extent),
"near_miss_unmatched_references": _geometry_summary(
possible_near_miss_ref,
bbox_field="reference_bbox",
extent=extent,
),
"matched_high_text_error_references": _geometry_summary(
high_text_error,
bbox_field="reference_bbox",
extent=extent,
),
"matched_high_text_error_predictions": _geometry_summary(
high_text_error,
bbox_field="predicted_bbox",
extent=extent,
),
},
"top_unmatched_references": _top_rows(ref_unmatched, args.top_n, "text_codepoint_ref_len", extent=extent),
"top_unmatched_predictions": _top_rows(pred_unmatched, args.top_n, "iou", extent=extent),
"top_high_iou_unmatched_references": _top_rows(high_iou_ref, args.top_n, "iou", extent=extent),
"top_matched_high_text_error": _top_rows(high_text_error, args.top_n, "text_cer_codepoint", extent=extent),
}
def _write_markdown(path: Path, report: dict[str, Any]) -> None:
lines = [
"# Admitted Page OCR Candidate Audit",
"",
f"- selected threshold: {report['selected_threshold']}",
f"- page count: {report['page_count']}",
f"- total matched: {report['totals']['matched']}",
f"- total unmatched references: {report['totals']['reference_unmatched']}",
f"- total unmatched predictions: {report['totals']['prediction_unmatched']}",
f"- artifact integrity passed: {report['artifact_integrity_passed']}",
f"- claim safe: {report['claim_safe']}",
"",
"## Pages",
"",
]
for page in report["pages"]:
rates = page["rates"]
geometry = page["geometry_summary"]["unmatched_references"]
lines.append(
"- {label}: matched={matched} ref_unmatched={ref_unmatched} pred_unmatched={pred_unmatched} "
"recall={recall} precision={precision} artifact_integrity={integrity} ref_miss_vertical={vertical}".format(
label=page["label"],
matched=page["kind_counts"]["matched"],
ref_unmatched=page["kind_counts"]["reference_unmatched"],
pred_unmatched=page["kind_counts"]["prediction_unmatched"],
recall=rates["reference_recall"],
precision=rates["prediction_precision"],
integrity=page["artifact_integrity"]["verified"],
vertical=geometry["by_vertical_zone"],
)
)
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
def audit_candidate(args: argparse.Namespace) -> dict[str, Any]:
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
admission_path = Path(args.admission_report)
admission = _read_json(admission_path, "threshold admission report")
selected = admission.get("selected_candidate")
if not isinstance(selected, dict):
raise ValueError(f"admission report has no selected_candidate object: {admission_path}")
per_page = selected.get("per_page")
if not isinstance(per_page, list) or not per_page:
raise ValueError(f"selected_candidate has no per_page rows: {admission_path}")
pages = [_audit_page(page, args) for page in per_page if isinstance(page, dict)]
totals = {
"matched": sum(page["kind_counts"]["matched"] for page in pages),
"reference_unmatched": sum(page["kind_counts"]["reference_unmatched"] for page in pages),
"prediction_unmatched": sum(page["kind_counts"]["prediction_unmatched"] for page in pages),
"high_iou_unmatched_references": sum(page["residual_counts"]["high_iou_unmatched_references"] for page in pages),
"near_miss_unmatched_references": sum(page["residual_counts"]["near_miss_unmatched_references"] for page in pages),
"matched_high_text_error": sum(page["residual_counts"]["matched_high_text_error"] for page in pages),
}
artifact_integrity_issues = [
f"{page['label']}: {issue}"
for page in pages
for issue in page["artifact_integrity"]["issues"]
]
report = {
"created_at": datetime.now(timezone.utc).isoformat(),
"tool": "audit_admitted_page_ocr_candidate.py",
"claim_safe": False,
"artifact_integrity_passed": not artifact_integrity_issues,
"artifact_integrity_issues": artifact_integrity_issues,
"claim_boundary": (
"Residual operational audit for an admitted pruning threshold. Inputs are draft references and "
"non-claim per-page scores; this guides layout work but does not certify final OCR quality."
),
"admission_report": str(admission_path),
"admission_report_sha256": sha256_file(admission_path),
"selected_threshold": admission.get("selected_threshold"),
"selected_threshold_id": admission.get("selected_threshold_id"),
"page_count": len(pages),
"thresholds": {
"high_iou_unmatched_threshold": args.high_iou_unmatched_threshold,
"near_miss_iou_threshold": args.near_miss_iou_threshold,
"bad_codepoint_cer": args.bad_codepoint_cer,
"bad_grapheme_cer": args.bad_grapheme_cer,
},
"totals": totals,
"pages": pages,
}
_write_json(out / REPORT_JSON, report)
_write_markdown(out / REPORT_MD, report)
return report
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--admission-report", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--high-iou-unmatched-threshold", type=float, default=0.5)
parser.add_argument("--near-miss-iou-threshold", type=float, default=0.3)
parser.add_argument("--bad-codepoint-cer", type=float, default=0.25)
parser.add_argument("--bad-grapheme-cer", type=float, default=0.35)
parser.add_argument("--top-n", type=int, default=20)
args = parser.parse_args(argv)
report = audit_candidate(args)
print(Path(args.out) / REPORT_JSON)
print(
f"threshold={report['selected_threshold']} pages={report['page_count']} "
f"matched={report['totals']['matched']} ref_unmatched={report['totals']['reference_unmatched']} "
f"pred_unmatched={report['totals']['prediction_unmatched']} claim_safe={report['claim_safe']}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())