ocr-projects-archive / ocr-tech /tools /audit_candidate_page_line_application_effect.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
11.5 kB
#!/usr/bin/env python3
"""Audit page candidate-line insertion effects against a baseline page score."""
from __future__ import annotations
import argparse
import csv
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from ocrtech.manifest import sha256_file
REPORT_JSON = "candidate-page-line-application-effect-audit.json"
INSERTED_TSV = "candidate-page-line-application-inserted-lines.tsv"
OBJECTIVE_METRICS = (
"text_cer_codepoint_micro",
"text_cer_grapheme_micro",
"line_detection_f1_micro",
"line_detection_precision_micro",
"line_detection_recall_micro",
"line_unmatched_reference_count",
"line_unmatched_prediction_count",
"line_blank_prediction_count",
"matched_line_reading_order_pair_accuracy",
"matched_line_text_cer_codepoint_micro",
)
def _read_json(path: Path, label: str) -> dict[str, Any]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise ValueError(f"{label} is missing: {path}") from exc
except json.JSONDecodeError as exc:
raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc
if not isinstance(payload, dict):
raise ValueError(f"{label} must contain a JSON object: {path}")
return payload
def _float(value: Any) -> float | None:
if value in ("", None):
return None
try:
return float(value)
except (TypeError, ValueError):
return None
def _metric_delta(candidate: Any, baseline: Any) -> float | None:
candidate_float = _float(candidate)
baseline_float = _float(baseline)
if candidate_float is None or baseline_float is None:
return None
return candidate_float - baseline_float
def _read_alignment_by_prediction(path: Path) -> dict[str, dict[str, Any]]:
try:
handle = path.open("r", encoding="utf-8", newline="")
except FileNotFoundError as exc:
raise ValueError(f"line alignment CSV is missing: {path}") from exc
rows: dict[str, dict[str, Any]] = {}
with handle:
reader = csv.DictReader(handle)
if reader.fieldnames is None:
raise ValueError(f"line alignment CSV has no header: {path}")
for row_number, row in enumerate(reader, start=2):
predicted_line_id = str(row.get("predicted_line_id") or "").strip()
if not predicted_line_id:
continue
if predicted_line_id in rows:
raise ValueError(f"duplicate predicted_line_id in line alignment at {path}:{row_number}: {predicted_line_id}")
rows[predicted_line_id] = dict(row)
return rows
def _inserted_rows(application: dict[str, Any], alignment_by_prediction: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
applied_rows = application.get("applied_rows")
if not isinstance(applied_rows, list):
raise ValueError("application report missing applied_rows list")
inserted: list[dict[str, Any]] = []
for row in applied_rows:
if not isinstance(row, dict):
raise ValueError("application applied_rows entries must be objects")
inserted_line_id = str(row.get("inserted_line_id") or "").strip()
if not inserted_line_id:
raise ValueError(f"applied row missing inserted_line_id: {row.get('row_id')!r}")
alignment = alignment_by_prediction.get(inserted_line_id)
if alignment is None:
inserted.append(
{
"row_id": row.get("row_id") or "",
"sample_id": row.get("sample_id") or "",
"candidate_line_id": row.get("candidate_line_id") or "",
"inserted_line_id": inserted_line_id,
"alignment_kind": "alignment_missing",
"reference_line_id": "",
"iou": None,
"text_codepoint_edits": None,
"text_codepoint_ref_len": None,
"text_cer_codepoint": None,
"inserted_text": row.get("inserted_text") or "",
"selection_policy": row.get("selection_policy") or "",
}
)
continue
inserted.append(
{
"row_id": row.get("row_id") or "",
"sample_id": row.get("sample_id") or alignment.get("sample_id") or "",
"candidate_line_id": row.get("candidate_line_id") or "",
"inserted_line_id": inserted_line_id,
"alignment_kind": alignment.get("kind") or "",
"reference_line_id": alignment.get("reference_line_id") or "",
"iou": _float(alignment.get("iou")),
"text_codepoint_edits": _float(alignment.get("text_codepoint_edits")),
"text_codepoint_ref_len": _float(alignment.get("text_codepoint_ref_len")),
"text_cer_codepoint": _float(alignment.get("text_cer_codepoint")),
"inserted_text": row.get("inserted_text") or "",
"selection_policy": row.get("selection_policy") or "",
}
)
return inserted
def _write_inserted_tsv(path: Path, rows: list[dict[str, Any]]) -> None:
fieldnames = [
"row_id",
"sample_id",
"candidate_line_id",
"inserted_line_id",
"alignment_kind",
"reference_line_id",
"iou",
"text_codepoint_edits",
"text_codepoint_ref_len",
"text_cer_codepoint",
"selection_policy",
]
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, delimiter="\t", fieldnames=fieldnames)
writer.writeheader()
for row in rows:
writer.writerow({field: row.get(field, "") for field in fieldnames})
def _metric_summary(baseline: dict[str, Any], candidate: dict[str, Any]) -> dict[str, dict[str, Any]]:
summary: dict[str, dict[str, Any]] = {}
for metric in OBJECTIVE_METRICS:
baseline_value = baseline.get(metric)
candidate_value = candidate.get(metric)
summary[metric] = {
"baseline": baseline_value,
"candidate": candidate_value,
"delta_candidate_minus_baseline": _metric_delta(candidate_value, baseline_value),
}
return summary
def _book_ocr_issues(metrics: dict[str, dict[str, Any]], inserted_counts: dict[str, int]) -> list[str]:
issues: list[str] = []
cp_delta = metrics["text_cer_codepoint_micro"]["delta_candidate_minus_baseline"]
if cp_delta is None or cp_delta > 0:
issues.append(f"text_cer_codepoint_micro does not improve or tie: delta={cp_delta}")
graph_delta = metrics["text_cer_grapheme_micro"]["delta_candidate_minus_baseline"]
if graph_delta is None or graph_delta > 0:
issues.append(f"text_cer_grapheme_micro does not improve or tie: delta={graph_delta}")
line_f1_delta = metrics["line_detection_f1_micro"]["delta_candidate_minus_baseline"]
if line_f1_delta is None or line_f1_delta < 0:
issues.append(f"line_detection_f1_micro regresses: delta={line_f1_delta}")
extra_delta = metrics["line_unmatched_prediction_count"]["delta_candidate_minus_baseline"]
if extra_delta is None or extra_delta > 0:
issues.append(f"line_unmatched_prediction_count increases: delta={extra_delta}")
missed_delta = metrics["line_unmatched_reference_count"]["delta_candidate_minus_baseline"]
if missed_delta is None or missed_delta > 0:
issues.append(f"line_unmatched_reference_count increases: delta={missed_delta}")
order_delta = metrics["matched_line_reading_order_pair_accuracy"]["delta_candidate_minus_baseline"]
if order_delta is None or order_delta < 0:
issues.append(f"matched_line_reading_order_pair_accuracy regresses: delta={order_delta}")
if inserted_counts.get("prediction_unmatched", 0) > 0:
issues.append(f"inserted candidate rows include unmatched extra predictions: {inserted_counts['prediction_unmatched']}")
if inserted_counts.get("alignment_missing", 0) > 0:
issues.append(f"inserted candidate rows missing from line alignment: {inserted_counts['alignment_missing']}")
return issues
def audit_application_effect(args: argparse.Namespace) -> dict[str, Any]:
application_path = Path(args.application_report)
baseline_metrics_path = Path(args.baseline_score_dir) / "metrics.json"
candidate_metrics_path = Path(args.candidate_score_dir) / "metrics.json"
alignment_path = Path(args.candidate_score_dir) / "line-alignment.csv"
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
application = _read_json(application_path, "candidate page-line application report")
baseline_metrics = _read_json(baseline_metrics_path, "baseline metrics")
candidate_metrics = _read_json(candidate_metrics_path, "candidate metrics")
alignment = _read_alignment_by_prediction(alignment_path)
inserted = _inserted_rows(application, alignment)
inserted_counts: dict[str, int] = {}
for row in inserted:
kind = str(row.get("alignment_kind") or "unknown")
inserted_counts[kind] = inserted_counts.get(kind, 0) + 1
metrics = _metric_summary(baseline_metrics, candidate_metrics)
issues = _book_ocr_issues(metrics, inserted_counts)
inserted_tsv = out / INSERTED_TSV
_write_inserted_tsv(inserted_tsv, inserted)
report_path = out / REPORT_JSON
report = {
"tool": "tools/audit_candidate_page_line_application_effect.py",
"created_at_utc": datetime.now(UTC).isoformat(),
"application_report": str(application_path),
"application_report_sha256": sha256_file(application_path),
"baseline_score_dir": str(args.baseline_score_dir),
"baseline_metrics_sha256": sha256_file(baseline_metrics_path),
"candidate_score_dir": str(args.candidate_score_dir),
"candidate_metrics_sha256": sha256_file(candidate_metrics_path),
"line_alignment_sha256": sha256_file(alignment_path),
"inserted_line_count": len(inserted),
"inserted_alignment_counts": inserted_counts,
"objective_metric_comparison": metrics,
"book_ocr_objective_safe": not issues,
"issues": issues,
"inserted_lines_tsv": str(inserted_tsv),
"inserted_lines_tsv_sha256": sha256_file(inserted_tsv),
"claim_boundary": (
"Page-candidate application objective audit. It compares a candidate projection to a baseline "
"on page OCR objective metrics and inserted-line alignment; it does not certify claim-grade OCR."
),
}
report["report_json"] = str(report_path)
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return report
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--application-report", required=True, type=Path)
parser.add_argument("--baseline-score-dir", required=True, type=Path)
parser.add_argument("--candidate-score-dir", required=True, type=Path)
parser.add_argument("--out", required=True, type=Path)
args = parser.parse_args()
report = audit_application_effect(args)
print(report["report_json"])
print(
f"book_ocr_objective_safe={report['book_ocr_objective_safe']} "
f"inserted={report['inserted_line_count']} issues={len(report['issues'])}"
)
if __name__ == "__main__":
main()