Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/audit_candidate_page_line_application_effect.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 11.5 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/audit_candidate_page_line_application_effect.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/audit_candidate_page_line_application_effect.py
-
curl -L -o audit_candidate_page_line_application_effect.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/audit_candidate_page_line_application_effect.py
11.5 kB
| #!/usr/bin/env python3 | |
| """Audit page candidate-line insertion effects against a baseline page score.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from typing import Any | |
| from ocrtech.manifest import sha256_file | |
| REPORT_JSON = "candidate-page-line-application-effect-audit.json" | |
| INSERTED_TSV = "candidate-page-line-application-inserted-lines.tsv" | |
| OBJECTIVE_METRICS = ( | |
| "text_cer_codepoint_micro", | |
| "text_cer_grapheme_micro", | |
| "line_detection_f1_micro", | |
| "line_detection_precision_micro", | |
| "line_detection_recall_micro", | |
| "line_unmatched_reference_count", | |
| "line_unmatched_prediction_count", | |
| "line_blank_prediction_count", | |
| "matched_line_reading_order_pair_accuracy", | |
| "matched_line_text_cer_codepoint_micro", | |
| ) | |
| def _read_json(path: Path, label: str) -> dict[str, Any]: | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except FileNotFoundError as exc: | |
| raise ValueError(f"{label} is missing: {path}") from exc | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise ValueError(f"{label} must contain a JSON object: {path}") | |
| return payload | |
| def _float(value: Any) -> float | None: | |
| if value in ("", None): | |
| return None | |
| try: | |
| return float(value) | |
| except (TypeError, ValueError): | |
| return None | |
| def _metric_delta(candidate: Any, baseline: Any) -> float | None: | |
| candidate_float = _float(candidate) | |
| baseline_float = _float(baseline) | |
| if candidate_float is None or baseline_float is None: | |
| return None | |
| return candidate_float - baseline_float | |
| def _read_alignment_by_prediction(path: Path) -> dict[str, dict[str, Any]]: | |
| try: | |
| handle = path.open("r", encoding="utf-8", newline="") | |
| except FileNotFoundError as exc: | |
| raise ValueError(f"line alignment CSV is missing: {path}") from exc | |
| rows: dict[str, dict[str, Any]] = {} | |
| with handle: | |
| reader = csv.DictReader(handle) | |
| if reader.fieldnames is None: | |
| raise ValueError(f"line alignment CSV has no header: {path}") | |
| for row_number, row in enumerate(reader, start=2): | |
| predicted_line_id = str(row.get("predicted_line_id") or "").strip() | |
| if not predicted_line_id: | |
| continue | |
| if predicted_line_id in rows: | |
| raise ValueError(f"duplicate predicted_line_id in line alignment at {path}:{row_number}: {predicted_line_id}") | |
| rows[predicted_line_id] = dict(row) | |
| return rows | |
| def _inserted_rows(application: dict[str, Any], alignment_by_prediction: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: | |
| applied_rows = application.get("applied_rows") | |
| if not isinstance(applied_rows, list): | |
| raise ValueError("application report missing applied_rows list") | |
| inserted: list[dict[str, Any]] = [] | |
| for row in applied_rows: | |
| if not isinstance(row, dict): | |
| raise ValueError("application applied_rows entries must be objects") | |
| inserted_line_id = str(row.get("inserted_line_id") or "").strip() | |
| if not inserted_line_id: | |
| raise ValueError(f"applied row missing inserted_line_id: {row.get('row_id')!r}") | |
| alignment = alignment_by_prediction.get(inserted_line_id) | |
| if alignment is None: | |
| inserted.append( | |
| { | |
| "row_id": row.get("row_id") or "", | |
| "sample_id": row.get("sample_id") or "", | |
| "candidate_line_id": row.get("candidate_line_id") or "", | |
| "inserted_line_id": inserted_line_id, | |
| "alignment_kind": "alignment_missing", | |
| "reference_line_id": "", | |
| "iou": None, | |
| "text_codepoint_edits": None, | |
| "text_codepoint_ref_len": None, | |
| "text_cer_codepoint": None, | |
| "inserted_text": row.get("inserted_text") or "", | |
| "selection_policy": row.get("selection_policy") or "", | |
| } | |
| ) | |
| continue | |
| inserted.append( | |
| { | |
| "row_id": row.get("row_id") or "", | |
| "sample_id": row.get("sample_id") or alignment.get("sample_id") or "", | |
| "candidate_line_id": row.get("candidate_line_id") or "", | |
| "inserted_line_id": inserted_line_id, | |
| "alignment_kind": alignment.get("kind") or "", | |
| "reference_line_id": alignment.get("reference_line_id") or "", | |
| "iou": _float(alignment.get("iou")), | |
| "text_codepoint_edits": _float(alignment.get("text_codepoint_edits")), | |
| "text_codepoint_ref_len": _float(alignment.get("text_codepoint_ref_len")), | |
| "text_cer_codepoint": _float(alignment.get("text_cer_codepoint")), | |
| "inserted_text": row.get("inserted_text") or "", | |
| "selection_policy": row.get("selection_policy") or "", | |
| } | |
| ) | |
| return inserted | |
| def _write_inserted_tsv(path: Path, rows: list[dict[str, Any]]) -> None: | |
| fieldnames = [ | |
| "row_id", | |
| "sample_id", | |
| "candidate_line_id", | |
| "inserted_line_id", | |
| "alignment_kind", | |
| "reference_line_id", | |
| "iou", | |
| "text_codepoint_edits", | |
| "text_codepoint_ref_len", | |
| "text_cer_codepoint", | |
| "selection_policy", | |
| ] | |
| with path.open("w", encoding="utf-8", newline="") as handle: | |
| writer = csv.DictWriter(handle, delimiter="\t", fieldnames=fieldnames) | |
| writer.writeheader() | |
| for row in rows: | |
| writer.writerow({field: row.get(field, "") for field in fieldnames}) | |
| def _metric_summary(baseline: dict[str, Any], candidate: dict[str, Any]) -> dict[str, dict[str, Any]]: | |
| summary: dict[str, dict[str, Any]] = {} | |
| for metric in OBJECTIVE_METRICS: | |
| baseline_value = baseline.get(metric) | |
| candidate_value = candidate.get(metric) | |
| summary[metric] = { | |
| "baseline": baseline_value, | |
| "candidate": candidate_value, | |
| "delta_candidate_minus_baseline": _metric_delta(candidate_value, baseline_value), | |
| } | |
| return summary | |
| def _book_ocr_issues(metrics: dict[str, dict[str, Any]], inserted_counts: dict[str, int]) -> list[str]: | |
| issues: list[str] = [] | |
| cp_delta = metrics["text_cer_codepoint_micro"]["delta_candidate_minus_baseline"] | |
| if cp_delta is None or cp_delta > 0: | |
| issues.append(f"text_cer_codepoint_micro does not improve or tie: delta={cp_delta}") | |
| graph_delta = metrics["text_cer_grapheme_micro"]["delta_candidate_minus_baseline"] | |
| if graph_delta is None or graph_delta > 0: | |
| issues.append(f"text_cer_grapheme_micro does not improve or tie: delta={graph_delta}") | |
| line_f1_delta = metrics["line_detection_f1_micro"]["delta_candidate_minus_baseline"] | |
| if line_f1_delta is None or line_f1_delta < 0: | |
| issues.append(f"line_detection_f1_micro regresses: delta={line_f1_delta}") | |
| extra_delta = metrics["line_unmatched_prediction_count"]["delta_candidate_minus_baseline"] | |
| if extra_delta is None or extra_delta > 0: | |
| issues.append(f"line_unmatched_prediction_count increases: delta={extra_delta}") | |
| missed_delta = metrics["line_unmatched_reference_count"]["delta_candidate_minus_baseline"] | |
| if missed_delta is None or missed_delta > 0: | |
| issues.append(f"line_unmatched_reference_count increases: delta={missed_delta}") | |
| order_delta = metrics["matched_line_reading_order_pair_accuracy"]["delta_candidate_minus_baseline"] | |
| if order_delta is None or order_delta < 0: | |
| issues.append(f"matched_line_reading_order_pair_accuracy regresses: delta={order_delta}") | |
| if inserted_counts.get("prediction_unmatched", 0) > 0: | |
| issues.append(f"inserted candidate rows include unmatched extra predictions: {inserted_counts['prediction_unmatched']}") | |
| if inserted_counts.get("alignment_missing", 0) > 0: | |
| issues.append(f"inserted candidate rows missing from line alignment: {inserted_counts['alignment_missing']}") | |
| return issues | |
| def audit_application_effect(args: argparse.Namespace) -> dict[str, Any]: | |
| application_path = Path(args.application_report) | |
| baseline_metrics_path = Path(args.baseline_score_dir) / "metrics.json" | |
| candidate_metrics_path = Path(args.candidate_score_dir) / "metrics.json" | |
| alignment_path = Path(args.candidate_score_dir) / "line-alignment.csv" | |
| out = Path(args.out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| application = _read_json(application_path, "candidate page-line application report") | |
| baseline_metrics = _read_json(baseline_metrics_path, "baseline metrics") | |
| candidate_metrics = _read_json(candidate_metrics_path, "candidate metrics") | |
| alignment = _read_alignment_by_prediction(alignment_path) | |
| inserted = _inserted_rows(application, alignment) | |
| inserted_counts: dict[str, int] = {} | |
| for row in inserted: | |
| kind = str(row.get("alignment_kind") or "unknown") | |
| inserted_counts[kind] = inserted_counts.get(kind, 0) + 1 | |
| metrics = _metric_summary(baseline_metrics, candidate_metrics) | |
| issues = _book_ocr_issues(metrics, inserted_counts) | |
| inserted_tsv = out / INSERTED_TSV | |
| _write_inserted_tsv(inserted_tsv, inserted) | |
| report_path = out / REPORT_JSON | |
| report = { | |
| "tool": "tools/audit_candidate_page_line_application_effect.py", | |
| "created_at_utc": datetime.now(UTC).isoformat(), | |
| "application_report": str(application_path), | |
| "application_report_sha256": sha256_file(application_path), | |
| "baseline_score_dir": str(args.baseline_score_dir), | |
| "baseline_metrics_sha256": sha256_file(baseline_metrics_path), | |
| "candidate_score_dir": str(args.candidate_score_dir), | |
| "candidate_metrics_sha256": sha256_file(candidate_metrics_path), | |
| "line_alignment_sha256": sha256_file(alignment_path), | |
| "inserted_line_count": len(inserted), | |
| "inserted_alignment_counts": inserted_counts, | |
| "objective_metric_comparison": metrics, | |
| "book_ocr_objective_safe": not issues, | |
| "issues": issues, | |
| "inserted_lines_tsv": str(inserted_tsv), | |
| "inserted_lines_tsv_sha256": sha256_file(inserted_tsv), | |
| "claim_boundary": ( | |
| "Page-candidate application objective audit. It compares a candidate projection to a baseline " | |
| "on page OCR objective metrics and inserted-line alignment; it does not certify claim-grade OCR." | |
| ), | |
| } | |
| report["report_json"] = str(report_path) | |
| report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| return report | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--application-report", required=True, type=Path) | |
| parser.add_argument("--baseline-score-dir", required=True, type=Path) | |
| parser.add_argument("--candidate-score-dir", required=True, type=Path) | |
| parser.add_argument("--out", required=True, type=Path) | |
| args = parser.parse_args() | |
| report = audit_application_effect(args) | |
| print(report["report_json"]) | |
| print( | |
| f"book_ocr_objective_safe={report['book_ocr_objective_safe']} " | |
| f"inserted={report['inserted_line_count']} issues={len(report['issues'])}" | |
| ) | |
| if __name__ == "__main__": | |
| main() | |