Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/tools/audit_page_ocr_documents.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 19.9 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/tools/audit_page_ocr_documents.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/tools/audit_page_ocr_documents.py
-
curl -L -o audit_page_ocr_documents.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/tools/audit_page_ocr_documents.py
19.9 kB
| #!/usr/bin/env python3 | |
| """Audit page OCR document outputs for page-level evaluation. | |
| This audit is intentionally narrower than `ocrtech audit-limbu-output`: it works | |
| on page-eval-compatible `document.json` directories produced by the page | |
| materializers and filters. It writes `limbu-output-audit.json` into each sample | |
| directory so `tools/run_page_ocr_eval.py --require-output-audit` can enforce the | |
| check before scoring live detector outputs. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import platform | |
| from collections import Counter | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from typing import Any | |
| from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file | |
| from ocrtech.schemas import Document | |
| def _resolve_manifest_path(path_value: str | Path, manifest_dir: Path) -> Path: | |
| path = Path(path_value) | |
| if path.is_absolute() or path.exists(): | |
| return path | |
| candidate = manifest_dir / path | |
| if candidate.exists(): | |
| return candidate | |
| return path | |
| def _resolve_existing_path(path_value: object, *, base_dirs: list[Path]) -> Path | None: | |
| if not isinstance(path_value, str) or not path_value: | |
| return None | |
| path = Path(path_value) | |
| candidates = [path] if path.is_absolute() else [path, *(base / path for base in base_dirs)] | |
| for candidate in candidates: | |
| if candidate.is_file(): | |
| return candidate | |
| return path | |
| def _document_candidates(entry: ManifestEntry, input_path: Path, documents_dir: Path) -> list[Path]: | |
| return [ | |
| documents_dir / entry.sample_id / "document.json", | |
| documents_dir / input_path.stem / "document.json", | |
| documents_dir / f"{entry.sample_id}.json", | |
| documents_dir / f"{input_path.stem}.json", | |
| ] | |
| def _load_document(path: Path, issues: list[str]) -> Document | None: | |
| if not path.is_file(): | |
| issues.append(f"document.json is missing: {path}") | |
| return None | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except json.JSONDecodeError as exc: | |
| issues.append(f"document.json is invalid JSON: {path}: {exc}") | |
| return None | |
| if not isinstance(payload, dict): | |
| issues.append(f"document.json must contain an object: {path}") | |
| return None | |
| try: | |
| return Document.from_dict(payload) | |
| except Exception as exc: | |
| issues.append(f"document.json failed schema load: {type(exc).__name__}: {exc}") | |
| return None | |
| def _same_file_or_hash(left: Path, right: Path) -> tuple[bool, str]: | |
| try: | |
| if left.resolve() == right.resolve(): | |
| return True, "same_path" | |
| except OSError: | |
| if str(left) == str(right): | |
| return True, "same_path_unresolved" | |
| if left.is_file() and right.is_file() and sha256_file(left) == sha256_file(right): | |
| return True, "same_sha256" | |
| return False, "different" | |
| def _audit_source_binding( | |
| *, | |
| sample_id: str, | |
| document: Document, | |
| document_dir: Path, | |
| input_path: Path, | |
| args: argparse.Namespace, | |
| issues: list[str], | |
| warnings: list[str], | |
| ) -> dict[str, Any]: | |
| source_value = document.source_path | |
| source_path = _resolve_existing_path(source_value, base_dirs=[document_dir, Path.cwd()]) | |
| binding: dict[str, Any] = { | |
| "document_source_path": source_value, | |
| "resolved_document_source_path": str(source_path) if source_path is not None else None, | |
| "eval_input_path": str(input_path), | |
| "match_kind": "not_checked", | |
| } | |
| if not source_value: | |
| issues.append(f"{sample_id}: document.source_path is missing") | |
| return binding | |
| if args.require_source_image and (source_path is None or not source_path.is_file()): | |
| issues.append(f"{sample_id}: document.source_path is not a readable file: {source_value}") | |
| return binding | |
| if source_path is not None and source_path.is_file(): | |
| binding["document_source_sha256"] = sha256_file(source_path) | |
| if input_path.is_file(): | |
| binding["eval_input_sha256"] = sha256_file(input_path) | |
| if args.require_document_source_match: | |
| if source_path is None or not source_path.is_file() or not input_path.is_file(): | |
| issues.append(f"{sample_id}: cannot verify document.source_path against eval input") | |
| return binding | |
| matched, match_kind = _same_file_or_hash(source_path, input_path) | |
| binding["match_kind"] = match_kind | |
| if not matched: | |
| issues.append( | |
| f"{sample_id}: document.source_path does not match eval input by path or SHA-256: " | |
| f"document={source_value!r} eval={str(input_path)!r}" | |
| ) | |
| else: | |
| warnings.append(f"{sample_id}: document.source_path/eval input match was not required") | |
| return binding | |
| def _audit_page_and_lines( | |
| *, | |
| sample_id: str, | |
| document: Document, | |
| document_dir: Path, | |
| args: argparse.Namespace, | |
| issues: list[str], | |
| warnings: list[str], | |
| ) -> dict[str, Any]: | |
| line_count = 0 | |
| empty_line_count = 0 | |
| confidence_values: list[float] = [] | |
| structural_roles: Counter[str] = Counter() | |
| detector_source_passes: Counter[str] = Counter() | |
| crop_count = 0 | |
| crop_hash_checked = 0 | |
| for page in document.pages: | |
| if page.width is None or page.width <= 0: | |
| issues.append(f"{sample_id}: page {page.page_index} width must be positive") | |
| if page.height is None or page.height <= 0: | |
| issues.append(f"{sample_id}: page {page.page_index} height must be positive") | |
| page_line_ids: set[str] = set() | |
| for index, line in enumerate(page.text_lines): | |
| line_count += 1 | |
| label = f"{sample_id}: page {page.page_index} line {index} ({line.line_id or 'no-id'})" | |
| if line.line_id: | |
| if line.line_id in page_line_ids: | |
| issues.append(f"{label}: duplicate line_id on page") | |
| page_line_ids.add(line.line_id) | |
| else: | |
| issues.append(f"{label}: missing line_id") | |
| if not line.text.strip(): | |
| empty_line_count += 1 | |
| if not args.allow_empty_text: | |
| issues.append(f"{label}: empty OCR text") | |
| if line.confidence is not None: | |
| confidence_values.append(float(line.confidence)) | |
| if line.confidence < 0 or line.confidence > 1: | |
| issues.append(f"{label}: confidence must be between 0 and 1") | |
| elif args.require_line_confidence: | |
| issues.append(f"{label}: missing confidence") | |
| if line.bbox.w <= 0 or line.bbox.h <= 0: | |
| issues.append(f"{label}: bbox width/height must be positive") | |
| if page.width and page.height: | |
| tolerance = float(args.bbox_tolerance_px) | |
| if line.bbox.x < -tolerance or line.bbox.y < -tolerance: | |
| issues.append(f"{label}: bbox starts outside page bounds") | |
| if line.bbox.right > page.width + tolerance or line.bbox.bottom > page.height + tolerance: | |
| issues.append(f"{label}: bbox extends outside page bounds") | |
| metadata = line.metadata if isinstance(line.metadata, dict) else {} | |
| role = str(metadata.get("structural_role") or "text") | |
| structural_roles[role] += 1 | |
| source_pass = metadata.get("detector_source_pass") | |
| if isinstance(source_pass, str) and source_pass: | |
| detector_source_passes[source_pass] += 1 | |
| if args.expected_detector_source_pass and source_pass != args.expected_detector_source_pass: | |
| issues.append( | |
| f"{label}: detector_source_pass {source_pass!r} does not match " | |
| f"{args.expected_detector_source_pass!r}" | |
| ) | |
| crop_path = _resolve_existing_path(metadata.get("crop_path"), base_dirs=[document_dir, Path.cwd()]) | |
| if crop_path is not None and isinstance(metadata.get("crop_path"), str): | |
| crop_count += 1 | |
| if not crop_path.is_file(): | |
| issues.append(f"{label}: crop_path is not a readable file: {metadata.get('crop_path')}") | |
| else: | |
| recorded_sha = metadata.get("crop_sha256") | |
| if isinstance(recorded_sha, str) and recorded_sha: | |
| crop_hash_checked += 1 | |
| actual_sha = sha256_file(crop_path) | |
| if actual_sha != recorded_sha: | |
| issues.append( | |
| f"{label}: crop_sha256 mismatch for {crop_path}: " | |
| f"recorded={recorded_sha} actual={actual_sha}" | |
| ) | |
| elif args.require_line_crops: | |
| issues.append(f"{label}: missing crop_path") | |
| known_line_ids = {line.line_id for line in page.text_lines if line.line_id} | |
| block_orders: set[int] = set() | |
| for block in page.blocks: | |
| if block.order in block_orders: | |
| issues.append(f"{sample_id}: page {page.page_index} duplicate block order {block.order}") | |
| block_orders.add(block.order) | |
| for line_id in block.line_ids: | |
| if line_id not in known_line_ids: | |
| issues.append(f"{sample_id}: block {block.block_id} references missing line_id {line_id!r}") | |
| if not document.pages: | |
| issues.append(f"{sample_id}: document has no pages") | |
| if args.min_line_count is not None and line_count < args.min_line_count: | |
| issues.append(f"{sample_id}: line_count {line_count} below minimum {args.min_line_count}") | |
| if args.max_line_count is not None and line_count > args.max_line_count: | |
| issues.append(f"{sample_id}: line_count {line_count} above maximum {args.max_line_count}") | |
| metadata = document.metadata if isinstance(document.metadata, dict) else {} | |
| expected_kind = str(args.expected_ocr_output_kind or "") | |
| observed_kind = metadata.get("ocr_output_kind") | |
| if expected_kind and observed_kind != expected_kind: | |
| issues.append(f"{sample_id}: document metadata ocr_output_kind {observed_kind!r} does not match {expected_kind!r}") | |
| recorded_line_count = metadata.get("line_count") | |
| if isinstance(recorded_line_count, int) and recorded_line_count != line_count: | |
| issues.append(f"{sample_id}: document metadata line_count {recorded_line_count} does not match actual {line_count}") | |
| elif recorded_line_count is not None and not isinstance(recorded_line_count, int): | |
| warnings.append(f"{sample_id}: document metadata line_count is not an integer: {recorded_line_count!r}") | |
| return { | |
| "page_count": len(document.pages), | |
| "line_count": line_count, | |
| "empty_line_count": empty_line_count, | |
| "average_line_confidence": sum(confidence_values) / len(confidence_values) if confidence_values else None, | |
| "line_confidence_count": len(confidence_values), | |
| "structural_role_counts": dict(sorted(structural_roles.items())), | |
| "detector_source_pass_counts": dict(sorted(detector_source_passes.items())), | |
| "crop_count": crop_count, | |
| "crop_hash_checked_count": crop_hash_checked, | |
| } | |
| def _write_sample_audit(report: dict[str, Any], output_dir: Path) -> None: | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| (output_dir / "limbu-output-audit.json").write_text( | |
| json.dumps(report, ensure_ascii=False, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| lines = [ | |
| "# Page OCR Document Output Audit", | |
| "", | |
| f"- sample: `{report.get('sample_id')}`", | |
| f"- passed: `{report.get('passed')}`", | |
| f"- document: `{report.get('document_path')}`", | |
| f"- eval input: `{report.get('eval_input_path')}`", | |
| f"- source match: `{report.get('source_binding', {}).get('match_kind')}`", | |
| f"- line count: `{report.get('line_count')}`", | |
| f"- policy: `{json.dumps(report.get('policy', {}), ensure_ascii=False, sort_keys=True)}`", | |
| "", | |
| "## Issues", | |
| "", | |
| ] | |
| issues = report.get("issues") | |
| lines.extend(f"- {issue}" for issue in issues) if isinstance(issues, list) and issues else lines.append("- none") | |
| lines.extend(["", "## Warnings", ""]) | |
| warnings = report.get("warnings") | |
| lines.extend(f"- {warning}" for warning in warnings) if isinstance(warnings, list) and warnings else lines.append("- none") | |
| lines.append("") | |
| (output_dir / "limbu-output-audit.md").write_text("\n".join(lines), encoding="utf-8") | |
| def _audit_sample(entry: ManifestEntry, *, manifest_dir: Path, documents_dir: Path, args: argparse.Namespace) -> dict[str, Any]: | |
| issues: list[str] = [] | |
| warnings: list[str] = [] | |
| input_path = _resolve_manifest_path(entry.image_path, manifest_dir) | |
| if not input_path.is_file(): | |
| issues.append(f"{entry.sample_id}: eval input is missing: {input_path}") | |
| elif entry.sha256 and sha256_file(input_path) != entry.sha256: | |
| issues.append(f"{entry.sample_id}: eval input SHA-256 mismatch") | |
| candidates = _document_candidates(entry, input_path, documents_dir) | |
| document_path = next((candidate for candidate in candidates if candidate.is_file()), candidates[0]) | |
| document = _load_document(document_path, issues) | |
| source_binding: dict[str, Any] = {} | |
| summary: dict[str, Any] = { | |
| "page_count": 0, | |
| "line_count": 0, | |
| "empty_line_count": 0, | |
| "average_line_confidence": None, | |
| "line_confidence_count": 0, | |
| "structural_role_counts": {}, | |
| "detector_source_pass_counts": {}, | |
| "crop_count": 0, | |
| "crop_hash_checked_count": 0, | |
| } | |
| if document is not None: | |
| source_binding = _audit_source_binding( | |
| sample_id=entry.sample_id, | |
| document=document, | |
| document_dir=document_path.parent, | |
| input_path=input_path, | |
| args=args, | |
| issues=issues, | |
| warnings=warnings, | |
| ) | |
| summary = _audit_page_and_lines( | |
| sample_id=entry.sample_id, | |
| document=document, | |
| document_dir=document_path.parent, | |
| args=args, | |
| issues=issues, | |
| warnings=warnings, | |
| ) | |
| policy = { | |
| "expected_ocr_output_kind": args.expected_ocr_output_kind, | |
| "expected_detector_source_pass": args.expected_detector_source_pass, | |
| "require_source_image": args.require_source_image, | |
| "require_document_source_match": args.require_document_source_match, | |
| "require_line_crops": args.require_line_crops, | |
| "require_line_confidence": args.require_line_confidence, | |
| "allow_empty_text": args.allow_empty_text, | |
| "bbox_tolerance_px": args.bbox_tolerance_px, | |
| "min_line_count": args.min_line_count, | |
| "max_line_count": args.max_line_count, | |
| } | |
| report: dict[str, Any] = { | |
| "created_at": datetime.now(UTC).isoformat(), | |
| "machine": platform.node(), | |
| "tool": "tools/audit_page_ocr_documents.py", | |
| "sample_id": entry.sample_id, | |
| "eval_input_path": str(input_path), | |
| "eval_input_sha256": sha256_file(input_path) if input_path.is_file() else None, | |
| "document_path": str(document_path), | |
| "document_sha256": sha256_file(document_path) if document_path.is_file() else None, | |
| "source_binding": source_binding, | |
| "policy": policy, | |
| "passed": not issues, | |
| "issues": issues, | |
| "warnings": warnings, | |
| "claim_boundary": "document artifact audit for page OCR outputs; not a text-quality or reference correctness claim", | |
| **summary, | |
| } | |
| _write_sample_audit(report, document_path.parent) | |
| return report | |
| def audit_page_ocr_documents(args: argparse.Namespace) -> dict[str, Any]: | |
| manifest = load_manifest(args.eval_manifest) | |
| if not manifest: | |
| raise ValueError(f"eval manifest is empty: {args.eval_manifest}") | |
| documents_dir = Path(args.documents_dir) | |
| out = Path(args.out) if args.out is not None else documents_dir | |
| out.mkdir(parents=True, exist_ok=True) | |
| sample_reports = [ | |
| _audit_sample(entry, manifest_dir=Path(args.eval_manifest).parent, documents_dir=documents_dir, args=args) | |
| for entry in manifest | |
| ] | |
| aggregate: dict[str, Any] = { | |
| "created_at": datetime.now(UTC).isoformat(), | |
| "machine": platform.node(), | |
| "tool": "tools/audit_page_ocr_documents.py", | |
| "eval_manifest": str(args.eval_manifest), | |
| "eval_manifest_sha256": sha256_file(Path(args.eval_manifest)), | |
| "documents_dir": str(documents_dir), | |
| "sample_count": len(sample_reports), | |
| "passed_count": sum(1 for report in sample_reports if report.get("passed") is True), | |
| "failed_count": sum(1 for report in sample_reports if report.get("passed") is not True), | |
| "line_count": sum(int(report.get("line_count") or 0) for report in sample_reports), | |
| "empty_line_count": sum(int(report.get("empty_line_count") or 0) for report in sample_reports), | |
| "passed": all(report.get("passed") is True for report in sample_reports), | |
| "policy": sample_reports[0].get("policy", {}) if sample_reports else {}, | |
| "samples": sample_reports, | |
| } | |
| (out / "page-ocr-documents-audit.json").write_text( | |
| json.dumps(aggregate, ensure_ascii=False, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| lines = [ | |
| "# Page OCR Documents Audit", | |
| "", | |
| f"- passed: `{aggregate['passed']}`", | |
| f"- eval manifest: `{args.eval_manifest}`", | |
| f"- documents dir: `{documents_dir}`", | |
| f"- samples: `{aggregate['sample_count']}`", | |
| f"- passed samples: `{aggregate['passed_count']}`", | |
| f"- failed samples: `{aggregate['failed_count']}`", | |
| f"- lines: `{aggregate['line_count']}`", | |
| f"- empty lines: `{aggregate['empty_line_count']}`", | |
| "", | |
| "## Failed Samples", | |
| "", | |
| ] | |
| failed = [report for report in sample_reports if report.get("passed") is not True] | |
| if failed: | |
| for report in failed: | |
| lines.append(f"- `{report.get('sample_id')}`: {len(report.get('issues') or [])} issues") | |
| else: | |
| lines.append("- none") | |
| lines.append("") | |
| (out / "page-ocr-documents-audit.md").write_text("\n".join(lines), encoding="utf-8") | |
| return aggregate | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--eval-manifest", type=Path, required=True) | |
| parser.add_argument("--documents-dir", type=Path, required=True) | |
| parser.add_argument("--out", type=Path) | |
| parser.add_argument("--expected-ocr-output-kind") | |
| parser.add_argument("--expected-detector-source-pass") | |
| parser.add_argument("--no-require-source-image", dest="require_source_image", action="store_false") | |
| parser.set_defaults(require_source_image=True) | |
| parser.add_argument("--no-require-document-source-match", dest="require_document_source_match", action="store_false") | |
| parser.set_defaults(require_document_source_match=True) | |
| parser.add_argument("--require-line-crops", action="store_true") | |
| parser.add_argument("--require-line-confidence", action="store_true") | |
| parser.add_argument("--allow-empty-text", action="store_true") | |
| parser.add_argument("--bbox-tolerance-px", type=float, default=2.0) | |
| parser.add_argument("--min-line-count", type=int) | |
| parser.add_argument("--max-line-count", type=int) | |
| args = parser.parse_args(argv) | |
| result = audit_page_ocr_documents(args) | |
| print( | |
| f"page-ocr-documents audit {'passed' if result['passed'] else 'failed'} " | |
| f"samples={result['sample_count']} failed={result['failed_count']}" | |
| ) | |
| return 0 if result["passed"] else 1 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |