Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/apply_page_reading_order_geometry_profile.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 18.3 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/apply_page_reading_order_geometry_profile.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/apply_page_reading_order_geometry_profile.py
-
curl -L -o apply_page_reading_order_geometry_profile.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/apply_page_reading_order_geometry_profile.py
18.3 kB
| #!/usr/bin/env python3 | |
| """Apply a reading-order geometry profile to page OCR document projections.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from typing import Any | |
| from ocrtech.markdown import render_document_markdown | |
| from ocrtech.manifest import sha256_file | |
| from ocrtech.schemas import Block, Document, TextLine | |
| REPORT_JSON = "reading-order-geometry-profile-application.json" | |
| ASSIGNMENTS_JSONL = "reading-order-geometry-profile-assignments.jsonl" | |
| def _read_json_object(path: Path) -> dict[str, Any]: | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except FileNotFoundError as exc: | |
| raise ValueError(f"required JSON file is missing: {path}") from exc | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"invalid JSON in {path}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise ValueError(f"JSON file must contain an object: {path}") | |
| return payload | |
| def _document_paths(documents_dir: Path) -> dict[str, Path]: | |
| if not documents_dir.is_dir(): | |
| raise ValueError(f"documents directory does not exist: {documents_dir}") | |
| paths: dict[str, Path] = {} | |
| for child in sorted(documents_dir.iterdir()): | |
| if not child.is_dir(): | |
| continue | |
| document_path = child / "document.json" | |
| if document_path.is_file(): | |
| paths[child.name] = document_path | |
| if not paths: | |
| raise ValueError(f"documents directory contains no sample document.json files: {documents_dir}") | |
| return paths | |
| def _read_chunks_from_profile(profile_path: Path, profile: dict[str, Any]) -> list[dict[str, Any]]: | |
| chunks = profile.get("chunks") | |
| if not isinstance(chunks, list) or not chunks: | |
| raise ValueError(f"reading-order profile must contain non-empty chunks list: {profile_path}") | |
| normalized: list[dict[str, Any]] = [] | |
| for index, chunk in enumerate(chunks, start=1): | |
| if not isinstance(chunk, dict): | |
| raise ValueError(f"profile chunk {index} must be an object") | |
| try: | |
| chunk_index = int(chunk.get("chunk_index") or index) | |
| cx_median = float(chunk["cx_median"]) | |
| y_min = float(chunk["y_min"]) | |
| y_max = float(chunk["y_max"]) | |
| width_median = float(chunk["width_median"]) | |
| except (KeyError, TypeError, ValueError) as exc: | |
| raise ValueError(f"profile chunk {index} missing numeric chunk_index/cx_median/y_min/y_max/width_median") from exc | |
| if width_median <= 0: | |
| raise ValueError(f"profile chunk {index} width_median must be positive") | |
| normalized.append( | |
| { | |
| "chunk_index": chunk_index, | |
| "cx_median": cx_median, | |
| "y_min": y_min, | |
| "y_max": y_max, | |
| "width_median": width_median, | |
| "start_line_id": str(chunk.get("start_line_id") or ""), | |
| "end_line_id": str(chunk.get("end_line_id") or ""), | |
| "start_reference_line_id": str(chunk.get("start_reference_line_id") or ""), | |
| "end_reference_line_id": str(chunk.get("end_reference_line_id") or ""), | |
| } | |
| ) | |
| normalized.sort(key=lambda item: item["chunk_index"]) | |
| return normalized | |
| def _resolve_profile_path(value: str, base_dir: Path) -> Path: | |
| path = Path(value) | |
| if path.is_absolute() or path.exists(): | |
| return path | |
| candidate = base_dir / path | |
| if candidate.exists(): | |
| return candidate | |
| return path | |
| def _read_profile_map(profile_map_path: Path) -> tuple[dict[str, Path], Path | None]: | |
| payload = _read_json_object(profile_map_path) | |
| base_dir = profile_map_path.parent | |
| sample_profiles: dict[str, Path] = {} | |
| default_profile: Path | None = None | |
| if "profiles" in payload: | |
| profiles = payload.get("profiles") | |
| if not isinstance(profiles, list): | |
| raise ValueError(f"profile map profiles must be a list: {profile_map_path}") | |
| for index, row in enumerate(profiles, start=1): | |
| if not isinstance(row, dict): | |
| raise ValueError(f"profile map row {index} must be an object") | |
| sample_id = str(row.get("sample_id") or "").strip() | |
| profile_value = row.get("profile") | |
| if not sample_id: | |
| raise ValueError(f"profile map row {index} missing sample_id") | |
| if not isinstance(profile_value, str) or not profile_value: | |
| raise ValueError(f"profile map row {index} missing profile") | |
| if sample_id in sample_profiles: | |
| raise ValueError(f"profile map has duplicate sample_id: {sample_id}") | |
| sample_profiles[sample_id] = _resolve_profile_path(profile_value, base_dir) | |
| else: | |
| for sample_id, profile_value in payload.items(): | |
| if sample_id == "default_profile": | |
| continue | |
| if not isinstance(profile_value, str) or not profile_value: | |
| raise ValueError(f"profile map value for {sample_id!r} must be a profile path string") | |
| sample_profiles[str(sample_id)] = _resolve_profile_path(profile_value, base_dir) | |
| default_value = payload.get("default_profile") | |
| if isinstance(default_value, str) and default_value: | |
| default_profile = _resolve_profile_path(default_value, base_dir) | |
| return sample_profiles, default_profile | |
| def _load_profile_bundle( | |
| profile_path: Path, | |
| *, | |
| require_profile_claim_safe: bool, | |
| cache: dict[Path, dict[str, Any]], | |
| ) -> dict[str, Any]: | |
| resolved = profile_path.resolve() | |
| if resolved in cache: | |
| return cache[resolved] | |
| profile = _read_json_object(profile_path) | |
| if require_profile_claim_safe and profile.get("profile_claim_safe") is not True: | |
| raise ValueError(f"--require-profile-claim-safe requested but profile_claim_safe is not true: {profile_path}") | |
| bundle = { | |
| "path": profile_path, | |
| "sha256": sha256_file(profile_path), | |
| "profile_claim_safe": bool(profile.get("profile_claim_safe", False)), | |
| "chunks": _read_chunks_from_profile(profile_path, profile), | |
| } | |
| cache[resolved] = bundle | |
| return bundle | |
| def _profile_bundle_for_sample( | |
| sample_id: str, | |
| *, | |
| single_profile_path: Path | None, | |
| profile_map: dict[str, Path], | |
| default_profile: Path | None, | |
| require_profile_claim_safe: bool, | |
| cache: dict[Path, dict[str, Any]], | |
| ) -> dict[str, Any]: | |
| profile_path = single_profile_path or profile_map.get(sample_id) or default_profile | |
| if profile_path is None: | |
| raise ValueError(f"no reading-order profile configured for sample_id={sample_id!r}") | |
| return _load_profile_bundle( | |
| profile_path, | |
| require_profile_claim_safe=require_profile_claim_safe, | |
| cache=cache, | |
| ) | |
| def _chunk_score( | |
| line: TextLine, | |
| chunk: dict[str, Any], | |
| *, | |
| x_scale: float, | |
| y_scale: float, | |
| outside_weight: float, | |
| ) -> float: | |
| center_x = line.bbox.x + line.bbox.w / 2.0 | |
| x_scale_value = max(80.0, float(chunk["width_median"]) * x_scale) | |
| dx = abs(center_x - float(chunk["cx_median"])) / x_scale_value | |
| if line.bbox.y < float(chunk["y_min"]): | |
| outside_y = float(chunk["y_min"]) - line.bbox.y | |
| elif line.bbox.y > float(chunk["y_max"]): | |
| outside_y = line.bbox.y - float(chunk["y_max"]) | |
| else: | |
| outside_y = 0.0 | |
| dy = outside_y / y_scale | |
| return dx + outside_weight * dy | |
| def _assign_line( | |
| line: TextLine, | |
| chunks: list[dict[str, Any]], | |
| *, | |
| x_scale: float, | |
| y_scale: float, | |
| outside_weight: float, | |
| ) -> tuple[dict[str, Any], float]: | |
| best_chunk = chunks[0] | |
| best_score = _chunk_score(line, best_chunk, x_scale=x_scale, y_scale=y_scale, outside_weight=outside_weight) | |
| for chunk in chunks[1:]: | |
| score = _chunk_score(line, chunk, x_scale=x_scale, y_scale=y_scale, outside_weight=outside_weight) | |
| if score < best_score: | |
| best_chunk = chunk | |
| best_score = score | |
| return best_chunk, best_score | |
| def _block_key(block: Block, line_keys: dict[str, tuple[int, float, float, float, float, str]]) -> tuple[int, float, float, float, float, str]: | |
| if block.line_ids: | |
| key = line_keys.get(block.line_ids[0]) | |
| if key is not None: | |
| return key | |
| return (10**9, block.bbox.y, block.bbox.x, block.bbox.h, block.bbox.w, block.block_id) | |
| def _write_document_artifacts(document_dir: Path, document: Document) -> None: | |
| document_dir.mkdir(parents=True, exist_ok=True) | |
| (document_dir / "document.json").write_text(json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| (document_dir / "document.md").write_text(render_document_markdown(document), encoding="utf-8") | |
| (document_dir / "document.body.md").write_text(render_document_markdown(document, include_structural_roles=False), encoding="utf-8") | |
| def apply_profile(args: argparse.Namespace) -> dict[str, Any]: | |
| source_documents_dir = Path(args.source_documents_dir) | |
| raw_profile = getattr(args, "profile", None) | |
| raw_profile_map = getattr(args, "profile_map", None) | |
| profile_path = Path(raw_profile) if raw_profile else None | |
| profile_map_path = Path(raw_profile_map) if raw_profile_map else None | |
| out = Path(args.out) | |
| if source_documents_dir.resolve() == out.resolve(): | |
| raise ValueError("--out must not be the same directory as --source-documents-dir") | |
| if float(args.x_scale) <= 0 or float(args.y_scale) <= 0 or float(args.outside_weight) < 0: | |
| raise ValueError("--x-scale/--y-scale must be positive and --outside-weight must be non-negative") | |
| if (profile_path is None) == (profile_map_path is None): | |
| raise ValueError("exactly one of --profile or --profile-map is required") | |
| profile_map: dict[str, Path] = {} | |
| default_profile: Path | None = None | |
| if profile_map_path is not None: | |
| profile_map, default_profile = _read_profile_map(profile_map_path) | |
| source_paths = _document_paths(source_documents_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| assignments: list[dict[str, Any]] = [] | |
| page_reports: list[dict[str, Any]] = [] | |
| profile_cache: dict[Path, dict[str, Any]] = {} | |
| profile_usage: dict[str, dict[str, Any]] = {} | |
| require_profile_claim_safe = bool(getattr(args, "require_profile_claim_safe", False)) | |
| for sample_id, document_path in sorted(source_paths.items()): | |
| profile_bundle = _profile_bundle_for_sample( | |
| sample_id, | |
| single_profile_path=profile_path, | |
| profile_map=profile_map, | |
| default_profile=default_profile, | |
| require_profile_claim_safe=require_profile_claim_safe, | |
| cache=profile_cache, | |
| ) | |
| chunks = profile_bundle["chunks"] | |
| profile_key = str(profile_bundle["path"]) | |
| profile_usage.setdefault( | |
| profile_key, | |
| { | |
| "profile": profile_key, | |
| "profile_sha256": profile_bundle["sha256"], | |
| "profile_claim_safe": profile_bundle["profile_claim_safe"], | |
| "chunk_count": len(chunks), | |
| "sample_ids": [], | |
| }, | |
| )["sample_ids"].append(sample_id) | |
| document = Document.from_dict(_read_json_object(document_path)) | |
| for page in document.pages: | |
| line_keys: dict[str, tuple[int, float, float, float, float, str]] = {} | |
| for line in page.text_lines: | |
| chunk, score = _assign_line( | |
| line, | |
| chunks, | |
| x_scale=float(args.x_scale), | |
| y_scale=float(args.y_scale), | |
| outside_weight=float(args.outside_weight), | |
| ) | |
| line_id = line.line_id or "" | |
| line_keys[line_id] = ( | |
| int(chunk["chunk_index"]), | |
| line.bbox.y, | |
| line.bbox.x, | |
| line.bbox.h, | |
| line.bbox.w, | |
| line_id, | |
| ) | |
| metadata = line.metadata if isinstance(line.metadata, dict) else {} | |
| metadata["reading_order_geometry_profile_assignment"] = { | |
| "profile": str(profile_bundle["path"]), | |
| "profile_sha256": profile_bundle["sha256"], | |
| "chunk_index": int(chunk["chunk_index"]), | |
| "score": score, | |
| "start_reference_line_id": chunk.get("start_reference_line_id") or "", | |
| "end_reference_line_id": chunk.get("end_reference_line_id") or "", | |
| "start_line_id": chunk.get("start_line_id") or "", | |
| "end_line_id": chunk.get("end_line_id") or "", | |
| "claim_boundary": "profile-based reading-order assignment; profile provenance controls claim safety", | |
| } | |
| line.metadata = metadata | |
| assignments.append( | |
| { | |
| "sample_id": sample_id, | |
| "page_index": page.page_index, | |
| "line_id": line_id, | |
| "profile": str(profile_bundle["path"]), | |
| "profile_sha256": profile_bundle["sha256"], | |
| "chunk_index": int(chunk["chunk_index"]), | |
| "score": score, | |
| "bbox": line.bbox.to_list(), | |
| "text_length": len(line.text or ""), | |
| } | |
| ) | |
| page.text_lines = sorted(page.text_lines, key=lambda line: line_keys.get(line.line_id or "", (10**9, line.bbox.y, line.bbox.x, line.bbox.h, line.bbox.w, line.line_id or ""))) | |
| lines_by_id = {line.line_id or "": line for line in page.text_lines} | |
| for block in page.blocks: | |
| if not block.line_ids: | |
| continue | |
| block.line_ids = sorted( | |
| block.line_ids, | |
| key=lambda line_id: line_keys.get(line_id, (10**9, block.bbox.y, block.bbox.x, block.bbox.h, block.bbox.w, line_id)), | |
| ) | |
| block_lines = [lines_by_id[line_id] for line_id in block.line_ids if line_id in lines_by_id] | |
| if block_lines: | |
| block.text = "\n".join(line.text for line in block_lines) | |
| page.blocks = sorted(page.blocks, key=lambda block: _block_key(block, line_keys)) | |
| for order, block in enumerate(page.blocks, start=1): | |
| block.order = order | |
| target_dir = out / sample_id | |
| _write_document_artifacts(target_dir, document) | |
| line_count = sum(len(page.text_lines) for page in document.pages) | |
| page_reports.append( | |
| { | |
| "sample_id": sample_id, | |
| "source_document": str(document_path), | |
| "source_document_sha256": sha256_file(document_path), | |
| "profile": str(profile_bundle["path"]), | |
| "profile_sha256": profile_bundle["sha256"], | |
| "profile_claim_safe": profile_bundle["profile_claim_safe"], | |
| "document_json": str(target_dir / "document.json"), | |
| "document_json_sha256": sha256_file(target_dir / "document.json"), | |
| "document_md_sha256": sha256_file(target_dir / "document.md"), | |
| "document_body_md_sha256": sha256_file(target_dir / "document.body.md"), | |
| "line_count": line_count, | |
| } | |
| ) | |
| assignments_path = out / ASSIGNMENTS_JSONL | |
| assignments_path.write_text("".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in assignments), encoding="utf-8") | |
| report_path = out / REPORT_JSON | |
| report = { | |
| "tool": "tools/apply_page_reading_order_geometry_profile.py", | |
| "created_at_utc": datetime.now(UTC).isoformat(), | |
| "source_documents_dir": str(source_documents_dir), | |
| "profile": str(profile_path) if profile_path is not None else "", | |
| "profile_map": str(profile_map_path) if profile_map_path is not None else "", | |
| "profile_map_sha256": sha256_file(profile_map_path) if profile_map_path is not None else "", | |
| "profile_sha256": sha256_file(profile_path) if profile_path is not None else "", | |
| "profile_claim_safe": all(bool(item["profile_claim_safe"]) for item in profile_usage.values()), | |
| "profile_usage": list(profile_usage.values()), | |
| "out": str(out), | |
| "config": { | |
| "x_scale": float(args.x_scale), | |
| "y_scale": float(args.y_scale), | |
| "outside_weight": float(args.outside_weight), | |
| "require_profile_claim_safe": require_profile_claim_safe, | |
| }, | |
| "chunk_count": sum(int(item["chunk_count"]) for item in profile_usage.values()), | |
| "document_count": len(source_paths), | |
| "assignment_count": len(assignments), | |
| "pages": page_reports, | |
| "assignments_jsonl": str(assignments_path), | |
| "assignments_jsonl_sha256": sha256_file(assignments_path), | |
| "claim_boundary": "profile-based reading-order application; not claim-safe unless profile provenance is claim-safe", | |
| } | |
| report["report_json"] = str(report_path) | |
| report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| return report | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--source-documents-dir", type=Path, required=True) | |
| profile_group = parser.add_mutually_exclusive_group(required=True) | |
| profile_group.add_argument("--profile", type=Path) | |
| profile_group.add_argument("--profile-map", type=Path) | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument("--x-scale", type=float, default=1.0) | |
| parser.add_argument("--y-scale", type=float, default=80.0) | |
| parser.add_argument("--outside-weight", type=float, default=2.0) | |
| parser.add_argument("--require-profile-claim-safe", action="store_true") | |
| args = parser.parse_args() | |
| report = apply_profile(args) | |
| print(report["report_json"]) | |
| print(f"documents={report['document_count']} assignments={report['assignment_count']} chunks={report['chunk_count']}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |