#!/usr/bin/env python3 """Apply a reading-order geometry profile to page OCR document projections.""" from __future__ import annotations import argparse import json from datetime import UTC, datetime from pathlib import Path from typing import Any from ocrtech.markdown import render_document_markdown from ocrtech.manifest import sha256_file from ocrtech.schemas import Block, Document, TextLine REPORT_JSON = "reading-order-geometry-profile-application.json" ASSIGNMENTS_JSONL = "reading-order-geometry-profile-assignments.jsonl" def _read_json_object(path: Path) -> dict[str, Any]: try: payload = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError as exc: raise ValueError(f"required JSON file is missing: {path}") from exc except json.JSONDecodeError as exc: raise ValueError(f"invalid JSON in {path}: {exc}") from exc if not isinstance(payload, dict): raise ValueError(f"JSON file must contain an object: {path}") return payload def _document_paths(documents_dir: Path) -> dict[str, Path]: if not documents_dir.is_dir(): raise ValueError(f"documents directory does not exist: {documents_dir}") paths: dict[str, Path] = {} for child in sorted(documents_dir.iterdir()): if not child.is_dir(): continue document_path = child / "document.json" if document_path.is_file(): paths[child.name] = document_path if not paths: raise ValueError(f"documents directory contains no sample document.json files: {documents_dir}") return paths def _read_chunks_from_profile(profile_path: Path, profile: dict[str, Any]) -> list[dict[str, Any]]: chunks = profile.get("chunks") if not isinstance(chunks, list) or not chunks: raise ValueError(f"reading-order profile must contain non-empty chunks list: {profile_path}") normalized: list[dict[str, Any]] = [] for index, chunk in enumerate(chunks, start=1): if not isinstance(chunk, dict): raise ValueError(f"profile chunk {index} must be an object") try: chunk_index = int(chunk.get("chunk_index") or index) cx_median = float(chunk["cx_median"]) y_min = float(chunk["y_min"]) y_max = float(chunk["y_max"]) width_median = float(chunk["width_median"]) except (KeyError, TypeError, ValueError) as exc: raise ValueError(f"profile chunk {index} missing numeric chunk_index/cx_median/y_min/y_max/width_median") from exc if width_median <= 0: raise ValueError(f"profile chunk {index} width_median must be positive") normalized.append( { "chunk_index": chunk_index, "cx_median": cx_median, "y_min": y_min, "y_max": y_max, "width_median": width_median, "start_line_id": str(chunk.get("start_line_id") or ""), "end_line_id": str(chunk.get("end_line_id") or ""), "start_reference_line_id": str(chunk.get("start_reference_line_id") or ""), "end_reference_line_id": str(chunk.get("end_reference_line_id") or ""), } ) normalized.sort(key=lambda item: item["chunk_index"]) return normalized def _resolve_profile_path(value: str, base_dir: Path) -> Path: path = Path(value) if path.is_absolute() or path.exists(): return path candidate = base_dir / path if candidate.exists(): return candidate return path def _read_profile_map(profile_map_path: Path) -> tuple[dict[str, Path], Path | None]: payload = _read_json_object(profile_map_path) base_dir = profile_map_path.parent sample_profiles: dict[str, Path] = {} default_profile: Path | None = None if "profiles" in payload: profiles = payload.get("profiles") if not isinstance(profiles, list): raise ValueError(f"profile map profiles must be a list: {profile_map_path}") for index, row in enumerate(profiles, start=1): if not isinstance(row, dict): raise ValueError(f"profile map row {index} must be an object") sample_id = str(row.get("sample_id") or "").strip() profile_value = row.get("profile") if not sample_id: raise ValueError(f"profile map row {index} missing sample_id") if not isinstance(profile_value, str) or not profile_value: raise ValueError(f"profile map row {index} missing profile") if sample_id in sample_profiles: raise ValueError(f"profile map has duplicate sample_id: {sample_id}") sample_profiles[sample_id] = _resolve_profile_path(profile_value, base_dir) else: for sample_id, profile_value in payload.items(): if sample_id == "default_profile": continue if not isinstance(profile_value, str) or not profile_value: raise ValueError(f"profile map value for {sample_id!r} must be a profile path string") sample_profiles[str(sample_id)] = _resolve_profile_path(profile_value, base_dir) default_value = payload.get("default_profile") if isinstance(default_value, str) and default_value: default_profile = _resolve_profile_path(default_value, base_dir) return sample_profiles, default_profile def _load_profile_bundle( profile_path: Path, *, require_profile_claim_safe: bool, cache: dict[Path, dict[str, Any]], ) -> dict[str, Any]: resolved = profile_path.resolve() if resolved in cache: return cache[resolved] profile = _read_json_object(profile_path) if require_profile_claim_safe and profile.get("profile_claim_safe") is not True: raise ValueError(f"--require-profile-claim-safe requested but profile_claim_safe is not true: {profile_path}") bundle = { "path": profile_path, "sha256": sha256_file(profile_path), "profile_claim_safe": bool(profile.get("profile_claim_safe", False)), "chunks": _read_chunks_from_profile(profile_path, profile), } cache[resolved] = bundle return bundle def _profile_bundle_for_sample( sample_id: str, *, single_profile_path: Path | None, profile_map: dict[str, Path], default_profile: Path | None, require_profile_claim_safe: bool, cache: dict[Path, dict[str, Any]], ) -> dict[str, Any]: profile_path = single_profile_path or profile_map.get(sample_id) or default_profile if profile_path is None: raise ValueError(f"no reading-order profile configured for sample_id={sample_id!r}") return _load_profile_bundle( profile_path, require_profile_claim_safe=require_profile_claim_safe, cache=cache, ) def _chunk_score( line: TextLine, chunk: dict[str, Any], *, x_scale: float, y_scale: float, outside_weight: float, ) -> float: center_x = line.bbox.x + line.bbox.w / 2.0 x_scale_value = max(80.0, float(chunk["width_median"]) * x_scale) dx = abs(center_x - float(chunk["cx_median"])) / x_scale_value if line.bbox.y < float(chunk["y_min"]): outside_y = float(chunk["y_min"]) - line.bbox.y elif line.bbox.y > float(chunk["y_max"]): outside_y = line.bbox.y - float(chunk["y_max"]) else: outside_y = 0.0 dy = outside_y / y_scale return dx + outside_weight * dy def _assign_line( line: TextLine, chunks: list[dict[str, Any]], *, x_scale: float, y_scale: float, outside_weight: float, ) -> tuple[dict[str, Any], float]: best_chunk = chunks[0] best_score = _chunk_score(line, best_chunk, x_scale=x_scale, y_scale=y_scale, outside_weight=outside_weight) for chunk in chunks[1:]: score = _chunk_score(line, chunk, x_scale=x_scale, y_scale=y_scale, outside_weight=outside_weight) if score < best_score: best_chunk = chunk best_score = score return best_chunk, best_score def _block_key(block: Block, line_keys: dict[str, tuple[int, float, float, float, float, str]]) -> tuple[int, float, float, float, float, str]: if block.line_ids: key = line_keys.get(block.line_ids[0]) if key is not None: return key return (10**9, block.bbox.y, block.bbox.x, block.bbox.h, block.bbox.w, block.block_id) def _write_document_artifacts(document_dir: Path, document: Document) -> None: document_dir.mkdir(parents=True, exist_ok=True) (document_dir / "document.json").write_text(json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (document_dir / "document.md").write_text(render_document_markdown(document), encoding="utf-8") (document_dir / "document.body.md").write_text(render_document_markdown(document, include_structural_roles=False), encoding="utf-8") def apply_profile(args: argparse.Namespace) -> dict[str, Any]: source_documents_dir = Path(args.source_documents_dir) raw_profile = getattr(args, "profile", None) raw_profile_map = getattr(args, "profile_map", None) profile_path = Path(raw_profile) if raw_profile else None profile_map_path = Path(raw_profile_map) if raw_profile_map else None out = Path(args.out) if source_documents_dir.resolve() == out.resolve(): raise ValueError("--out must not be the same directory as --source-documents-dir") if float(args.x_scale) <= 0 or float(args.y_scale) <= 0 or float(args.outside_weight) < 0: raise ValueError("--x-scale/--y-scale must be positive and --outside-weight must be non-negative") if (profile_path is None) == (profile_map_path is None): raise ValueError("exactly one of --profile or --profile-map is required") profile_map: dict[str, Path] = {} default_profile: Path | None = None if profile_map_path is not None: profile_map, default_profile = _read_profile_map(profile_map_path) source_paths = _document_paths(source_documents_dir) out.mkdir(parents=True, exist_ok=True) assignments: list[dict[str, Any]] = [] page_reports: list[dict[str, Any]] = [] profile_cache: dict[Path, dict[str, Any]] = {} profile_usage: dict[str, dict[str, Any]] = {} require_profile_claim_safe = bool(getattr(args, "require_profile_claim_safe", False)) for sample_id, document_path in sorted(source_paths.items()): profile_bundle = _profile_bundle_for_sample( sample_id, single_profile_path=profile_path, profile_map=profile_map, default_profile=default_profile, require_profile_claim_safe=require_profile_claim_safe, cache=profile_cache, ) chunks = profile_bundle["chunks"] profile_key = str(profile_bundle["path"]) profile_usage.setdefault( profile_key, { "profile": profile_key, "profile_sha256": profile_bundle["sha256"], "profile_claim_safe": profile_bundle["profile_claim_safe"], "chunk_count": len(chunks), "sample_ids": [], }, )["sample_ids"].append(sample_id) document = Document.from_dict(_read_json_object(document_path)) for page in document.pages: line_keys: dict[str, tuple[int, float, float, float, float, str]] = {} for line in page.text_lines: chunk, score = _assign_line( line, chunks, x_scale=float(args.x_scale), y_scale=float(args.y_scale), outside_weight=float(args.outside_weight), ) line_id = line.line_id or "" line_keys[line_id] = ( int(chunk["chunk_index"]), line.bbox.y, line.bbox.x, line.bbox.h, line.bbox.w, line_id, ) metadata = line.metadata if isinstance(line.metadata, dict) else {} metadata["reading_order_geometry_profile_assignment"] = { "profile": str(profile_bundle["path"]), "profile_sha256": profile_bundle["sha256"], "chunk_index": int(chunk["chunk_index"]), "score": score, "start_reference_line_id": chunk.get("start_reference_line_id") or "", "end_reference_line_id": chunk.get("end_reference_line_id") or "", "start_line_id": chunk.get("start_line_id") or "", "end_line_id": chunk.get("end_line_id") or "", "claim_boundary": "profile-based reading-order assignment; profile provenance controls claim safety", } line.metadata = metadata assignments.append( { "sample_id": sample_id, "page_index": page.page_index, "line_id": line_id, "profile": str(profile_bundle["path"]), "profile_sha256": profile_bundle["sha256"], "chunk_index": int(chunk["chunk_index"]), "score": score, "bbox": line.bbox.to_list(), "text_length": len(line.text or ""), } ) page.text_lines = sorted(page.text_lines, key=lambda line: line_keys.get(line.line_id or "", (10**9, line.bbox.y, line.bbox.x, line.bbox.h, line.bbox.w, line.line_id or ""))) lines_by_id = {line.line_id or "": line for line in page.text_lines} for block in page.blocks: if not block.line_ids: continue block.line_ids = sorted( block.line_ids, key=lambda line_id: line_keys.get(line_id, (10**9, block.bbox.y, block.bbox.x, block.bbox.h, block.bbox.w, line_id)), ) block_lines = [lines_by_id[line_id] for line_id in block.line_ids if line_id in lines_by_id] if block_lines: block.text = "\n".join(line.text for line in block_lines) page.blocks = sorted(page.blocks, key=lambda block: _block_key(block, line_keys)) for order, block in enumerate(page.blocks, start=1): block.order = order target_dir = out / sample_id _write_document_artifacts(target_dir, document) line_count = sum(len(page.text_lines) for page in document.pages) page_reports.append( { "sample_id": sample_id, "source_document": str(document_path), "source_document_sha256": sha256_file(document_path), "profile": str(profile_bundle["path"]), "profile_sha256": profile_bundle["sha256"], "profile_claim_safe": profile_bundle["profile_claim_safe"], "document_json": str(target_dir / "document.json"), "document_json_sha256": sha256_file(target_dir / "document.json"), "document_md_sha256": sha256_file(target_dir / "document.md"), "document_body_md_sha256": sha256_file(target_dir / "document.body.md"), "line_count": line_count, } ) assignments_path = out / ASSIGNMENTS_JSONL assignments_path.write_text("".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in assignments), encoding="utf-8") report_path = out / REPORT_JSON report = { "tool": "tools/apply_page_reading_order_geometry_profile.py", "created_at_utc": datetime.now(UTC).isoformat(), "source_documents_dir": str(source_documents_dir), "profile": str(profile_path) if profile_path is not None else "", "profile_map": str(profile_map_path) if profile_map_path is not None else "", "profile_map_sha256": sha256_file(profile_map_path) if profile_map_path is not None else "", "profile_sha256": sha256_file(profile_path) if profile_path is not None else "", "profile_claim_safe": all(bool(item["profile_claim_safe"]) for item in profile_usage.values()), "profile_usage": list(profile_usage.values()), "out": str(out), "config": { "x_scale": float(args.x_scale), "y_scale": float(args.y_scale), "outside_weight": float(args.outside_weight), "require_profile_claim_safe": require_profile_claim_safe, }, "chunk_count": sum(int(item["chunk_count"]) for item in profile_usage.values()), "document_count": len(source_paths), "assignment_count": len(assignments), "pages": page_reports, "assignments_jsonl": str(assignments_path), "assignments_jsonl_sha256": sha256_file(assignments_path), "claim_boundary": "profile-based reading-order application; not claim-safe unless profile provenance is claim-safe", } report["report_json"] = str(report_path) report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") return report def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--source-documents-dir", type=Path, required=True) profile_group = parser.add_mutually_exclusive_group(required=True) profile_group.add_argument("--profile", type=Path) profile_group.add_argument("--profile-map", type=Path) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--x-scale", type=float, default=1.0) parser.add_argument("--y-scale", type=float, default=80.0) parser.add_argument("--outside-weight", type=float, default=2.0) parser.add_argument("--require-profile-claim-safe", action="store_true") args = parser.parse_args() report = apply_profile(args) print(report["report_json"]) print(f"documents={report['document_count']} assignments={report['assignment_count']} chunks={report['chunk_count']}") return 0 if __name__ == "__main__": raise SystemExit(main())