#!/usr/bin/env python3 """Reorder page-OCR documents with the learned reading-order comparator. Loads a documents dir (``/document.json`` in the ocrtech Document schema, ``pages[].blocks[].bbox`` and ``pages[].text_lines[].bbox`` as xywh in rendered-pixel space), normalizes each block box by page width/height, scores all ordered box pairs with the trained comparator, aggregates to a total order via Copeland scoring, rewrites ``block.order`` (which is what ``ocrtech.schemas.Document.text`` concatenates for scoring), reorders the ``text_lines`` and ``blocks`` lists to match, and writes the same schema to ``--out``. """ from __future__ import annotations import argparse import json import shutil import sys from datetime import datetime, timezone from pathlib import Path from typing import Any import numpy as np from ocrtech.markdown import render_document_markdown from ocrtech.manifest import sha256_file from ocrtech.schemas import Block, Document, Page, TextLine _TOOL_DIR = Path(__file__).resolve().parent if str(_TOOL_DIR) not in sys.path: sys.path.insert(0, str(_TOOL_DIR)) from reading_order_features import FEATURE_NAMES, copeland_order, page_context, pair_features # noqa: E402 REPORT_JSON = "learned-reading-order-application.json" def _read_json_object(path: Path) -> dict[str, Any]: try: payload = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError as exc: raise ValueError(f"required JSON file is missing: {path}") from exc except json.JSONDecodeError as exc: raise ValueError(f"invalid JSON in {path}: {exc}") from exc if not isinstance(payload, dict): raise ValueError(f"JSON file must contain an object: {path}") return payload def _document_paths(documents_dir: Path) -> dict[str, Path]: if not documents_dir.is_dir(): raise ValueError(f"documents directory does not exist: {documents_dir}") paths: dict[str, Path] = {} for child in sorted(documents_dir.iterdir()): if not child.is_dir(): continue document_path = child / "document.json" if document_path.is_file(): paths[child.name] = document_path if not paths: raise ValueError(f"documents directory contains no sample document.json files: {documents_dir}") return paths class Comparator: def __init__(self, spec_path: Path) -> None: spec = _read_json_object(spec_path) feature_names = spec.get("feature_names") if feature_names != list(FEATURE_NAMES): raise ValueError( "feature-spec feature_names do not match reading_order_features.FEATURE_NAMES; " "model and inference feature layout are out of sync" ) self.predict_type = spec.get("predict_type") model_rel = spec.get("model_filename") or Path(spec["model_path"]).name model_path = spec_path.parent / model_rel if not model_path.is_file(): model_path = Path(spec["model_path"]) if not model_path.is_file(): raise FileNotFoundError(f"model artifact not found next to spec or at recorded path: {model_path}") self.model_path = model_path if self.predict_type == "lightgbm": import lightgbm as lgb self._model = lgb.Booster(model_file=str(model_path)) elif self.predict_type == "sklearn": import joblib self._model = joblib.load(model_path) else: raise ValueError(f"unknown predict_type in feature-spec: {self.predict_type!r}") self.spec = spec def predict_prob(self, X: np.ndarray) -> np.ndarray: if self.predict_type == "lightgbm": return np.asarray(self._model.predict(X)) return np.asarray(self._model.predict_proba(X))[:, 1] def prob_before_matrix(self, boxes: np.ndarray) -> np.ndarray: n = boxes.shape[0] ctx = page_context(boxes) ii, jj = np.meshgrid(np.arange(n), np.arange(n), indexing="ij") a_boxes = boxes[ii.ravel()] b_boxes = boxes[jj.ravel()] X = pair_features(a_boxes, b_boxes, ctx) prob = self.predict_prob(X).reshape(n, n) return prob def _normalize_blocks(blocks: list[Block], width: float, height: float) -> np.ndarray: if width <= 0 or height <= 0: raise ValueError(f"page has non-positive dimensions: {width}x{height}") boxes = np.empty((len(blocks), 4), dtype=np.float64) for i, block in enumerate(blocks): b = block.bbox x0 = b.x / width y0 = b.y / height x1 = (b.x + b.w) / width y1 = (b.y + b.h) / height boxes[i] = [x0, y0, x1, y1] return boxes def _geometric_preorder(boxes: np.ndarray) -> np.ndarray: """Stable geometric pre-order (top-to-bottom, then left-to-right) so Copeland ties fall back to a sensible reading order.""" cy = (boxes[:, 1] + boxes[:, 3]) / 2.0 cx = (boxes[:, 0] + boxes[:, 2]) / 2.0 return np.lexsort((cx, cy)) # primary key = cy (last), secondary = cx def _reorder_page(page: Page, comparator: Comparator) -> dict[str, Any]: blocks = list(page.blocks) if len(blocks) < 2: return {"page_index": page.page_index, "blocks": len(blocks), "reordered": False} boxes = _normalize_blocks(blocks, float(page.width or 0.0), float(page.height or 0.0)) pre = _geometric_preorder(boxes) pre_blocks = [blocks[i] for i in pre] pre_boxes = boxes[pre] prob = comparator.prob_before_matrix(pre_boxes) order = copeland_order(prob) # indices into pre_blocks, ties keep geometric order ordered_blocks = [pre_blocks[i] for i in order] for new_order, block in enumerate(ordered_blocks): block.order = new_order page.blocks = ordered_blocks # Reorder text_lines to follow the new block order via line_ids. lines_by_id: dict[str, TextLine] = {} for line in page.text_lines: key = line.line_id or "" if key: lines_by_id[key] = line ordered_lines: list[TextLine] = [] consumed: set[int] = set() for block in ordered_blocks: for line_id in block.line_ids: line = lines_by_id.get(line_id) if line is not None and id(line) not in consumed: ordered_lines.append(line) consumed.add(id(line)) for line in page.text_lines: # append any orphan lines not covered by a block if id(line) not in consumed: ordered_lines.append(line) consumed.add(id(line)) page.text_lines = ordered_lines page.metadata = dict(page.metadata or {}) page.metadata["learned_reading_order"] = { "model_path": str(comparator.model_path), "predict_type": comparator.predict_type, "aggregation": "copeland", "block_count": len(ordered_blocks), } return {"page_index": page.page_index, "blocks": len(ordered_blocks), "reordered": True} def _write_document_artifacts(document_dir: Path, document: Document) -> None: document_dir.mkdir(parents=True, exist_ok=True) (document_dir / "document.json").write_text( json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) (document_dir / "document.md").write_text(render_document_markdown(document), encoding="utf-8") (document_dir / "document.body.md").write_text( render_document_markdown(document, include_structural_roles=False), encoding="utf-8" ) def apply(args: argparse.Namespace) -> dict[str, Any]: documents_dir = Path(args.documents_dir) out = Path(args.out) spec_path = Path(args.feature_spec) if documents_dir.resolve() == out.resolve(): raise ValueError("--out must not equal --documents-dir") if out.exists() and any(out.iterdir()): if not args.force: raise FileExistsError(f"output directory exists and is not empty: {out}") if out.resolve() in {Path.cwd().resolve(), Path("/").resolve()}: raise ValueError(f"refusing to remove unsafe output directory: {out}") shutil.rmtree(out) out.mkdir(parents=True, exist_ok=True) comparator = Comparator(spec_path) source_paths = _document_paths(documents_dir) docs: list[dict[str, Any]] = [] for sample_id, document_path in sorted(source_paths.items()): document = Document.from_dict(_read_json_object(document_path)) page_reports = [_reorder_page(page, comparator) for page in document.pages] document.metadata = dict(document.metadata or {}) document.metadata["learned_reading_order"] = { "model_path": str(comparator.model_path), "feature_spec": str(spec_path), } target_dir = out / sample_id _write_document_artifacts(target_dir, document) docs.append( { "sample_id": sample_id, "source_document": str(document_path), "source_document_sha256": sha256_file(document_path), "document_json": str(target_dir / "document.json"), "document_json_sha256": sha256_file(target_dir / "document.json"), "page_reports": page_reports, } ) report = { "tool": "tools/apply_learned_reading_order.py", "created_at_utc": datetime.now(timezone.utc).isoformat(), "command_line": " ".join(sys.argv), "documents_dir": str(documents_dir), "out": str(out), "feature_spec": str(spec_path), "model_path": str(comparator.model_path), "predict_type": comparator.predict_type, "aggregation": "copeland", "document_count": len(docs), "documents": docs, } report_path = out / REPORT_JSON report["report_json"] = str(report_path) report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") return report def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--documents-dir", required=True) parser.add_argument("--out", required=True) parser.add_argument("--feature-spec", required=True, help="feature-spec.json emitted by the trainer") parser.add_argument("--force", action="store_true") args = parser.parse_args(argv) report = apply(args) print(report["report_json"]) print(f"documents={report['document_count']} model={report['predict_type']} out={report['out']}", flush=True) return 0 if __name__ == "__main__": raise SystemExit(main())