Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/apply_learned_reading_order.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 10.6 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/apply_learned_reading_order.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/apply_learned_reading_order.py
-
curl -L -o apply_learned_reading_order.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/apply_learned_reading_order.py
10.6 kB
| #!/usr/bin/env python3 | |
| """Reorder page-OCR documents with the learned reading-order comparator. | |
| Loads a documents dir (``<sample>/document.json`` in the ocrtech Document schema, | |
| ``pages[].blocks[].bbox`` and ``pages[].text_lines[].bbox`` as xywh in | |
| rendered-pixel space), normalizes each block box by page width/height, scores all | |
| ordered box pairs with the trained comparator, aggregates to a total order via | |
| Copeland scoring, rewrites ``block.order`` (which is what | |
| ``ocrtech.schemas.Document.text`` concatenates for scoring), reorders the | |
| ``text_lines`` and ``blocks`` lists to match, and writes the same schema to | |
| ``--out``. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import shutil | |
| import sys | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from typing import Any | |
| import numpy as np | |
| from ocrtech.markdown import render_document_markdown | |
| from ocrtech.manifest import sha256_file | |
| from ocrtech.schemas import Block, Document, Page, TextLine | |
| _TOOL_DIR = Path(__file__).resolve().parent | |
| if str(_TOOL_DIR) not in sys.path: | |
| sys.path.insert(0, str(_TOOL_DIR)) | |
| from reading_order_features import FEATURE_NAMES, copeland_order, page_context, pair_features # noqa: E402 | |
| REPORT_JSON = "learned-reading-order-application.json" | |
| def _read_json_object(path: Path) -> dict[str, Any]: | |
| try: | |
| payload = json.loads(path.read_text(encoding="utf-8")) | |
| except FileNotFoundError as exc: | |
| raise ValueError(f"required JSON file is missing: {path}") from exc | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"invalid JSON in {path}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise ValueError(f"JSON file must contain an object: {path}") | |
| return payload | |
| def _document_paths(documents_dir: Path) -> dict[str, Path]: | |
| if not documents_dir.is_dir(): | |
| raise ValueError(f"documents directory does not exist: {documents_dir}") | |
| paths: dict[str, Path] = {} | |
| for child in sorted(documents_dir.iterdir()): | |
| if not child.is_dir(): | |
| continue | |
| document_path = child / "document.json" | |
| if document_path.is_file(): | |
| paths[child.name] = document_path | |
| if not paths: | |
| raise ValueError(f"documents directory contains no sample document.json files: {documents_dir}") | |
| return paths | |
| class Comparator: | |
| def __init__(self, spec_path: Path) -> None: | |
| spec = _read_json_object(spec_path) | |
| feature_names = spec.get("feature_names") | |
| if feature_names != list(FEATURE_NAMES): | |
| raise ValueError( | |
| "feature-spec feature_names do not match reading_order_features.FEATURE_NAMES; " | |
| "model and inference feature layout are out of sync" | |
| ) | |
| self.predict_type = spec.get("predict_type") | |
| model_rel = spec.get("model_filename") or Path(spec["model_path"]).name | |
| model_path = spec_path.parent / model_rel | |
| if not model_path.is_file(): | |
| model_path = Path(spec["model_path"]) | |
| if not model_path.is_file(): | |
| raise FileNotFoundError(f"model artifact not found next to spec or at recorded path: {model_path}") | |
| self.model_path = model_path | |
| if self.predict_type == "lightgbm": | |
| import lightgbm as lgb | |
| self._model = lgb.Booster(model_file=str(model_path)) | |
| elif self.predict_type == "sklearn": | |
| import joblib | |
| self._model = joblib.load(model_path) | |
| else: | |
| raise ValueError(f"unknown predict_type in feature-spec: {self.predict_type!r}") | |
| self.spec = spec | |
| def predict_prob(self, X: np.ndarray) -> np.ndarray: | |
| if self.predict_type == "lightgbm": | |
| return np.asarray(self._model.predict(X)) | |
| return np.asarray(self._model.predict_proba(X))[:, 1] | |
| def prob_before_matrix(self, boxes: np.ndarray) -> np.ndarray: | |
| n = boxes.shape[0] | |
| ctx = page_context(boxes) | |
| ii, jj = np.meshgrid(np.arange(n), np.arange(n), indexing="ij") | |
| a_boxes = boxes[ii.ravel()] | |
| b_boxes = boxes[jj.ravel()] | |
| X = pair_features(a_boxes, b_boxes, ctx) | |
| prob = self.predict_prob(X).reshape(n, n) | |
| return prob | |
| def _normalize_blocks(blocks: list[Block], width: float, height: float) -> np.ndarray: | |
| if width <= 0 or height <= 0: | |
| raise ValueError(f"page has non-positive dimensions: {width}x{height}") | |
| boxes = np.empty((len(blocks), 4), dtype=np.float64) | |
| for i, block in enumerate(blocks): | |
| b = block.bbox | |
| x0 = b.x / width | |
| y0 = b.y / height | |
| x1 = (b.x + b.w) / width | |
| y1 = (b.y + b.h) / height | |
| boxes[i] = [x0, y0, x1, y1] | |
| return boxes | |
| def _geometric_preorder(boxes: np.ndarray) -> np.ndarray: | |
| """Stable geometric pre-order (top-to-bottom, then left-to-right) so Copeland | |
| ties fall back to a sensible reading order.""" | |
| cy = (boxes[:, 1] + boxes[:, 3]) / 2.0 | |
| cx = (boxes[:, 0] + boxes[:, 2]) / 2.0 | |
| return np.lexsort((cx, cy)) # primary key = cy (last), secondary = cx | |
| def _reorder_page(page: Page, comparator: Comparator) -> dict[str, Any]: | |
| blocks = list(page.blocks) | |
| if len(blocks) < 2: | |
| return {"page_index": page.page_index, "blocks": len(blocks), "reordered": False} | |
| boxes = _normalize_blocks(blocks, float(page.width or 0.0), float(page.height or 0.0)) | |
| pre = _geometric_preorder(boxes) | |
| pre_blocks = [blocks[i] for i in pre] | |
| pre_boxes = boxes[pre] | |
| prob = comparator.prob_before_matrix(pre_boxes) | |
| order = copeland_order(prob) # indices into pre_blocks, ties keep geometric order | |
| ordered_blocks = [pre_blocks[i] for i in order] | |
| for new_order, block in enumerate(ordered_blocks): | |
| block.order = new_order | |
| page.blocks = ordered_blocks | |
| # Reorder text_lines to follow the new block order via line_ids. | |
| lines_by_id: dict[str, TextLine] = {} | |
| for line in page.text_lines: | |
| key = line.line_id or "" | |
| if key: | |
| lines_by_id[key] = line | |
| ordered_lines: list[TextLine] = [] | |
| consumed: set[int] = set() | |
| for block in ordered_blocks: | |
| for line_id in block.line_ids: | |
| line = lines_by_id.get(line_id) | |
| if line is not None and id(line) not in consumed: | |
| ordered_lines.append(line) | |
| consumed.add(id(line)) | |
| for line in page.text_lines: # append any orphan lines not covered by a block | |
| if id(line) not in consumed: | |
| ordered_lines.append(line) | |
| consumed.add(id(line)) | |
| page.text_lines = ordered_lines | |
| page.metadata = dict(page.metadata or {}) | |
| page.metadata["learned_reading_order"] = { | |
| "model_path": str(comparator.model_path), | |
| "predict_type": comparator.predict_type, | |
| "aggregation": "copeland", | |
| "block_count": len(ordered_blocks), | |
| } | |
| return {"page_index": page.page_index, "blocks": len(ordered_blocks), "reordered": True} | |
| def _write_document_artifacts(document_dir: Path, document: Document) -> None: | |
| document_dir.mkdir(parents=True, exist_ok=True) | |
| (document_dir / "document.json").write_text( | |
| json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8" | |
| ) | |
| (document_dir / "document.md").write_text(render_document_markdown(document), encoding="utf-8") | |
| (document_dir / "document.body.md").write_text( | |
| render_document_markdown(document, include_structural_roles=False), encoding="utf-8" | |
| ) | |
| def apply(args: argparse.Namespace) -> dict[str, Any]: | |
| documents_dir = Path(args.documents_dir) | |
| out = Path(args.out) | |
| spec_path = Path(args.feature_spec) | |
| if documents_dir.resolve() == out.resolve(): | |
| raise ValueError("--out must not equal --documents-dir") | |
| if out.exists() and any(out.iterdir()): | |
| if not args.force: | |
| raise FileExistsError(f"output directory exists and is not empty: {out}") | |
| if out.resolve() in {Path.cwd().resolve(), Path("/").resolve()}: | |
| raise ValueError(f"refusing to remove unsafe output directory: {out}") | |
| shutil.rmtree(out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| comparator = Comparator(spec_path) | |
| source_paths = _document_paths(documents_dir) | |
| docs: list[dict[str, Any]] = [] | |
| for sample_id, document_path in sorted(source_paths.items()): | |
| document = Document.from_dict(_read_json_object(document_path)) | |
| page_reports = [_reorder_page(page, comparator) for page in document.pages] | |
| document.metadata = dict(document.metadata or {}) | |
| document.metadata["learned_reading_order"] = { | |
| "model_path": str(comparator.model_path), | |
| "feature_spec": str(spec_path), | |
| } | |
| target_dir = out / sample_id | |
| _write_document_artifacts(target_dir, document) | |
| docs.append( | |
| { | |
| "sample_id": sample_id, | |
| "source_document": str(document_path), | |
| "source_document_sha256": sha256_file(document_path), | |
| "document_json": str(target_dir / "document.json"), | |
| "document_json_sha256": sha256_file(target_dir / "document.json"), | |
| "page_reports": page_reports, | |
| } | |
| ) | |
| report = { | |
| "tool": "tools/apply_learned_reading_order.py", | |
| "created_at_utc": datetime.now(timezone.utc).isoformat(), | |
| "command_line": " ".join(sys.argv), | |
| "documents_dir": str(documents_dir), | |
| "out": str(out), | |
| "feature_spec": str(spec_path), | |
| "model_path": str(comparator.model_path), | |
| "predict_type": comparator.predict_type, | |
| "aggregation": "copeland", | |
| "document_count": len(docs), | |
| "documents": docs, | |
| } | |
| report_path = out / REPORT_JSON | |
| report["report_json"] = str(report_path) | |
| report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| return report | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--documents-dir", required=True) | |
| parser.add_argument("--out", required=True) | |
| parser.add_argument("--feature-spec", required=True, help="feature-spec.json emitted by the trainer") | |
| parser.add_argument("--force", action="store_true") | |
| args = parser.parse_args(argv) | |
| report = apply(args) | |
| print(report["report_json"]) | |
| print(f"documents={report['document_count']} model={report['predict_type']} out={report['out']}", flush=True) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |