File size: 10,588 Bytes
d82db76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
#!/usr/bin/env python3
"""Reorder page-OCR documents with the learned reading-order comparator.

Loads a documents dir (``<sample>/document.json`` in the ocrtech Document schema,
``pages[].blocks[].bbox`` and ``pages[].text_lines[].bbox`` as xywh in
rendered-pixel space), normalizes each block box by page width/height, scores all
ordered box pairs with the trained comparator, aggregates to a total order via
Copeland scoring, rewrites ``block.order`` (which is what
``ocrtech.schemas.Document.text`` concatenates for scoring), reorders the
``text_lines`` and ``blocks`` lists to match, and writes the same schema to
``--out``.
"""

from __future__ import annotations

import argparse
import json
import shutil
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

import numpy as np

from ocrtech.markdown import render_document_markdown
from ocrtech.manifest import sha256_file
from ocrtech.schemas import Block, Document, Page, TextLine

_TOOL_DIR = Path(__file__).resolve().parent
if str(_TOOL_DIR) not in sys.path:
    sys.path.insert(0, str(_TOOL_DIR))

from reading_order_features import FEATURE_NAMES, copeland_order, page_context, pair_features  # noqa: E402


REPORT_JSON = "learned-reading-order-application.json"


def _read_json_object(path: Path) -> dict[str, Any]:
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except FileNotFoundError as exc:
        raise ValueError(f"required JSON file is missing: {path}") from exc
    except json.JSONDecodeError as exc:
        raise ValueError(f"invalid JSON in {path}: {exc}") from exc
    if not isinstance(payload, dict):
        raise ValueError(f"JSON file must contain an object: {path}")
    return payload


def _document_paths(documents_dir: Path) -> dict[str, Path]:
    if not documents_dir.is_dir():
        raise ValueError(f"documents directory does not exist: {documents_dir}")
    paths: dict[str, Path] = {}
    for child in sorted(documents_dir.iterdir()):
        if not child.is_dir():
            continue
        document_path = child / "document.json"
        if document_path.is_file():
            paths[child.name] = document_path
    if not paths:
        raise ValueError(f"documents directory contains no sample document.json files: {documents_dir}")
    return paths


class Comparator:
    def __init__(self, spec_path: Path) -> None:
        spec = _read_json_object(spec_path)
        feature_names = spec.get("feature_names")
        if feature_names != list(FEATURE_NAMES):
            raise ValueError(
                "feature-spec feature_names do not match reading_order_features.FEATURE_NAMES; "
                "model and inference feature layout are out of sync"
            )
        self.predict_type = spec.get("predict_type")
        model_rel = spec.get("model_filename") or Path(spec["model_path"]).name
        model_path = spec_path.parent / model_rel
        if not model_path.is_file():
            model_path = Path(spec["model_path"])
        if not model_path.is_file():
            raise FileNotFoundError(f"model artifact not found next to spec or at recorded path: {model_path}")
        self.model_path = model_path
        if self.predict_type == "lightgbm":
            import lightgbm as lgb

            self._model = lgb.Booster(model_file=str(model_path))
        elif self.predict_type == "sklearn":
            import joblib

            self._model = joblib.load(model_path)
        else:
            raise ValueError(f"unknown predict_type in feature-spec: {self.predict_type!r}")
        self.spec = spec

    def predict_prob(self, X: np.ndarray) -> np.ndarray:
        if self.predict_type == "lightgbm":
            return np.asarray(self._model.predict(X))
        return np.asarray(self._model.predict_proba(X))[:, 1]

    def prob_before_matrix(self, boxes: np.ndarray) -> np.ndarray:
        n = boxes.shape[0]
        ctx = page_context(boxes)
        ii, jj = np.meshgrid(np.arange(n), np.arange(n), indexing="ij")
        a_boxes = boxes[ii.ravel()]
        b_boxes = boxes[jj.ravel()]
        X = pair_features(a_boxes, b_boxes, ctx)
        prob = self.predict_prob(X).reshape(n, n)
        return prob


def _normalize_blocks(blocks: list[Block], width: float, height: float) -> np.ndarray:
    if width <= 0 or height <= 0:
        raise ValueError(f"page has non-positive dimensions: {width}x{height}")
    boxes = np.empty((len(blocks), 4), dtype=np.float64)
    for i, block in enumerate(blocks):
        b = block.bbox
        x0 = b.x / width
        y0 = b.y / height
        x1 = (b.x + b.w) / width
        y1 = (b.y + b.h) / height
        boxes[i] = [x0, y0, x1, y1]
    return boxes


def _geometric_preorder(boxes: np.ndarray) -> np.ndarray:
    """Stable geometric pre-order (top-to-bottom, then left-to-right) so Copeland
    ties fall back to a sensible reading order."""
    cy = (boxes[:, 1] + boxes[:, 3]) / 2.0
    cx = (boxes[:, 0] + boxes[:, 2]) / 2.0
    return np.lexsort((cx, cy))  # primary key = cy (last), secondary = cx


def _reorder_page(page: Page, comparator: Comparator) -> dict[str, Any]:
    blocks = list(page.blocks)
    if len(blocks) < 2:
        return {"page_index": page.page_index, "blocks": len(blocks), "reordered": False}

    boxes = _normalize_blocks(blocks, float(page.width or 0.0), float(page.height or 0.0))
    pre = _geometric_preorder(boxes)
    pre_blocks = [blocks[i] for i in pre]
    pre_boxes = boxes[pre]

    prob = comparator.prob_before_matrix(pre_boxes)
    order = copeland_order(prob)  # indices into pre_blocks, ties keep geometric order
    ordered_blocks = [pre_blocks[i] for i in order]

    for new_order, block in enumerate(ordered_blocks):
        block.order = new_order
    page.blocks = ordered_blocks

    # Reorder text_lines to follow the new block order via line_ids.
    lines_by_id: dict[str, TextLine] = {}
    for line in page.text_lines:
        key = line.line_id or ""
        if key:
            lines_by_id[key] = line
    ordered_lines: list[TextLine] = []
    consumed: set[int] = set()
    for block in ordered_blocks:
        for line_id in block.line_ids:
            line = lines_by_id.get(line_id)
            if line is not None and id(line) not in consumed:
                ordered_lines.append(line)
                consumed.add(id(line))
    for line in page.text_lines:  # append any orphan lines not covered by a block
        if id(line) not in consumed:
            ordered_lines.append(line)
            consumed.add(id(line))
    page.text_lines = ordered_lines

    page.metadata = dict(page.metadata or {})
    page.metadata["learned_reading_order"] = {
        "model_path": str(comparator.model_path),
        "predict_type": comparator.predict_type,
        "aggregation": "copeland",
        "block_count": len(ordered_blocks),
    }
    return {"page_index": page.page_index, "blocks": len(ordered_blocks), "reordered": True}


def _write_document_artifacts(document_dir: Path, document: Document) -> None:
    document_dir.mkdir(parents=True, exist_ok=True)
    (document_dir / "document.json").write_text(
        json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
    )
    (document_dir / "document.md").write_text(render_document_markdown(document), encoding="utf-8")
    (document_dir / "document.body.md").write_text(
        render_document_markdown(document, include_structural_roles=False), encoding="utf-8"
    )


def apply(args: argparse.Namespace) -> dict[str, Any]:
    documents_dir = Path(args.documents_dir)
    out = Path(args.out)
    spec_path = Path(args.feature_spec)
    if documents_dir.resolve() == out.resolve():
        raise ValueError("--out must not equal --documents-dir")
    if out.exists() and any(out.iterdir()):
        if not args.force:
            raise FileExistsError(f"output directory exists and is not empty: {out}")
        if out.resolve() in {Path.cwd().resolve(), Path("/").resolve()}:
            raise ValueError(f"refusing to remove unsafe output directory: {out}")
        shutil.rmtree(out)
    out.mkdir(parents=True, exist_ok=True)

    comparator = Comparator(spec_path)
    source_paths = _document_paths(documents_dir)
    docs: list[dict[str, Any]] = []
    for sample_id, document_path in sorted(source_paths.items()):
        document = Document.from_dict(_read_json_object(document_path))
        page_reports = [_reorder_page(page, comparator) for page in document.pages]
        document.metadata = dict(document.metadata or {})
        document.metadata["learned_reading_order"] = {
            "model_path": str(comparator.model_path),
            "feature_spec": str(spec_path),
        }
        target_dir = out / sample_id
        _write_document_artifacts(target_dir, document)
        docs.append(
            {
                "sample_id": sample_id,
                "source_document": str(document_path),
                "source_document_sha256": sha256_file(document_path),
                "document_json": str(target_dir / "document.json"),
                "document_json_sha256": sha256_file(target_dir / "document.json"),
                "page_reports": page_reports,
            }
        )
    report = {
        "tool": "tools/apply_learned_reading_order.py",
        "created_at_utc": datetime.now(timezone.utc).isoformat(),
        "command_line": " ".join(sys.argv),
        "documents_dir": str(documents_dir),
        "out": str(out),
        "feature_spec": str(spec_path),
        "model_path": str(comparator.model_path),
        "predict_type": comparator.predict_type,
        "aggregation": "copeland",
        "document_count": len(docs),
        "documents": docs,
    }
    report_path = out / REPORT_JSON
    report["report_json"] = str(report_path)
    report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    return report


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--documents-dir", required=True)
    parser.add_argument("--out", required=True)
    parser.add_argument("--feature-spec", required=True, help="feature-spec.json emitted by the trainer")
    parser.add_argument("--force", action="store_true")
    args = parser.parse_args(argv)
    report = apply(args)
    print(report["report_json"])
    print(f"documents={report['document_count']} model={report['predict_type']} out={report['out']}", flush=True)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())