File size: 7,379 Bytes
f9bc0c1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
"""Helpers for normalizing external OCR baseline outputs."""

from __future__ import annotations

import json
import re
from pathlib import Path
from typing import Any

from .schemas import BBox, Block, Document, Page, Table, TableCell, TextLine


_PAGE_BREAK_PATTERN = re.compile(r"(?:\n\s*<!--\s*pagebreak\s*-->\s*\n)|\f", re.IGNORECASE)
_TABLE_SEPARATOR_PATTERN = re.compile(r"^\s*\|?\s*:?-{3,}:?\s*(?:\|\s*:?-{3,}:?\s*)+\|?\s*$")


def bundle_page_paths(input_path: Path) -> list[Path]:
    """Return page image paths for a directory bundle or the single input path."""
    if not input_path.is_dir():
        return [input_path]
    allowed_suffixes = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp"}
    page_paths = [path for path in sorted(input_path.iterdir()) if path.is_file() and path.suffix.lower() in allowed_suffixes]
    if not page_paths:
        raise ValueError(f"page bundle directory has no supported image files: {input_path}")
    return page_paths


def materialize_external_markdown(
    source_path: Path,
    output_dir: Path,
    markdown_text: str,
    *,
    metadata: dict[str, Any] | None = None,
    raw_payload: Any | None = None,
) -> Document:
    """Write normalized markdown and document JSON for an external OCR result."""
    output_dir.mkdir(parents=True, exist_ok=True)
    cleaned = _clean_markdown(markdown_text)
    document = document_from_markdown(source_path, cleaned, metadata=metadata)
    (output_dir / "document.md").write_text(cleaned + ("\n" if cleaned and not cleaned.endswith("\n") else ""), encoding="utf-8")
    (output_dir / "document.json").write_text(
        json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n",
        encoding="utf-8",
    )
    if raw_payload is not None:
        (output_dir / "raw-output.json").write_text(
            json.dumps(raw_payload, ensure_ascii=False, indent=2) + "\n",
            encoding="utf-8",
        )
    return document


def document_from_markdown(source_path: Path, markdown_text: str, *, metadata: dict[str, Any] | None = None) -> Document:
    """Build a minimal structured document from markdown output."""
    pages: list[Page] = []
    tables: list[Table] = []
    markdown_pages = [part.strip("\n") for part in _PAGE_BREAK_PATTERN.split(markdown_text)] or [markdown_text]
    for page_index, page_markdown in enumerate(markdown_pages):
        page = _page_from_markdown(page_markdown, page_index=page_index, global_tables=tables)
        pages.append(page)
    return Document(source_path=str(source_path), pages=pages, tables=tables, figures=[], metadata=dict(metadata or {}))


def _clean_markdown(value: str) -> str:
    text = value.replace("\r\n", "\n").replace("\r", "\n").strip()
    if text.startswith("```") and text.endswith("```"):
        lines = text.splitlines()
        if len(lines) >= 2:
            text = "\n".join(lines[1:-1]).strip()
    return text


def _page_from_markdown(markdown_text: str, *, page_index: int, global_tables: list[Table]) -> Page:
    text_lines: list[TextLine] = []
    blocks: list[Block] = []
    lines = markdown_text.splitlines()
    order = 0
    y = 0.0
    index = 0
    while index < len(lines):
        line = lines[index]
        stripped = line.strip()
        if not stripped:
            index += 1
            y += 20
            continue
        table_result = _parse_markdown_table(lines, start=index)
        if table_result is not None:
            table, consumed, table_text = table_result
            table.page_index = page_index
            global_tables.append(table)
            blocks.append(
                Block(
                    block_id=f"p{page_index}-b{order}",
                    block_type="table",
                    page_index=page_index,
                    bbox=table.bbox,
                    order=order,
                    text=table_text,
                    table_id=table.table_id,
                )
            )
            order += 1
            y = table.bbox.bottom + 24
            index += consumed
            continue
        text = _strip_markdown_prefix(stripped)
        bbox = BBox(0, y, max(50, len(text) * 9), 22)
        line_id = f"p{page_index}-l{len(text_lines)}"
        text_lines.append(TextLine(text=text, bbox=bbox, confidence=None, page_index=page_index, line_id=line_id))
        block_type = "title" if stripped.startswith("#") else "caption" if _looks_like_caption(text) else "text"
        blocks.append(
            Block(
                block_id=f"p{page_index}-b{order}",
                block_type=block_type,
                page_index=page_index,
                bbox=bbox,
                order=order,
                text=text,
                line_ids=[line_id],
            )
        )
        order += 1
        y += 28
        index += 1
    return Page(page_index=page_index, text_lines=text_lines, blocks=blocks, metadata={"source": "external-markdown"})


def _parse_markdown_table(lines: list[str], *, start: int) -> tuple[Table, int, str] | None:
    if start + 1 >= len(lines):
        return None
    header_line = lines[start].strip()
    separator_line = lines[start + 1].strip()
    if "|" not in header_line or not _TABLE_SEPARATOR_PATTERN.match(separator_line):
        return None
    data_lines: list[str] = [lines[start], lines[start + 1]]
    index = start + 2
    while index < len(lines):
        candidate = lines[index].strip()
        if not candidate or "|" not in candidate:
            break
        data_lines.append(lines[index])
        index += 1
    rows = [_split_markdown_row(item) for item in data_lines if "|" in item]
    if len(rows) < 2:
        return None
    header = rows[0]
    body = rows[2:] if len(rows) > 2 else []
    cells: list[TableCell] = []
    row_count = 0
    if header:
        for col_index, value in enumerate(header):
            cells.append(TableCell(row=0, col=col_index, text=value))
        row_count = 1
    for body_row in body:
        for col_index, value in enumerate(body_row):
            cells.append(TableCell(row=row_count, col=col_index, text=value))
        row_count += 1
    col_count = max((len(row) for row in [header, *body]), default=1)
    table_height = max(32.0, 28.0 * max(1, row_count))
    table_width = max(120.0, 120.0 * col_count)
    table_text = "\n".join(line.rstrip() for line in data_lines)
    table = Table(
        table_id=f"table-{start}",
        page_index=0,
        bbox=BBox(0, start * 28.0, table_width, table_height),
        cells=cells,
        html=None,
        caption=None,
        metadata={"source": "external-markdown"},
    )
    return table, len(data_lines), table_text


def _split_markdown_row(line: str) -> list[str]:
    trimmed = line.strip()
    if trimmed.startswith("|"):
        trimmed = trimmed[1:]
    if trimmed.endswith("|"):
        trimmed = trimmed[:-1]
    return [part.strip() for part in trimmed.split("|")]


def _strip_markdown_prefix(text: str) -> str:
    value = text.lstrip("#").strip() if text.startswith("#") else text
    if value.startswith("- "):
        return value[2:].strip()
    if value.startswith("* "):
        return value[2:].strip()
    return value


def _looks_like_caption(text: str) -> bool:
    lowered = text.casefold()
    return lowered.startswith("figure ") or lowered.startswith("fig. ") or lowered.startswith("चित्र ")