File size: 4,694 Bytes
7b2177e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | # GENERATED by hub/build_hub_package.py from src/bodhan_genai/ocr/engine/reconstruct.py -- do not edit.
# Vendored so this repo is self-contained: `pip install transformers torch pillow` is the
# whole install. See indic_doc_parser.py for usage.
"""Assemble transcribed blocks into the page's markdown.
stdlib-only -- importable with no GPU stack and no PIL.
"""
from __future__ import annotations
import re
from idp_types import Block
from idp_contract import DROP_TYPES
# Dashes as codepoints: several of the seven are indistinguishable in a source file.
# 002D HYPHEN-MINUS | 2010-2014 HYPHEN..EM DASH | 2212 MINUS SIGN
_HYPHEN_BREAK = re.compile("(\\w)[-\\u2010-\\u2014\\u2212]\\n[ \\t]*(\\w)")
_MATH_DELIMITERS = ("$", "\\[", "\\(")
# A math span: $$...$$ (display) or $...$ (inline).
_MATH_SPAN = re.compile(r"\$\$(.+?)\$\$|(?<!\$)\$(?!\$)([^$\n]+?)\$(?!\$)", re.S)
# Runs of the scripts IndicBlockOCR transcribes: Arabic (Urdu, Kashmiri, Sindhi),
# Devanagari..Malayalam, and Ol Chiki (Santali). Intervening spaces and the ZW(N)J joiners that
# Indic shaping relies on are kept inside the run so one run does not fragment into many.
# The run must both start and end on a script character, so a trailing space stays outside the
# \text{} and keeps separating it from what follows.
_NON_LATIN_RUN = re.compile(
"[\\u0600-\\u06ff\\u0900-\\u0d7f\\u1c50-\\u1c7f]"
"(?:[\\u0600-\\u06ff\\u0900-\\u0d7f\\u1c50-\\u1c7f \\u200c\\u200d]*"
"[\\u0600-\\u06ff\\u0900-\\u0d7f\\u1c50-\\u1c7f])?"
)
_TEXT_CMD = re.compile(r"\\text\{[^{}]*\}")
def dehyphenate(text: str) -> str:
"""Rejoin words split by a hyphen at a line break.
Iterates to a fixpoint: re.sub matches non-overlappingly, so in "a-\\nb-\\nc" the first pass
consumes the "b" the second break needs.
"""
previous = None
while previous != text:
previous = text
text = _HYPHEN_BREAK.sub(r"\1\2", text)
return text
def _repair_expression(tex: str, display: bool) -> str:
"""Make one transcribed expression valid LaTeX.
Two repairs, both for things the recognizer emits that no LaTeX engine accepts:
* **Indic script in math mode.** ``$$প্রোটন = 9$$`` is invalid -- math mode has no glyphs for
those codepoints, so KaTeX, MathJax and a real TeX run all fail on it. Each run is wrapped
in ``\\text{}``, which is what the recognizer itself does when it gets it right. Runs
already inside ``\\text{}`` are left alone.
* **Bare newlines in display math**, which are a syntax error; ``\\\\`` is the row separator.
"""
protected: list[str] = []
def stash(match: re.Match) -> str:
protected.append(match.group(0))
return f"\x00{len(protected) - 1}\x00"
tex = _TEXT_CMD.sub(stash, tex)
tex = _NON_LATIN_RUN.sub(lambda m: f"\\text{{{m.group(0)}}}", tex)
tex = re.sub(r"\x00(\d+)\x00", lambda m: protected[int(m.group(1))], tex)
if display:
tex = re.sub(r"\s*\n\s*", r" \\\\ ", tex.strip())
return tex
def repair_math(text: str) -> str:
"""Repair every math span in a markdown string. Prose outside ``$`` is untouched."""
def fix(match: re.Match) -> str:
display = match.group(1) is not None
inner = _repair_expression(match.group(1) or match.group(2), display)
return f"$${inner}$$" if display else f"${inner}$"
return _MATH_SPAN.sub(fix, text)
def reconstruct(blocks: list[Block], repair: bool = True) -> str:
"""Reading-ordered markdown. Blocks with no text contribute nothing but are not removed --
they still appear in the JSON with text "" .
``repair=False`` emits the recognizer's math verbatim, including expressions no LaTeX engine
can render. Only useful for comparing byte-for-byte against output produced before the repair
existed.
"""
kept = sorted((b for b in blocks if b.type not in DROP_TYPES), key=lambda b: b.order)
parts = []
for block in kept:
text = (block.text or "").strip()
if not text:
continue
# Bare LaTeX would render as literal source, so wrap it -- but only when the recognizer
# supplied no delimiters at all. Checking just the first character is not enough: an
# Equation block often comes back as a prose prefix followed by already-delimited math
# ("বা, $\\frac{a}{b}$"), and wrapping that produces "$$...$$$", which nothing can parse.
if block.type == "Equation" and "$" not in text and not text.startswith(_MATH_DELIMITERS):
text = f"$${text}$$"
parts.append(text)
markdown = dehyphenate("\n\n".join(parts))
return repair_math(markdown) if repair else markdown
|