# GENERATED by hub/build_hub_package.py from src/bodhan_genai/ocr/engine/reconstruct.py -- do not edit. # Vendored so this repo is self-contained: `pip install transformers torch pillow` is the # whole install. See indic_doc_parser.py for usage. """Assemble transcribed blocks into the page's markdown. stdlib-only -- importable with no GPU stack and no PIL. """ from __future__ import annotations import re from idp_types import Block from idp_contract import DROP_TYPES # Dashes as codepoints: several of the seven are indistinguishable in a source file. # 002D HYPHEN-MINUS | 2010-2014 HYPHEN..EM DASH | 2212 MINUS SIGN _HYPHEN_BREAK = re.compile("(\\w)[-\\u2010-\\u2014\\u2212]\\n[ \\t]*(\\w)") _MATH_DELIMITERS = ("$", "\\[", "\\(") # A math span: $$...$$ (display) or $...$ (inline). _MATH_SPAN = re.compile(r"\$\$(.+?)\$\$|(? str: """Rejoin words split by a hyphen at a line break. Iterates to a fixpoint: re.sub matches non-overlappingly, so in "a-\\nb-\\nc" the first pass consumes the "b" the second break needs. """ previous = None while previous != text: previous = text text = _HYPHEN_BREAK.sub(r"\1\2", text) return text def _repair_expression(tex: str, display: bool) -> str: """Make one transcribed expression valid LaTeX. Two repairs, both for things the recognizer emits that no LaTeX engine accepts: * **Indic script in math mode.** ``$$প্রোটন = 9$$`` is invalid -- math mode has no glyphs for those codepoints, so KaTeX, MathJax and a real TeX run all fail on it. Each run is wrapped in ``\\text{}``, which is what the recognizer itself does when it gets it right. Runs already inside ``\\text{}`` are left alone. * **Bare newlines in display math**, which are a syntax error; ``\\\\`` is the row separator. """ protected: list[str] = [] def stash(match: re.Match) -> str: protected.append(match.group(0)) return f"\x00{len(protected) - 1}\x00" tex = _TEXT_CMD.sub(stash, tex) tex = _NON_LATIN_RUN.sub(lambda m: f"\\text{{{m.group(0)}}}", tex) tex = re.sub(r"\x00(\d+)\x00", lambda m: protected[int(m.group(1))], tex) if display: tex = re.sub(r"\s*\n\s*", r" \\\\ ", tex.strip()) return tex def repair_math(text: str) -> str: """Repair every math span in a markdown string. Prose outside ``$`` is untouched.""" def fix(match: re.Match) -> str: display = match.group(1) is not None inner = _repair_expression(match.group(1) or match.group(2), display) return f"$${inner}$$" if display else f"${inner}$" return _MATH_SPAN.sub(fix, text) def reconstruct(blocks: list[Block], repair: bool = True) -> str: """Reading-ordered markdown. Blocks with no text contribute nothing but are not removed -- they still appear in the JSON with text "" . ``repair=False`` emits the recognizer's math verbatim, including expressions no LaTeX engine can render. Only useful for comparing byte-for-byte against output produced before the repair existed. """ kept = sorted((b for b in blocks if b.type not in DROP_TYPES), key=lambda b: b.order) parts = [] for block in kept: text = (block.text or "").strip() if not text: continue # Bare LaTeX would render as literal source, so wrap it -- but only when the recognizer # supplied no delimiters at all. Checking just the first character is not enough: an # Equation block often comes back as a prose prefix followed by already-delimited math # ("বা, $\\frac{a}{b}$"), and wrapping that produces "$$...$$$", which nothing can parse. if block.type == "Equation" and "$" not in text and not text.startswith(_MATH_DELIMITERS): text = f"$${text}$$" parts.append(text) markdown = dehyphenate("\n\n".join(parts)) return repair_math(markdown) if repair else markdown