File size: 4,921 Bytes
2cb1336 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 | """Render MinerU's markup back into readable prose for `Chunk.text`.
Why this exists — measured, not assumed. Running the same extraction pipeline,
same document, same gold set, changing only the parse:
PyMuPDF (plain text) recall 0.8537 35/41
MinerU, raw markup in text recall 0.7561 31/41
MinerU, formula + table rendered recall 0.8293 34/41
The term filter is an NER model reading prose. MinerU writes formulas with every
character spaced out — `{ \\mathrm { P u r c h a s i n g ~ c o s t s } }` — and
tables as HTML. Neither yields a single mention, even for terms the same model
finds easily in plain text.
The markup is not discarded: `Chunk.latex` and `Chunk.table_html` keep it
verbatim, because the formula branch needs exactly that form.
"""
from __future__ import annotations
import html
import re
# --- LaTeX ---------------------------------------------------------------
_DROP_COMMANDS = re.compile(
r"\\(?:qquad|quad|left|right|displaystyle|limits|nolimits)\b|\\[!,;:]"
)
_WRAPPERS = re.compile(r"\\(?:mathrm|mathbf|mathit|mathsf|text|textrm|operatorname)\s*")
# Operators map to non-letter glyphs on purpose. A letter here (e.g. "x" for
# \times) would itself look like a spelled-out single character and get merged
# into the token beside it — "C_p x T" becoming "C_p xT".
_SYMBOLS = {
r"\times": "×", r"\cdot": "·", r"\div": "/", r"\pm": "±",
r"\leq": "≤", r"\geq": "≥", r"\neq": "≠", r"\approx": "≈",
r"\%": "%", r"\$": "$", r"\&": "&",
}
# "P u r c h a s i n g" -> "Purchasing", "1 0 0" -> "100". Runs of >= 2 single
# characters; letters and digits are joined separately so "C 5" is left alone.
# Subscripts are folded first (below), so a real variable like `C_p` is already
# one token and never gets swallowed into a neighbouring word.
_SPACED_RUN = re.compile(
r"(?<!\S)(?:[A-Za-z]\s+){1,}[A-Za-z](?!\S)"
r"|(?<!\S)(?:\d\s+){1,}\d(?!\S)"
)
# `~` is a non-breaking SPACE in LaTeX, i.e. a real word boundary. It has to
# survive the whitespace pass, otherwise "c o s t s" after it merges into the
# preceding word and "Purchasing costs" becomes "Purchasingcosts".
_WORD_GAP = "\x00"
def render_latex(latex: str) -> str:
"""LaTeX -> a readable line. Best effort: it feeds a term filter, not a parser."""
s = latex.replace("$$", " ").replace("$", " ")
s = s.replace("~", _WORD_GAP)
# array/matrix scaffolding carries no meaning for a term filter
s = re.sub(r"\\(?:begin|end)\s*\{[^{}]*\}", " ", s)
s = re.sub(r"\{\s*(?:[rlc|]\s*){1,8}\}", " ", s) # column spec, e.g. { r l }
s = s.replace("\\\\", " ; ").replace("&", " ")
# fold sub/superscripts first, so `C _ { p }` becomes one token `C_p`
s = re.sub(r"\s*_\s*\{\s*([^{}]*?)\s*\}", r"_\1", s)
s = re.sub(r"\s*\^\s*\{\s*([^{}]*?)\s*\}", r"^\1", s)
s = re.sub(r"\s*_\s*([A-Za-z0-9])", r"_\1", s)
s = re.sub(r"\s*\^\s*([A-Za-z0-9])", r"^\1", s)
# \frac{a}{b} -> (a) / (b); repeat for nesting
for _ in range(4):
baru = re.sub(r"\\d?frac\s*\{([^{}]*)\}\s*\{([^{}]*)\}", r"(\1) / (\2)", s)
if baru == s:
break
s = baru
s = re.sub(r"\\sqrt\s*\{([^{}]*)\}", r"sqrt(\1)", s)
for k, v in _SYMBOLS.items():
s = s.replace(k, f" {v} ")
s = _WRAPPERS.sub(" ", s)
s = _DROP_COMMANDS.sub(" ", s)
s = re.sub(r"\\[A-Za-z]+", " ", s) # any command left over
s = s.replace("{", " ").replace("}", " ").replace("\\", " ")
s = re.sub(r"[ \t\r\n]+", " ", s).strip()
# now that spacing is uniform, rejoin the spelled-out words and numbers
s = _SPACED_RUN.sub(lambda m: m.group(0).replace(" ", ""), s)
s = s.replace(_WORD_GAP, " ")
return re.sub(r"\s+", " ", s).strip()
# --- HTML tables ---------------------------------------------------------
_CELL_END = re.compile(r"</\s*(?:td|th)\s*>", re.I)
_ROW_END = re.compile(r"</\s*tr\s*>", re.I)
_TAG = re.compile(r"<[^>]+>")
# Table cells routinely carry inline LaTeX ("$\frac{kVA}{...}$"). Stripping the
# HTML alone leaves that markup sitting in the prose, which is the same problem
# the formula rendering exists to solve — found in a real table on the first run.
_INLINE_MATH = re.compile(r"\$([^$]{1,400})\$")
def render_table(table_html: str) -> str:
"""HTML table -> plain rows, cells separated by ' | '.
Keeps the reading order a person would use, so a term appearing only in a
table header is still a mention.
"""
s = _ROW_END.sub("\n", table_html)
s = _CELL_END.sub(" | ", s)
s = _TAG.sub(" ", s)
s = html.unescape(s)
s = _INLINE_MATH.sub(lambda m: " " + render_latex(m.group(1)) + " ", s)
baris = []
for raw in s.split("\n"):
bersih = re.sub(r"[ \t]+", " ", raw).strip().strip("|").strip()
if bersih:
baris.append(re.sub(r"\s*\|\s*", " | ", bersih))
return "\n".join(baris)
|