Spaces:
Running on Zero
Running on Zero
File size: 3,011 Bytes
9009a09 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | """Repair of PDF text extraction artifacts.
Some of the textbooks in the library are typeset with Type 1 symbol fonts whose
embedded ToUnicode map is wrong. Every extractor -- pypdf and PyMuPDF alike --
therefore returns the same mojibake for their inline mathematics: the Laplace
transform definition comes out as `L½ f ðtÞ/C138 ¼FðsÞ` instead of
`L[f(t)] = F(s)`.
This mattered more than it looks. 71.5% of the chunks from Nise's *Control
Systems Engineering* -- the most heavily used classical-control reference in the
corpus -- were damaged this way, which is why retrieval for a query as ordinary
as "Routh-Hurwitz table construction" returned book index pages instead of the
relevant section.
The substitution is deterministic and was read off from context, not guessed:
`e/C0ð sþaÞt` is `e^-(s+a)t`, `5:6 /C2 10/C0 6s` is `5.6 x 10^-6 s`,
`26:57/C14` is `26.57°`, `þ/C1/C1/C1þ` is `+ ... +`. Codes that turn out to be
pieces of a multi-line bracket rather than a character of their own carry no
meaning on their own and are dropped.
"""
from __future__ import annotations
import re
# Single characters the broken font map emits in place of ASCII math.
_CHAR_MAP = str.maketrans({
"ð": "(",
"Þ": ")",
"¼": "=",
"þ": "+",
"½": "[",
})
# `/Cnn` glyph references, by the number that follows.
_GLYPH_MAP = {
"0": "-", # minus: 1899 /C0 3761z -> 1899 - 3761z
"1": "·", # middle dot: þ/C1/C1/C1þ -> + ... +
"2": "×", # times: 5:6 /C2 10 -> 5.6 x 10
"12": "|", # evaluation bar
"14": "°", # degree: 26:57/C14 -> 26.57 deg
"15": "•", # bullet in feature lists
"138": "]", # closing bracket, pairing with the "[" above
}
# Pieces of tall multi-line brackets. They are layout, not content, and the
# extractor emits them after the expression they were meant to enclose, so
# rendering them as brackets would be actively misleading.
_LAYOUT_GLYPHS = {"3", "6", "16", "17", "18", "19", "20", "21"}
_GLYPH_RE = re.compile(r"/C(\d+)")
# Detects whether a document needs any of this at all.
_DAMAGE_RE = re.compile(r"/C\d+|[ðÞ¼þ]")
def looks_damaged(text: str, threshold: int = 5) -> bool:
"""True if `text` carries enough artifacts to be worth repairing."""
return len(_DAMAGE_RE.findall(text)) >= threshold
def repair(text: str) -> str:
"""Undo the broken font mapping. Safe to call on undamaged text."""
if not _DAMAGE_RE.search(text):
return text
def _glyph(match: re.Match) -> str:
code = match.group(1)
if code in _GLYPH_MAP:
return _GLYPH_MAP[code]
if code in _LAYOUT_GLYPHS:
return " "
return " " # an unrecognised glyph is noise; a space beats a token of junk
text = _GLYPH_RE.sub(_glyph, text)
text = text.translate(_CHAR_MAP)
# The extractor also stamps a watermark onto every page of one scan.
text = text.replace("Apago PDF Enhancer", " ")
return re.sub(r"[ \t]{2,}", " ", text).strip()
|