controlai_rag/textfix.py

"""Repair of PDF text extraction artifacts.

Some of the textbooks in the library are typeset with Type 1 symbol fonts whose
embedded ToUnicode map is wrong. Every extractor -- pypdf and PyMuPDF alike --
therefore returns the same mojibake for their inline mathematics: the Laplace
transform definition comes out as `L½ f ðtÞ/C138 ¼FðsÞ` instead of
`L[f(t)] = F(s)`.

This mattered more than it looks. 71.5% of the chunks from Nise's *Control
Systems Engineering* -- the most heavily used classical-control reference in the
corpus -- were damaged this way, which is why retrieval for a query as ordinary
as "Routh-Hurwitz table construction" returned book index pages instead of the
relevant section.

The substitution is deterministic and was read off from context, not guessed:
`e/C0ð sþaÞt` is `e^-(s+a)t`, `5:6 /C2 10/C0 6s` is `5.6 x 10^-6 s`,
`26:57/C14` is `26.57°`, `þ/C1/C1/C1þ` is `+ ... +`. Codes that turn out to be
pieces of a multi-line bracket rather than a character of their own carry no
meaning on their own and are dropped.
"""

from __future__ import annotations

import re

# Single characters the broken font map emits in place of ASCII math.
_CHAR_MAP = str.maketrans({
    "ð": "(",
    "Þ": ")",
    "¼": "=",
    "þ": "+",
    "½": "[",
})

# `/Cnn` glyph references, by the number that follows.
_GLYPH_MAP = {
    "0": "-",     # minus:  1899 /C0 3761z  ->  1899 - 3761z
    "1": "·",  # middle dot:  þ/C1/C1/C1þ  ->  + ... +
    "2": "×",  # times:  5:6 /C2 10  ->  5.6 x 10
    "12": "|",    # evaluation bar
    "14": "°",  # degree:  26:57/C14  ->  26.57 deg
    "15": "•",  # bullet in feature lists
    "138": "]",   # closing bracket, pairing with the "[" above
}
# Pieces of tall multi-line brackets. They are layout, not content, and the
# extractor emits them after the expression they were meant to enclose, so
# rendering them as brackets would be actively misleading.
_LAYOUT_GLYPHS = {"3", "6", "16", "17", "18", "19", "20", "21"}

_GLYPH_RE = re.compile(r"/C(\d+)")
# Detects whether a document needs any of this at all.
_DAMAGE_RE = re.compile(r"/C\d+|[ðÞ¼þ]")


def looks_damaged(text: str, threshold: int = 5) -> bool:
    """True if `text` carries enough artifacts to be worth repairing."""
    return len(_DAMAGE_RE.findall(text)) >= threshold


def repair(text: str) -> str:
    """Undo the broken font mapping. Safe to call on undamaged text."""
    if not _DAMAGE_RE.search(text):
        return text

    def _glyph(match: re.Match) -> str:
        code = match.group(1)
        if code in _GLYPH_MAP:
            return _GLYPH_MAP[code]
        if code in _LAYOUT_GLYPHS:
            return " "
        return " "  # an unrecognised glyph is noise; a space beats a token of junk

    text = _GLYPH_RE.sub(_glyph, text)
    text = text.translate(_CHAR_MAP)
    # The extractor also stamps a watermark onto every page of one scan.
    text = text.replace("Apago PDF Enhancer", " ")
    return re.sub(r"[ \t]{2,}", " ", text).strip()