[NOTICKET] fix(knowledge-parsing): read word boundaries back from the source PDF

MinerU writes formulas one character at a time ("T o t a l H o u r s"), and
once those are rejoined the word boundaries are gone: "Total Hours" and
"TotalHours" are no longer distinguishable from the LaTeX alone. Everything
fuses into a single token.

The multiplication written as a letter "x" is not a separate bug. It is the
same one — an "x" between terms is just another spaced character, so
"MOHH x Qty x PA x UA x Pty" collapses to "MOHHxQtyxPAxUAxPty".

Two obvious fixes were tried and rejected:

- Rewriting a lone "x" to a multiplication sign guesses, and guesses wrong
on any document with a variable named x, silently. It also leaves
"TotalHours" untouched, which is the larger half of the damage.
- Emitting a warning fires on 7 of 11 equations. A warning that lights up on
64% of items is not actionable, and recovers nothing.

The source PDF still holds the boundaries: its text layer reads
"MOHH x Qty x PA x UA x Pty", properly spaced. So the answer is read from the
document rather than guessed. A token that is not already a word on that page
is split only if EVERY piece is a word on that page, and only if exactly one
shortest such split exists. Ambiguity is left alone; so is anything that
cannot be segmented. It fails closed.

NFKC normalisation is required — formulas use a mathematical font, so the page
literally contains U+1D440 and friends, and without it nothing matches.

Correct documents are untouched by construction: a token already present as a
word never enters the segmenter, so a document writing \times produces tokens
that are all in the vocabulary and skips this path entirely. Verified — the
second test document changed in no way at all.

Threshold measured, not picked: 6, 4 and 3 are identical on equations, and 3
additionally recovers "MOHHx" and "xPA_A" in table cells. 2 starts doing
damage, splitting "PA_ij" into "PA_i j".

Result on the 9-page BUMA standard: glossary terms readable as whole tokens
320 -> 397, across 9 of 31 chunks. Chunk.latex is untouched, so the formula
branch still receives the original form.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/knowledge_parsing/checks.py

@@ -12,6 +12,7 @@ turun diam-diam di antara ratusan dokumen.
 from __future__ import annotations
 
 import re
+import unicodedata
 from collections import Counter
 from pathlib import Path
 from typing import Any
@@ -64,6 +65,39 @@ def _angka_per_halaman(pdf: Path, min_digit: int = MIN_DIGIT) -> dict[int, set[s
     return hasil
 
 
+_KATA = re.compile(r"[A-Za-z][A-Za-z0-9]*")
+
+
+def kosakata_per_halaman(pdf: Path) -> dict[int, frozenset[str]] | None:
+    """Kata-kata pada lapisan teks PDF asli, per halaman.
+
+    Dipakai `render.pulihkan_batas_kata` sebagai wasit: MinerU menulis rumus per
+    karakter dan batas katanya hilang di situ, sementara PDF-nya masih menyimpan
+    batas itu utuh (`MOHH x Qty x PA`, bukan `MOHHxQtyxPAxUAxPty`).
+
+    NFKC dipakai karena rumus memakai font matematika — halaman ini menulis
+    `𝑀𝑂𝐻𝐻` (U+1D440…), bukan ASCII, dan tanpa normalisasi tidak ada satu kata
+    pun yang akan cocok.
+
+    Kembalikan None kalau PDF tidak terbaca; pemanggil lalu melewati pemulihan
+    diam-diam, sama seperti pemeriksaan angka di bawah.
+    """
+    try:
+        from pypdf import PdfReader
+        reader = PdfReader(str(pdf))
+    except Exception:
+        return None
+
+    hasil: dict[int, frozenset[str]] = {}
+    for i, hal in enumerate(reader.pages):
+        try:
+            teks = hal.extract_text() or ""
+        except Exception:
+            teks = ""
+        hasil[i] = frozenset(_KATA.findall(unicodedata.normalize("NFKC", teks)))
+    return hasil
+
+
 def periksa_items(
     items: list[dict[str, Any]],
     min_latex_len: int = 8,

src/knowledge_parsing/normalize.py

@@ -91,7 +91,7 @@ def _judul(item: dict[str, Any]) -> tuple[str | None, str | None, int | None]:
     return m.group(1), m.group(2), m.group(1).count(".") + 1
 
 
-def _teks_tabel(item: dict[str, Any]) -> str:
+def _teks_tabel(item: dict[str, Any], kosakata: frozenset[str] | None = None) -> str:
     """Caption + isi tabel sebagai teks terbaca.
 
     HTML mentahnya TIDAK ditaruh di sini — disimpan terpisah di
@@ -99,7 +99,7 @@ def _teks_tabel(item: dict[str, Any]) -> str:
     """
     bagian = list(item.get("table_caption") or [])
     if item.get("table_body"):
-        bagian.append(render_table(item["table_body"]))
+        bagian.append(render_table(item["table_body"], kosakata))
     bagian += list(item.get("table_footnote") or [])
     return "\n\n".join(b for b in bagian if b)
 
@@ -112,7 +112,21 @@ def _teks_chart(item: dict[str, Any]) -> str:
     return "\n\n".join(b for b in bagian if b)
 
 
-def normalisasi(items: list[dict[str, Any]], doc_id: str) -> list[Chunk]:
+def normalisasi(
+    items: list[dict[str, Any]],
+    doc_id: str,
+    kosakata_halaman: dict[int, frozenset[str]] | None = None,
+) -> list[Chunk]:
+    """`kosakata_halaman` opsional: kata-kata lapisan teks PDF asli per halaman.
+
+    Dipakai hanya untuk memulihkan batas kata yang hilang saat MinerU menulis
+    rumus per karakter (lihat `render.pulihkan_batas_kata`). Tanpa itu, hasilnya
+    persis seperti sebelumnya.
+    """
+
+    def kosakata_dari(hal: int) -> frozenset[str] | None:
+        return (kosakata_halaman or {}).get(hal)
+
     # Konteks bab per halaman, dari header berjalan
     bab_per_halaman: dict[int, str] = {}
     for x in items:
@@ -160,7 +174,7 @@ def normalisasi(items: list[dict[str, Any]], doc_id: str) -> list[Chunk]:
         if tipe == "table":
             tutup()
             c = buka("table", page)
-            c.text = _teks_tabel(item)
+            c.text = _teks_tabel(item, kosakata_dari(page))
             c.is_tabular = True
             c.table_html = item.get("table_body") or None
             c.source_items = [i]
@@ -193,7 +207,7 @@ def normalisasi(items: list[dict[str, Any]], doc_id: str) -> list[Chunk]:
                 berjalan.images.append(item["img_path"])
             if latex:
                 berjalan.latex.append(latex)          # mentah, untuk cabang formula
-                terbaca = render_latex(latex)
+                terbaca = render_latex(latex, kosakata_dari(page))
                 if terbaca:
                     potongan.append(terbaca)          # prosa, supaya NER menemukannya
             continue
@@ -262,6 +276,10 @@ def normalisasi(items: list[dict[str, Any]], doc_id: str) -> list[Chunk]:
     return chunks
 
 
-def normalisasi_dari_berkas(content_list: Path, doc_id: str) -> list[Chunk]:
+def normalisasi_dari_berkas(
+    content_list: Path,
+    doc_id: str,
+    kosakata_halaman: dict[int, frozenset[str]] | None = None,
+) -> list[Chunk]:
     items = json.loads(content_list.read_text(encoding="utf-8"))
-    return normalisasi(items, doc_id)
+    return normalisasi(items, doc_id, kosakata_halaman)

src/knowledge_parsing/render.py

@@ -71,6 +71,86 @@ def _rapatkan(teks: str) -> str:
     return _SPACED_RUN.sub(lambda m: m.group(0).replace(" ", ""), teks)
 
 
+# Panjang minimum token yang boleh dibelah. Diukur pada dokumen nyata: 6, 4 dan
+# 3 memberi hasil identik pada rumus, dan 3 juga memulihkan `MOHHx` / `xPA_A` di
+# sel tabel. Turun ke 2 mulai MERUSAK — `PA_ij` terbelah jadi `PA_i j`, karena
+# `i` dan `j` dua-duanya kata yang sah di halaman itu.
+MIN_TOKEN_SEGMEN = 3
+_MAKS_KATA = 24          # panjang wajar satu kata; membatasi kerja DP
+
+_TOKEN = re.compile(r"[A-Za-z][A-Za-z0-9]*")
+
+
+def _segmentasi(tok: str, kosakata: frozenset[str]) -> list[str] | None:
+    """Belah `tok` menjadi kata-kata yang SEMUANYA ada di `kosakata`.
+
+    Kembalikan None kalau tidak ada pembelahan yang sah, ATAU kalau ada lebih
+    dari satu pembelahan terpendek yang sama sahnya. Ambiguitas dibiarkan apa
+    adanya — modul ini tidak menebak, dan sebuah token yang tidak bisa
+    dipulihkan dengan pasti lebih baik dibiarkan rusak daripada diperbaiki
+    ke bentuk yang salah tanpa jejak.
+    """
+    n = len(tok)
+    # dp[i] = (jumlah kata minimum untuk tok[:i], berapa cara mencapainya, potongan)
+    dp: list[tuple[int, int, list[str]] | None] = [None] * (n + 1)
+    dp[0] = (0, 1, [])
+
+    for i in range(1, n + 1):
+        terbaik: int | None = None
+        cara = 0
+        potongan: list[str] = []
+        for j in range(max(0, i - _MAKS_KATA), i):
+            sebelum = dp[j]
+            if sebelum is None or tok[j:i] not in kosakata:
+                continue
+            panjang_j, cara_j, kata_j = sebelum
+            calon = panjang_j + 1
+            if terbaik is None or calon < terbaik:
+                terbaik, cara, potongan = calon, cara_j, kata_j + [tok[j:i]]
+            elif calon == terbaik:
+                cara += cara_j
+        if terbaik is not None:
+            dp[i] = (terbaik, cara, potongan)
+
+    hasil = dp[n]
+    if hasil is None:
+        return None
+    _, cara, potongan = hasil
+    if cara != 1 or len(potongan) < 2:
+        return None
+    return potongan
+
+
+def pulihkan_batas_kata(teks: str, kosakata: frozenset[str] | None) -> str:
+    """Kembalikan spasi antar kata yang hilang saat deret berspasi dirapatkan.
+
+    MinerU menulis rumus per karakter (`T o t a l   H o u r s`), dan begitu
+    dirapatkan batas katanya tidak bisa dipulihkan dari LaTeX-nya sendiri:
+    `Total Hours` dan `TotalHours` menjadi tidak terbedakan. Tanda kali yang
+    ditulis sebagai huruf `x` ikut melebur karena bentuknya sama saja dengan
+    karakter berspasi lain.
+
+    Lapisan teks PDF aslinya MASIH menyimpan batas itu, jadi jawabannya dibaca
+    dari dokumen alih-alih ditebak. Token yang sudah ada sebagai kata di sumber
+    tidak pernah disentuh — sehingga dokumen yang menulis rumusnya dengan benar
+    melewati jalur ini seluruhnya.
+    """
+    if not kosakata:
+        return teks
+
+    ganti: dict[str, str] = {}
+    for tok in set(_TOKEN.findall(teks)):
+        if len(tok) < MIN_TOKEN_SEGMEN or tok in kosakata:
+            continue
+        potongan = _segmentasi(tok, kosakata)
+        if potongan:
+            ganti[tok] = " ".join(potongan)
+
+    for tok, jadi in sorted(ganti.items(), key=lambda kv: -len(kv[0])):
+        teks = re.sub(rf"(?<![A-Za-z0-9]){re.escape(tok)}(?![A-Za-z0-9])", jadi, teks)
+    return teks
+
+
 def _argumen(s: str, i: int) -> tuple[str | None, int]:
     """Isi kurung SEIMBANG yang dimulai di `s[i] == '{'`.
 
@@ -122,8 +202,14 @@ def _render_frac(s: str) -> str:
     return s
 
 
-def render_latex(latex: str) -> str:
-    """LaTeX -> a readable line. Best effort: it feeds a term filter, not a parser."""
+def render_latex(latex: str, kosakata: frozenset[str] | None = None) -> str:
+    """LaTeX -> a readable line. Best effort: it feeds a term filter, not a parser.
+
+    `kosakata` is the source page's own word list, used only to put back word
+    boundaries this function cannot recover on its own — see
+    `pulihkan_batas_kata`. Omitting it reproduces the previous behaviour
+    exactly, so existing callers are unaffected.
+    """
     s = latex.replace("$$", " ").replace("$", " ")
     s = s.replace("~", _WORD_GAP)
 
@@ -169,7 +255,11 @@ def render_latex(latex: str) -> str:
     s = _rapatkan(s)
 
     s = s.replace(_WORD_GAP, " ")
-    return re.sub(r"\s+", " ", s).strip()
+    s = re.sub(r"\s+", " ", s).strip()
+
+    # Terakhir, setelah spasi normal kembali: pulihkan batas kata yang tidak
+    # bisa diketahui dari LaTeX-nya sendiri.
+    return pulihkan_batas_kata(s, kosakata)
 
 
 # --- HTML tables ---------------------------------------------------------
@@ -184,7 +274,7 @@ _TAG = re.compile(r"<[^>]+>")
 _INLINE_MATH = re.compile(r"\$([^$]{1,400})\$")
 
 
-def render_table(table_html: str) -> str:
+def render_table(table_html: str, kosakata: frozenset[str] | None = None) -> str:
     """HTML table -> plain rows, cells separated by ' | '.
 
     Keeps the reading order a person would use, so a term appearing only in a
@@ -194,7 +284,7 @@ def render_table(table_html: str) -> str:
     s = _CELL_END.sub(" | ", s)
     s = _TAG.sub(" ", s)
     s = html.unescape(s)
-    s = _INLINE_MATH.sub(lambda m: " " + render_latex(m.group(1)) + " ", s)
+    s = _INLINE_MATH.sub(lambda m: " " + render_latex(m.group(1), kosakata) + " ", s)
 
     baris = []
     for raw in s.split("\n"):

src/knowledge_parsing/run.py

@@ -20,7 +20,7 @@ import traceback
 from datetime import datetime
 from pathlib import Path
 
-from .checks import periksa_items
+from .checks import kosakata_per_halaman, periksa_items
 from .config import PipelineConfig
 from .contracts import ParsedDocument
 from .manifest import CatatanDokumen, Manifest
@@ -76,8 +76,18 @@ def jalankan(cfg: PipelineConfig) -> Manifest:
                 pdf_sumber=source, offset_halaman=cfg.start_page,
             )
 
+            # Kosakata halaman sumber, untuk memulihkan batas kata yang hilang
+            # saat MinerU menulis rumus per karakter. Dikunci ulang ke indeks
+            # halaman MinerU: page_idx 0 berarti halaman `start_page` di PDF.
+            mentah = kosakata_per_halaman(source)
+            kosakata = (
+                {p - cfg.start_page: kata for p, kata in mentah.items()
+                 if p >= cfg.start_page}
+                if mentah else None
+            )
+
             t0 = time.perf_counter()
-            chunks = normalisasi(items, doc_id)
+            chunks = normalisasi(items, doc_id, kosakata)
             catatan.detik_normalisasi = time.perf_counter() - t0
             catatan.jumlah_chunk = len(chunks)