src/knowledge_parsing/config.py

"""The single place pipeline settings live.

⭐ Official backend: `hybrid` at `effort="high"` — decided 2026-08-24 after all
four were measured on the same GPU, same documents (29 pages).

    hybrid/high     4.85 s/page   image descriptions YES   ⭐ in use
    hybrid/medium   1.88 s/page   descriptions FORCE-DISABLED by MinerU
    pipeline        2.67 s/page   never analyses images at all
    vlm             6.96 s/page   output IDENTICAL to hybrid/high, 100%

Two findings decided it:

  - `vlm` and `hybrid/high` produce the same LaTeX and the same descriptions,
    but `vlm` is slower. There is no reason to pick it.
  - `pipeline` is the only backend that copies the multiplication sign as a
    literal `x`, which fuses `MOHH x Qty x PA` into a single token and takes
    five terms with it. It is also slower than `hybrid/medium`, so it loses
    on both counts.

The `high` vs `medium` cost gap across the whole BUMA corpus (6,800 pages) is
$6.89 against $2.67 — too small to trade image descriptions for.

`hybrid` and `vlm` REQUIRE a GPU. To run on a laptop without one, ask for
`pipeline` explicitly:

    python -m src.knowledge_parsing.run --input data/knowledge_docs/ --backend pipeline

Determinism: SETTLED 2026-08-26. Two genuine re-parses of the same documents
produced byte-identical artifacts across all fields — including the four image
descriptions, which are the only model-written content and were the likeliest to
wobble. Frozen baselines do not drift; calibrating on this backend is safe.
Parse TIME varies (model loading), the output does not.
"""

from __future__ import annotations

import hashlib
import json
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any, Literal

Backend = Literal["pipeline", "vlm", "hybrid", "mistral", "paddleocr"]

# Storage root: the repo root (src/knowledge_parsing/ -> ../../).
# Everything written out (cache/, runs/) is DATA, not code — gitignored.
ROOT = Path(__file__).resolve().parent.parent.parent


@dataclass
class PipelineConfig:
    # --- changed most often ---
    # `mistral` is the primary backend (2026-09-02): Mistral OCR API, no GPU,
    # section recall 0.878 vs MinerU 0.829 on the BUMA fixture. `hybrid`/`vlm`/
    # `pipeline` are MinerU, kept as a fallback until a final call to drop them.
    backend: Backend = "mistral"

    # ⚠️ MinerU has NO Indonesian language code. Indonesian documents use "ch",
    # whose coverage is "Chinese, English, Japanese, Chinese Traditional, LATIN"
    # — and Latin script is what covers Indonesian. Passing "id" fails outright:
    # ValueError: Language id not supported.
    lang: str = "ch"

    # --- locations ---
    input_dir: Path = ROOT / "data" / "knowledge_docs"
    cache_dir: Path = ROOT / "data" / "knowledge_cache"   # parse output, keyed by document
    runs_dir: Path = ROOT / "data" / "knowledge_runs"     # normalised output + per-run manifest

    # --- MinerU options ---
    formula_enable: bool = True
    table_enable: bool = True
    # Applies to `hybrid` only, and is NOT merely fast-vs-thorough: MinerU
    # force-disables image analysis at `medium`. Dropping to "medium" throws
    # away image descriptions — the very reason hybrid was chosen.
    effort: str = "high"
    start_page: int = 0
    end_page: int | None = None

    # --- Mistral OCR options (backend="mistral") ---
    mistral_model: str = "mistral-ocr-latest"
    # include_blocks gives a typed block stream (title/text/equation/table/…)
    # ~1:1 with MinerU's content_list — required for section chunking + headings.
    mistral_include_blocks: bool = True
    # bbox_annotation: a Vision-LLM description per image (the generated_description
    # field). Costs an extra call per image; turn off to save spend.
    mistral_describe_images: bool = True
    mistral_table_format: str = "html"

    # --- PaddleOCR-VL options (backend="paddleocr", added 2026-09-24) ---
    # Baidu AI Studio's hosted job API. Selected by the v2 document process path
    # (settings.document_parser_backend); the default backend above is unchanged,
    # so the offline knowledge pipeline never reaches it unless asked. PDF and
    # images only — the API takes no DOCX.
    paddle_model: str = "PaddleOCR-VL-1.6"
    paddle_job_url: str = "https://paddleocr.aistudio-app.com/api/v2/ocr/jobs"
    # Upper bound on waiting for the hosted job. v2 ingest has its own 10-minute
    # budget around the whole document, which binds first.
    paddle_poll_timeout_s: float = 900.0

    # --- behaviour ---
    resume: bool = True                  # skip documents already finished
    # Keep asset METADATA but discard the bytes. The figure is still folded into
    # its prose chunk with its caption, description and content-addressed id —
    # only `storage_key` comes back null. For a run where the images are not
    # wanted on disk, or where storage is unavailable.
    no_assets: bool = False
    write_debug_pdf: bool = True         # _layout.pdf & _span.pdf: fastest quality check there is

    # --- quality-check thresholds (see checks.py) ---
    min_latex_len: int = 8               # latex shorter than this is suspected truncated

    extra: dict[str, Any] = field(default_factory=dict)

    def canonical_lang(self) -> str:
        """The language code MinerU will actually use.

        `validate_public_ocr_lang` both validates and canonicalises: the aliases
        'latin' / 'en' / 'japan' all resolve to 'ch', and unknown codes are
        rejected. This is the same function that raises "Language id not
        supported".

        The cache fingerprint uses it too: without canonicalising, `--lang latin`
        and `--lang ch` produce different cache keys for an identical parse, and
        the same document gets parsed twice.

        Note that only ImportError is caught (MinerU not installed). ValueError
        is deliberately LET THROUGH — that one means a genuinely wrong language
        code, and swallowing it would delay the failure until after the models
        have loaded.
        """
        try:
            from mineru.utils.ocr_language import validate_public_ocr_lang
        except ImportError:
            return self.lang
        return validate_public_ocr_lang(self.lang)

    def validate(self) -> None:
        """Fail fast on a bad configuration.

        Without this, a wrong language code only surfaces after ~30 seconds of
        model loading — and on a large batch, after the nth document.
        """
        try:
            self.canonical_lang()
        except ValueError as e:
            raise SystemExit(
                f"{e}\n"
                "  -> Indonesian documents use 'ch' (it covers Latin script). "
                "MinerU has no 'id' code."
            ) from e

        # The GPU precondition deliberately does NOT live here. A GPU is needed
        # to PARSE, not to run: with a warm cache, `hybrid` output can be
        # rebuilt on a laptop from MinerU's saved output, and that is the normal
        # way to pick up a normalisation fix without paying for a re-parse.
        # Checking here would refuse that. See `parse.require_gpu`, called
        # immediately before MinerU is invoked.

    def fingerprint(self) -> str:
        """Fingerprint of the settings that AFFECT the parse result.

        Used as part of the cache key. Options that do not change the output
        (resume, runs_dir, check thresholds) are deliberately excluded, so
        changing them does not invalidate a cache entry that is still valid.
        """
        material = {
            "backend": self.backend,
            "lang": self.canonical_lang(),
            "formula_enable": self.formula_enable,
            "table_enable": self.table_enable,
            "effort": self.effort if self.backend == "hybrid" else None,
            "start_page": self.start_page,
            "end_page": self.end_page,
        }
        # Mistral settings are added ONLY on the mistral backend, so the existing
        # MinerU cache keys stay byte-identical and no cached parse is invalidated.
        if self.backend == "mistral":
            material.update({
                "mistral_model": self.mistral_model,
                "mistral_include_blocks": self.mistral_include_blocks,
                "mistral_describe_images": self.mistral_describe_images,
                "mistral_table_format": self.mistral_table_format,
            })
        # Same rule for PaddleOCR: only on its own backend, so no other key moves.
        if self.backend == "paddleocr":
            material["paddle_model"] = self.paddle_model
        payload = json.dumps(material, sort_keys=True).encode()
        return hashlib.sha256(payload).hexdigest()[:12]

    def to_dict(self) -> dict[str, Any]:
        d = asdict(self)
        for k, v in d.items():
            if isinstance(v, Path):
                d[k] = str(v)
        d["fingerprint"] = self.fingerprint()
        return d