"""The single place pipeline settings live. ⭐ Official backend: `hybrid` at `effort="high"` — decided 2026-08-24 after all four were measured on the same GPU, same documents (29 pages). hybrid/high 4.85 s/page image descriptions YES ⭐ in use hybrid/medium 1.88 s/page descriptions FORCE-DISABLED by MinerU pipeline 2.67 s/page never analyses images at all vlm 6.96 s/page output IDENTICAL to hybrid/high, 100% Two findings decided it: - `vlm` and `hybrid/high` produce the same LaTeX and the same descriptions, but `vlm` is slower. There is no reason to pick it. - `pipeline` is the only backend that copies the multiplication sign as a literal `x`, which fuses `MOHH x Qty x PA` into a single token and takes five terms with it. It is also slower than `hybrid/medium`, so it loses on both counts. The `high` vs `medium` cost gap across the whole BUMA corpus (6,800 pages) is $6.89 against $2.67 — too small to trade image descriptions for. `hybrid` and `vlm` REQUIRE a GPU. To run on a laptop without one, ask for `pipeline` explicitly: python -m src.knowledge_parsing.run --input data/knowledge_docs/ --backend pipeline Determinism: SETTLED 2026-08-26. Two genuine re-parses of the same documents produced byte-identical artifacts across all fields — including the four image descriptions, which are the only model-written content and were the likeliest to wobble. Frozen baselines do not drift; calibrating on this backend is safe. Parse TIME varies (model loading), the output does not. """ from __future__ import annotations import hashlib import json from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any, Literal Backend = Literal["pipeline", "vlm", "hybrid", "mistral", "paddleocr"] # Storage root: the repo root (src/knowledge_parsing/ -> ../../). # Everything written out (cache/, runs/) is DATA, not code — gitignored. ROOT = Path(__file__).resolve().parent.parent.parent @dataclass class PipelineConfig: # --- changed most often --- # `mistral` is the primary backend (2026-09-02): Mistral OCR API, no GPU, # section recall 0.878 vs MinerU 0.829 on the BUMA fixture. `hybrid`/`vlm`/ # `pipeline` are MinerU, kept as a fallback until a final call to drop them. backend: Backend = "mistral" # ⚠️ MinerU has NO Indonesian language code. Indonesian documents use "ch", # whose coverage is "Chinese, English, Japanese, Chinese Traditional, LATIN" # — and Latin script is what covers Indonesian. Passing "id" fails outright: # ValueError: Language id not supported. lang: str = "ch" # --- locations --- input_dir: Path = ROOT / "data" / "knowledge_docs" cache_dir: Path = ROOT / "data" / "knowledge_cache" # parse output, keyed by document runs_dir: Path = ROOT / "data" / "knowledge_runs" # normalised output + per-run manifest # --- MinerU options --- formula_enable: bool = True table_enable: bool = True # Applies to `hybrid` only, and is NOT merely fast-vs-thorough: MinerU # force-disables image analysis at `medium`. Dropping to "medium" throws # away image descriptions — the very reason hybrid was chosen. effort: str = "high" start_page: int = 0 end_page: int | None = None # --- Mistral OCR options (backend="mistral") --- mistral_model: str = "mistral-ocr-latest" # include_blocks gives a typed block stream (title/text/equation/table/…) # ~1:1 with MinerU's content_list — required for section chunking + headings. mistral_include_blocks: bool = True # bbox_annotation: a Vision-LLM description per image (the generated_description # field). Costs an extra call per image; turn off to save spend. mistral_describe_images: bool = True mistral_table_format: str = "html" # --- PaddleOCR-VL options (backend="paddleocr", added 2026-09-24) --- # Baidu AI Studio's hosted job API. Selected by the v2 document process path # (settings.document_parser_backend); the default backend above is unchanged, # so the offline knowledge pipeline never reaches it unless asked. PDF and # images only — the API takes no DOCX. paddle_model: str = "PaddleOCR-VL-1.6" paddle_job_url: str = "https://paddleocr.aistudio-app.com/api/v2/ocr/jobs" # Upper bound on waiting for the hosted job. v2 ingest has its own 10-minute # budget around the whole document, which binds first. paddle_poll_timeout_s: float = 900.0 # --- behaviour --- resume: bool = True # skip documents already finished # Keep asset METADATA but discard the bytes. The figure is still folded into # its prose chunk with its caption, description and content-addressed id — # only `storage_key` comes back null. For a run where the images are not # wanted on disk, or where storage is unavailable. no_assets: bool = False write_debug_pdf: bool = True # _layout.pdf & _span.pdf: fastest quality check there is # --- quality-check thresholds (see checks.py) --- min_latex_len: int = 8 # latex shorter than this is suspected truncated extra: dict[str, Any] = field(default_factory=dict) def canonical_lang(self) -> str: """The language code MinerU will actually use. `validate_public_ocr_lang` both validates and canonicalises: the aliases 'latin' / 'en' / 'japan' all resolve to 'ch', and unknown codes are rejected. This is the same function that raises "Language id not supported". The cache fingerprint uses it too: without canonicalising, `--lang latin` and `--lang ch` produce different cache keys for an identical parse, and the same document gets parsed twice. Note that only ImportError is caught (MinerU not installed). ValueError is deliberately LET THROUGH — that one means a genuinely wrong language code, and swallowing it would delay the failure until after the models have loaded. """ try: from mineru.utils.ocr_language import validate_public_ocr_lang except ImportError: return self.lang return validate_public_ocr_lang(self.lang) def validate(self) -> None: """Fail fast on a bad configuration. Without this, a wrong language code only surfaces after ~30 seconds of model loading — and on a large batch, after the nth document. """ try: self.canonical_lang() except ValueError as e: raise SystemExit( f"{e}\n" " -> Indonesian documents use 'ch' (it covers Latin script). " "MinerU has no 'id' code." ) from e # The GPU precondition deliberately does NOT live here. A GPU is needed # to PARSE, not to run: with a warm cache, `hybrid` output can be # rebuilt on a laptop from MinerU's saved output, and that is the normal # way to pick up a normalisation fix without paying for a re-parse. # Checking here would refuse that. See `parse.require_gpu`, called # immediately before MinerU is invoked. def fingerprint(self) -> str: """Fingerprint of the settings that AFFECT the parse result. Used as part of the cache key. Options that do not change the output (resume, runs_dir, check thresholds) are deliberately excluded, so changing them does not invalidate a cache entry that is still valid. """ material = { "backend": self.backend, "lang": self.canonical_lang(), "formula_enable": self.formula_enable, "table_enable": self.table_enable, "effort": self.effort if self.backend == "hybrid" else None, "start_page": self.start_page, "end_page": self.end_page, } # Mistral settings are added ONLY on the mistral backend, so the existing # MinerU cache keys stay byte-identical and no cached parse is invalidated. if self.backend == "mistral": material.update({ "mistral_model": self.mistral_model, "mistral_include_blocks": self.mistral_include_blocks, "mistral_describe_images": self.mistral_describe_images, "mistral_table_format": self.mistral_table_format, }) # Same rule for PaddleOCR: only on its own backend, so no other key moves. if self.backend == "paddleocr": material["paddle_model"] = self.paddle_model payload = json.dumps(material, sort_keys=True).encode() return hashlib.sha256(payload).hexdigest()[:12] def to_dict(self) -> dict[str, Any]: d = asdict(self) for k, v in d.items(): if isinstance(v, Path): d[k] = str(v) d["fingerprint"] = self.fingerprint() return d