src/knowledge_parsing/run.py

"""Perangkai tahap 1-3: satu folder dokumen masuk, chunk + manifest keluar.

    py -m src.knowledge_parsing.run --input dokumen/
    py -m src.knowledge_parsing.run --input dokumen/ --backend vlm     # setelah GPU ada

Sifat yang disengaja:
  - satu dokumen gagal TIDAK menghentikan sisanya (dicatat di failures.jsonl)
  - bisa dilanjut dari tengah: dokumen yang sudah selesai dilewati (--no-resume
    untuk memaksa ulang)
  - output MinerU asli tidak disentuh, hanya ditunjuk dari runs/
"""

from __future__ import annotations

import argparse
import json
import time
import traceback
from datetime import datetime
from pathlib import Path

from .checks import periksa_items
from .config import PipelineConfig
from .contracts import ParsedDocument
from .manifest import CatatanDokumen, Manifest
from .normalize import normalisasi
from .parse import hash_berkas, parse_dokumen, versi_mineru

EKSTENSI = {".pdf", ".docx", ".pptx", ".doc", ".ppt"}


def kumpulkan_dokumen(folder: Path) -> list[Path]:
    return sorted(p for p in folder.rglob("*") if p.suffix.lower() in EKSTENSI)


def jalankan(cfg: PipelineConfig) -> Manifest:
    cfg.periksa()
    dokumen = kumpulkan_dokumen(cfg.input_dir)
    if not dokumen:
        raise SystemExit(f"Tidak ada dokumen di {cfg.input_dir}")

    manifest = Manifest.baru(cfg.to_dict())
    manifest.mineru_version = versi_mineru()

    run_dir = cfg.runs_dir / manifest.run_id
    run_dir.mkdir(parents=True, exist_ok=True)
    failures = run_dir / "failures.jsonl"

    print(f"Run {manifest.run_id} | backend={cfg.backend} | {len(dokumen)} dokumen")
    print(f"Hasil ke: {run_dir}\n")

    for n, source in enumerate(dokumen, 1):
        doc_id = source.stem
        tujuan = run_dir / doc_id
        chunk_file = tujuan / "chunks.json"

        if cfg.resume and chunk_file.exists():
            print(f"[{n}/{len(dokumen)}] {doc_id} — dilewati (sudah ada)")
            continue

        print(f"[{n}/{len(dokumen)}] {doc_id} … ", end="", flush=True)
        catatan = CatatanDokumen(doc_id=doc_id, source=str(source), status="ok")
        try:
            hasil = parse_dokumen(source, cfg)
            catatan.pages = hasil.pages
            catatan.detik_parse = hasil.seconds
            catatan.dari_cache = hasil.from_cache
            catatan.backend_tercatat = hasil.backend_tercatat

            items = json.loads(hasil.content_list.read_text(encoding="utf-8"))
            # PDF asli ikut dikirim: tanpa pembanding, angka yang berubah
            # diam-diam (bug EOQ) tidak akan pernah terdeteksi.
            catatan.pemeriksaan = periksa_items(
                items, cfg.min_latex_len,
                pdf_sumber=source, offset_halaman=cfg.start_page,
            )

            t0 = time.perf_counter()
            chunks = normalisasi(items, doc_id)
            catatan.detik_normalisasi = time.perf_counter() - t0
            catatan.jumlah_chunk = len(chunks)

            artifact = ParsedDocument(
                doc_id=doc_id,
                chunks=chunks,
                source_path=str(source),
                content_hash=hash_berkas(source, potong=64),
                n_pages=hasil.pages,
                parser_version=hasil.mineru_version or versi_mineru(),
                # backend dari _middle.json (yang benar-benar jalan), bukan dari config
                parser_backend=hasil.backend_tercatat or cfg.backend,
                parser_config=cfg.fingerprint(),
                created_at=datetime.now().isoformat(timespec="seconds"),
                raw_output_dir=str(hasil.cache_dir),
            )
            tujuan.mkdir(parents=True, exist_ok=True)
            chunk_file.write_text(
                artifact.model_dump_json(indent=2),
                encoding="utf-8",
            )
            # Penunjuk ke output MinerU asli — tidak disalin supaya tidak dobel.
            (tujuan / "sumber-mineru.txt").write_text(
                str(hasil.cache_dir), encoding="utf-8"
            )

            tanda = " (cache)" if hasil.from_cache else f" {hasil.seconds:.1f}s"
            peringatan = catatan.pemeriksaan.get("jumlah_peringatan", 0)
            tanda += f" | {len(chunks)} chunk"
            if peringatan:
                tanda += f" | ⚠ {peringatan} peringatan"
            print("ok" + tanda)

        except Exception as e:  # satu gagal tidak menghentikan sisanya
            catatan.status = "gagal"
            catatan.galat = f"{type(e).__name__}: {e}"
            print(f"GAGAL — {catatan.galat}")
            with failures.open("a", encoding="utf-8") as f:
                f.write(json.dumps({
                    "doc_id": doc_id, "source": str(source),
                    "galat": catatan.galat, "trace": traceback.format_exc(),
                }, ensure_ascii=False) + "\n")

        manifest.dokumen.append(catatan)
        manifest.simpan(run_dir / "manifest.json")

    r = manifest.ringkasan()
    print("\n--- ringkasan ---")
    print(f"  ok / gagal      : {r['dokumen_ok']} / {r['dokumen_gagal']}")
    print(f"  total halaman   : {r['total_halaman']}")
    if r["detik_per_halaman"]:
        print(f"  detik/halaman   : {r['detik_per_halaman']}  "
              f"(dari {r['halaman_terukur']} hal. yang benar-benar diparse)")
    if r["total_peringatan"]:
        print(f"  ⚠ peringatan mutu: {r['total_peringatan']} — lihat manifest.json")
    print(f"  manifest        : {run_dir / 'manifest.json'}")
    return manifest


def main() -> None:
    p = argparse.ArgumentParser(description="Pipeline parsing dokumen (MinerU)")
    p.add_argument("--input", type=Path, help="folder berisi dokumen")
    p.add_argument("--backend", choices=["pipeline", "vlm", "hybrid"],
                   help="pipeline=CPU, vlm=butuh GPU (target resmi)")
    p.add_argument("--lang", default=None)
    p.add_argument("--no-resume", action="store_true", help="proses ulang semua")
    p.add_argument("--start-page", type=int, default=None)
    p.add_argument("--end-page", type=int, default=None)
    a = p.parse_args()

    cfg = PipelineConfig()
    if a.input:
        cfg.input_dir = a.input
    if a.backend:
        cfg.backend = a.backend
    if a.lang:
        cfg.lang = a.lang
    if a.no_resume:
        cfg.resume = False
    if a.start_page is not None:
        cfg.start_page = a.start_page
    if a.end_page is not None:
        cfg.end_page = a.end_page

    jalankan(cfg)


if __name__ == "__main__":
    main()