File size: 6,446 Bytes
5840d20 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | """Perangkai tahap 1-3: satu folder dokumen masuk, chunk + manifest keluar.
py -m src.knowledge_parsing.run --input dokumen/
py -m src.knowledge_parsing.run --input dokumen/ --backend vlm # setelah GPU ada
Sifat yang disengaja:
- satu dokumen gagal TIDAK menghentikan sisanya (dicatat di failures.jsonl)
- bisa dilanjut dari tengah: dokumen yang sudah selesai dilewati (--no-resume
untuk memaksa ulang)
- output MinerU asli tidak disentuh, hanya ditunjuk dari runs/
"""
from __future__ import annotations
import argparse
import json
import time
import traceback
from datetime import datetime
from pathlib import Path
from .checks import periksa_items
from .config import PipelineConfig
from .contracts import ParsedDocument
from .manifest import CatatanDokumen, Manifest
from .normalize import normalisasi
from .parse import hash_berkas, parse_dokumen, versi_mineru
EKSTENSI = {".pdf", ".docx", ".pptx", ".doc", ".ppt"}
def kumpulkan_dokumen(folder: Path) -> list[Path]:
return sorted(p for p in folder.rglob("*") if p.suffix.lower() in EKSTENSI)
def jalankan(cfg: PipelineConfig) -> Manifest:
cfg.periksa()
dokumen = kumpulkan_dokumen(cfg.input_dir)
if not dokumen:
raise SystemExit(f"Tidak ada dokumen di {cfg.input_dir}")
manifest = Manifest.baru(cfg.to_dict())
manifest.mineru_version = versi_mineru()
run_dir = cfg.runs_dir / manifest.run_id
run_dir.mkdir(parents=True, exist_ok=True)
failures = run_dir / "failures.jsonl"
print(f"Run {manifest.run_id} | backend={cfg.backend} | {len(dokumen)} dokumen")
print(f"Hasil ke: {run_dir}\n")
for n, source in enumerate(dokumen, 1):
doc_id = source.stem
tujuan = run_dir / doc_id
chunk_file = tujuan / "chunks.json"
if cfg.resume and chunk_file.exists():
print(f"[{n}/{len(dokumen)}] {doc_id} — dilewati (sudah ada)")
continue
print(f"[{n}/{len(dokumen)}] {doc_id} … ", end="", flush=True)
catatan = CatatanDokumen(doc_id=doc_id, source=str(source), status="ok")
try:
hasil = parse_dokumen(source, cfg)
catatan.pages = hasil.pages
catatan.detik_parse = hasil.seconds
catatan.dari_cache = hasil.from_cache
catatan.backend_tercatat = hasil.backend_tercatat
items = json.loads(hasil.content_list.read_text(encoding="utf-8"))
# PDF asli ikut dikirim: tanpa pembanding, angka yang berubah
# diam-diam (bug EOQ) tidak akan pernah terdeteksi.
catatan.pemeriksaan = periksa_items(
items, cfg.min_latex_len,
pdf_sumber=source, offset_halaman=cfg.start_page,
)
t0 = time.perf_counter()
chunks = normalisasi(items, doc_id)
catatan.detik_normalisasi = time.perf_counter() - t0
catatan.jumlah_chunk = len(chunks)
artifact = ParsedDocument(
doc_id=doc_id,
chunks=chunks,
source_path=str(source),
content_hash=hash_berkas(source, potong=64),
n_pages=hasil.pages,
parser_version=hasil.mineru_version or versi_mineru(),
# backend dari _middle.json (yang benar-benar jalan), bukan dari config
parser_backend=hasil.backend_tercatat or cfg.backend,
parser_config=cfg.fingerprint(),
created_at=datetime.now().isoformat(timespec="seconds"),
raw_output_dir=str(hasil.cache_dir),
)
tujuan.mkdir(parents=True, exist_ok=True)
chunk_file.write_text(
artifact.model_dump_json(indent=2),
encoding="utf-8",
)
# Penunjuk ke output MinerU asli — tidak disalin supaya tidak dobel.
(tujuan / "sumber-mineru.txt").write_text(
str(hasil.cache_dir), encoding="utf-8"
)
tanda = " (cache)" if hasil.from_cache else f" {hasil.seconds:.1f}s"
peringatan = catatan.pemeriksaan.get("jumlah_peringatan", 0)
tanda += f" | {len(chunks)} chunk"
if peringatan:
tanda += f" | ⚠ {peringatan} peringatan"
print("ok" + tanda)
except Exception as e: # satu gagal tidak menghentikan sisanya
catatan.status = "gagal"
catatan.galat = f"{type(e).__name__}: {e}"
print(f"GAGAL — {catatan.galat}")
with failures.open("a", encoding="utf-8") as f:
f.write(json.dumps({
"doc_id": doc_id, "source": str(source),
"galat": catatan.galat, "trace": traceback.format_exc(),
}, ensure_ascii=False) + "\n")
manifest.dokumen.append(catatan)
manifest.simpan(run_dir / "manifest.json")
r = manifest.ringkasan()
print("\n--- ringkasan ---")
print(f" ok / gagal : {r['dokumen_ok']} / {r['dokumen_gagal']}")
print(f" total halaman : {r['total_halaman']}")
if r["detik_per_halaman"]:
print(f" detik/halaman : {r['detik_per_halaman']} "
f"(dari {r['halaman_terukur']} hal. yang benar-benar diparse)")
if r["total_peringatan"]:
print(f" ⚠ peringatan mutu: {r['total_peringatan']} — lihat manifest.json")
print(f" manifest : {run_dir / 'manifest.json'}")
return manifest
def main() -> None:
p = argparse.ArgumentParser(description="Pipeline parsing dokumen (MinerU)")
p.add_argument("--input", type=Path, help="folder berisi dokumen")
p.add_argument("--backend", choices=["pipeline", "vlm", "hybrid"],
help="pipeline=CPU, vlm=butuh GPU (target resmi)")
p.add_argument("--lang", default=None)
p.add_argument("--no-resume", action="store_true", help="proses ulang semua")
p.add_argument("--start-page", type=int, default=None)
p.add_argument("--end-page", type=int, default=None)
a = p.parse_args()
cfg = PipelineConfig()
if a.input:
cfg.input_dir = a.input
if a.backend:
cfg.backend = a.backend
if a.lang:
cfg.lang = a.lang
if a.no_resume:
cfg.resume = False
if a.start_page is not None:
cfg.start_page = a.start_page
if a.end_page is not None:
cfg.end_page = a.end_page
jalankan(cfg)
if __name__ == "__main__":
main()
|