[NOTICKET] fix(knowledge-parsing): stop the warning marker from failing documents
Browse filesA Windows console defaults to cp1252, which has no "warning" glyph. Printing
the quality-warning line raised UnicodeEncodeError INSIDE the per-document try
block, so a document that carried warnings was recorded as FAILED — even though
its parse had already succeeded and been written to the cache.
The effect was the exact inverse of the intent: clean documents passed, and the
ones whose quality needed attention were the ones thrown away. The same throw in
the run summary then took down the whole run with exit code 1.
Reproduced on the 9-page BUMA standard, which carries three warnings: it was
reported as "gagal" while its 31 chunks sat complete on disk. stdout and stderr
are now reconfigured to UTF-8 at CLI entry, so the run no longer depends on the
host codepage.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
- src/knowledge_parsing/run.py +12 -0
|
@@ -14,6 +14,7 @@ from __future__ import annotations
|
|
| 14 |
|
| 15 |
import argparse
|
| 16 |
import json
|
|
|
|
| 17 |
import time
|
| 18 |
import traceback
|
| 19 |
from datetime import datetime
|
|
@@ -137,6 +138,17 @@ def jalankan(cfg: PipelineConfig) -> Manifest:
|
|
| 137 |
|
| 138 |
|
| 139 |
def main() -> None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
p = argparse.ArgumentParser(description="Pipeline parsing dokumen (MinerU)")
|
| 141 |
p.add_argument("--input", type=Path, help="folder berisi dokumen")
|
| 142 |
p.add_argument("--backend", choices=["pipeline", "vlm", "hybrid"],
|
|
|
|
| 14 |
|
| 15 |
import argparse
|
| 16 |
import json
|
| 17 |
+
import sys
|
| 18 |
import time
|
| 19 |
import traceback
|
| 20 |
from datetime import datetime
|
|
|
|
| 138 |
|
| 139 |
|
| 140 |
def main() -> None:
|
| 141 |
+
# Konsol Windows default cp1252, yang tidak punya "⚠". Tanpa ini,
|
| 142 |
+
# mencetak peringatan mutu melempar UnicodeEncodeError DI DALAM blok try
|
| 143 |
+
# per dokumen — sehingga dokumen yang punya peringatan justru ditandai
|
| 144 |
+
# gagal, padahal hasil parse-nya sudah benar dan tersimpan. Kebalikan dari
|
| 145 |
+
# maksudnya: yang mutunya perlu diperhatikan malah yang dibuang.
|
| 146 |
+
for aliran in (sys.stdout, sys.stderr):
|
| 147 |
+
try:
|
| 148 |
+
aliran.reconfigure(encoding="utf-8", errors="replace") # type: ignore[union-attr]
|
| 149 |
+
except (AttributeError, OSError):
|
| 150 |
+
pass
|
| 151 |
+
|
| 152 |
p = argparse.ArgumentParser(description="Pipeline parsing dokumen (MinerU)")
|
| 153 |
p.add_argument("--input", type=Path, help="folder berisi dokumen")
|
| 154 |
p.add_argument("--backend", choices=["pipeline", "vlm", "hybrid"],
|