crazylemonade commited on
Commit
4a1bc37
·
verified ·
1 Parent(s): 84b8f32

Delete validate.py

Browse files
Files changed (1) hide show
  1. validate.py +0 -633
validate.py DELETED
@@ -1,633 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- Pre-flight validation script for MinerU OCR Service.
4
-
5
- Run by entrypoint.sh BEFORE uvicorn starts.
6
- Exits 0 if all checks pass.
7
- Exits 1 if any CRITICAL check fails — this crashes the container loudly
8
- so Hugging Face logs show an actionable error instead of a silent crash
9
- or a healthy-looking service that fails on every request.
10
-
11
- Usage:
12
- python validate.py # run all checks, exit 0/1
13
- python validate.py --soft # run all checks, always exit 0 (log only)
14
-
15
- ── FORENSIC NOTES (2025-06) ──────────────────────────────────────────────────
16
-
17
- OCR engine:
18
- The pipeline (full) backend uses paddleocr2pytorch — a self-contained
19
- PyTorch reimplementation of PaddleOCR bundled inside the magic-pdf wheel.
20
- It uses: torch, cv2, numpy, pyclipper, shapely, yaml.
21
- paddlepaddle and paddleocr packages are NOT installed and NOT needed.
22
-
23
- pp_structure_v2.py (which imports paddleocr) is only loaded in 'lite' model
24
- mode. Pipeline backend always uses 'full' mode (CustomPEKModel). That file is
25
- never imported at runtime.
26
-
27
- OCR model path resolution (from pytorch_paddle.py):
28
- ocr_models_dir = os.path.join(get_local_models_dir(), 'OCR', 'paddleocr_torch')
29
- det_model_path = os.path.join(ocr_models_dir, det_filename)
30
- where det_filename comes from models_config.yml keyed by language.
31
-
32
- Default CPU path: lang='ch' → forced to 'ch_lite' on CPU device.
33
- After Dockerfile Layer 3.5 patch:
34
- ch_lite.det = ch_PP-OCRv5_det_infer.pth (was ch_PP-OCRv3 — not in HF repo)
35
- ch_lite.rec = ch_PP-OCRv5_rec_infer.pth (unchanged — already in HF repo)
36
-
37
- Arch config lookup (from pytorchocr_utility.py):
38
- get_arch_config(model_path) uses Path(model_path).stem as the key into
39
- arch_config.yaml (bundled in magic-pdf wheel). Both replacement filenames
40
- have entries in arch_config.yaml — verified before patch was written.
41
-
42
- OpenCV conflict handling:
43
- doclayout-yolo, ultralytics, and rapid-table all declare opencv-python
44
- (non-headless) as a required dep. pip installs the full build in Layer 3.
45
- Layer 4 force-reinstalls opencv-python-headless to overwrite cv2. Both
46
- packages expose an identical cv2 API so all callers work correctly at
47
- runtime. pip-check shows warnings but they are harmless.
48
-
49
- onnxruntime:
50
- rapid-table declares onnxruntime>1.17.0 as a required (non-optional) dep.
51
- pip resolves it automatically when magic-pdf[full] is installed in Layer 3.
52
-
53
- slanet-plus.onnx (table model):
54
- Bundled inside the magic-pdf wheel at:
55
- magic_pdf/resources/slanet_plus/slanet-plus.onnx
56
- NOT downloaded from HF Hub — no separate download needed.
57
- """
58
-
59
- import importlib
60
- import json
61
- import os
62
- import shutil
63
- import sys
64
- import tempfile
65
- import traceback
66
-
67
- SOFT_MODE = "--soft" in sys.argv # never exit 1, just print
68
-
69
- MODELS_DIR = "/app/models"
70
- EXTRACT_KIT_MODELS = os.path.join(MODELS_DIR, "PDF-Extract-Kit-1.0", "models")
71
- LAYOUT_MARKER = os.path.join(EXTRACT_KIT_MODELS, "Layout") # canary directory
72
- CONFIG_PATH = os.path.expanduser("~/magic-pdf.json")
73
-
74
-
75
- # ── helpers ────────────────────────────────────────────────────────────────────
76
- def ok(label: str, detail: str = "") -> None:
77
- suffix = f" ({detail})" if detail else ""
78
- print(f" ✓ {label}{suffix}", flush=True)
79
-
80
-
81
- def fail(label: str, detail: str, critical: bool = True) -> None:
82
- tag = "CRITICAL" if critical else "WARNING"
83
- print(f" ✗ [{tag}] {label}: {detail}", flush=True)
84
-
85
-
86
- def section(title: str) -> None:
87
- print(f"\n{'─' * 60}", flush=True)
88
- print(f" {title}", flush=True)
89
- print(f"{'─' * 60}", flush=True)
90
-
91
-
92
- # ── check registry ─────────────────────────────────────────────────────────────
93
- failures: list[tuple[str, str]] = []
94
- warnings: list[tuple[str, str]] = []
95
-
96
-
97
- def record_fail(label: str, detail: str, critical: bool = True) -> None:
98
- fail(label, detail, critical)
99
- if critical:
100
- failures.append((label, detail))
101
- else:
102
- warnings.append((label, detail))
103
-
104
-
105
- # ═══════════════════════════════════════════════════════════════════════════════
106
- print("\n" + "═" * 60, flush=True)
107
- print(" MinerU OCR Service — Pre-flight Validation", flush=True)
108
- print("═" * 60, flush=True)
109
-
110
- # ── 1. Python version ──────────────────────────────────────────────────────────
111
- section("1. Python runtime")
112
- pv = sys.version_info
113
- if pv >= (3, 10):
114
- ok("Python version", f"{pv.major}.{pv.minor}.{pv.micro}")
115
- else:
116
- record_fail("Python version",
117
- f"{pv.major}.{pv.minor} detected — magic-pdf requires >= 3.10")
118
-
119
- # ── 2. cv2 ─────────────────────────────────────────────────────────────────────
120
- section("2. OpenCV (cv2)")
121
- try:
122
- import cv2
123
- ok("cv2 import", f"version {cv2.__version__}")
124
- build = cv2.getBuildInformation()
125
- if "GTK" in build or "Qt" in build:
126
- record_fail("cv2 build", "GUI backend detected — use opencv-python-headless",
127
- critical=False)
128
- else:
129
- ok("cv2 headless", "no GUI backend detected")
130
- except ImportError as exc:
131
- record_fail(
132
- "cv2 import",
133
- f"{exc}. "
134
- "Layer 4 force-reinstall of opencv-python-headless may have failed. "
135
- "Check Docker build log for the 'pip install --force-reinstall opencv-python-headless' step.",
136
- )
137
- except Exception as exc:
138
- record_fail("cv2 import", f"unexpected error: {exc}")
139
-
140
- # ── 3. PyTorch ─────────────────────────────────────────────────────────────────
141
- section("3. PyTorch + TorchVision")
142
- try:
143
- import torch
144
- ok("torch import", f"version {torch.__version__}")
145
- if torch.cuda.is_available():
146
- record_fail("torch CUDA", "CUDA detected on CPU-only space — unexpected",
147
- critical=False)
148
- else:
149
- ok("torch device", "CPU-only (expected for free tier)")
150
- except ImportError as exc:
151
- record_fail(
152
- "torch import",
153
- f"{exc}. "
154
- "Install from PyTorch CPU index BEFORE magic-pdf in Dockerfile Layer 2: "
155
- "pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision",
156
- )
157
- except Exception as exc:
158
- record_fail("torch import", f"unexpected: {exc}")
159
-
160
- try:
161
- import torchvision
162
- ok("torchvision import", f"version {torchvision.__version__}")
163
- except ImportError as exc:
164
- record_fail("torchvision import", str(exc))
165
- except Exception as exc:
166
- record_fail("torchvision import", f"unexpected: {exc}")
167
-
168
- # ── 4. ultralytics ─────────────────────────────────────────────────────────────
169
- section("4. ultralytics (YOLO — required by doclayout_yolo)")
170
- try:
171
- import ultralytics
172
- ok("ultralytics import", f"version {ultralytics.__version__}")
173
- except ImportError as exc:
174
- record_fail(
175
- "ultralytics import",
176
- f"{exc}. "
177
- "Provided by magic-pdf[full]. "
178
- "ROOT CAUSE: [full-cpu] is NOT a valid extra in magic-pdf 1.3.12 — "
179
- "pip silently installed only the base package when given an unknown extra. "
180
- "Dockerfile Layer 3 must use magic-pdf[full]==1.3.12 (not [full-cpu]).",
181
- )
182
- except Exception as exc:
183
- record_fail("ultralytics import", f"unexpected: {exc}")
184
-
185
- # ── 5. doclayout_yolo ──────────────────────────────────────────────────────────
186
- section("5. doclayout_yolo (layout detection model)")
187
- try:
188
- import doclayout_yolo
189
- ok("doclayout_yolo import", f"version {getattr(doclayout_yolo, '__version__', 'unknown')}")
190
- except ImportError as exc:
191
- record_fail(
192
- "doclayout_yolo import",
193
- f"{exc}. "
194
- "Provided by magic-pdf[full] (version 0.0.2b1). "
195
- "doclayout-yolo==0.0.2b1 is only on the myhloli custom wheel index — "
196
- "Dockerfile Layer 3 must include: "
197
- "--extra-index-url https://myhloli.github.io/wheels/",
198
- )
199
- except Exception as exc:
200
- record_fail("doclayout_yolo import", f"unexpected: {exc}")
201
-
202
- # ── 6. rapid_table ─────────────────────────────────────────────────────────────
203
- section("6. rapid_table (table extraction)")
204
- try:
205
- import rapid_table
206
- ok("rapid_table import", f"version {getattr(rapid_table, '__version__', 'unknown')}")
207
- except ImportError as exc:
208
- record_fail(
209
- "rapid_table import",
210
- f"{exc}. Provided by magic-pdf[full]. Check Layer 3 install.",
211
- )
212
- except Exception as exc:
213
- record_fail("rapid_table import", f"unexpected: {exc}")
214
-
215
- # ── 7. onnxruntime ─────────────────────────────────────────────────────────────
216
- section("7. onnxruntime (required by rapid-table for table model inference)")
217
- # onnxruntime is a required (non-optional) dep of rapid-table>=1.0.5.
218
- # pip resolves it automatically when magic-pdf[full] is installed in Layer 3.
219
- # If it is missing it means rapid-table itself failed to install.
220
- try:
221
- import onnxruntime
222
- ok("onnxruntime import", f"version {onnxruntime.__version__}")
223
- except ImportError as exc:
224
- record_fail(
225
- "onnxruntime import",
226
- f"{exc}. "
227
- "onnxruntime is a required dep of rapid-table>=1.0.5. "
228
- "Its absence means rapid-table failed to install in Layer 3. "
229
- "Check Docker build log for rapid-table install errors.",
230
- )
231
- except Exception as exc:
232
- record_fail("onnxruntime import", f"unexpected: {exc}")
233
-
234
- # ── 8. magic_pdf core imports ──────────────────────────────────────────────────
235
- section("8. magic_pdf core imports")
236
-
237
- REQUIRED_IMPORTS = [
238
- ("magic_pdf.data.dataset", ["PymuDocDataset", "ImageDataset"]),
239
- ("magic_pdf.data.data_reader_writer", ["FileBasedDataReader", "FileBasedDataWriter"]),
240
- ("magic_pdf.model.doc_analyze_by_custom_model", ["doc_analyze"]),
241
- ("magic_pdf.config.enums", ["SupportedPdfParseMethod"]),
242
- ]
243
-
244
- for module_path, symbols in REQUIRED_IMPORTS:
245
- try:
246
- mod = importlib.import_module(module_path)
247
- missing = [s for s in symbols if not hasattr(mod, s)]
248
- if missing:
249
- record_fail(f"{module_path}", f"missing symbols: {missing}")
250
- else:
251
- ok(module_path, ", ".join(symbols))
252
- except ImportError as exc:
253
- record_fail(module_path, str(exc))
254
- except Exception as exc:
255
- record_fail(module_path, f"unexpected: {exc}")
256
-
257
- # ── 8b. paddleocr2pytorch (OCR engine bundled inside magic-pdf wheel) ──────────
258
- section("8b. paddleocr2pytorch (PyTorch OCR — bundled in magic-pdf wheel)")
259
- # This is the actual OCR engine for the pipeline backend.
260
- # It is NOT a separate pip package — it lives inside the magic-pdf wheel at
261
- # magic_pdf/model/sub_modules/ocr/paddleocr2pytorch/
262
- # If it is missing, the entire magic-pdf package did not install correctly.
263
- try:
264
- from magic_pdf.model.sub_modules.ocr.paddleocr2pytorch.pytorch_paddle import PytorchPaddleOCR
265
- ok("PytorchPaddleOCR (paddleocr2pytorch)", "bundled inside magic-pdf wheel — no paddlepaddle pkg needed")
266
- except ImportError as exc:
267
- record_fail(
268
- "PytorchPaddleOCR import",
269
- f"{exc}. "
270
- "This module is bundled inside magic_pdf/model/sub_modules/ocr/paddleocr2pytorch/. "
271
- "If missing, magic-pdf itself did not install correctly.",
272
- )
273
- except Exception as exc:
274
- record_fail("PytorchPaddleOCR import", f"unexpected: {exc}")
275
-
276
- # ── 8c. Deprecated API check ───────────────────────────────────────────────────
277
- section("8c. Deprecated API check (should NOT exist)")
278
- OBSOLETE = [
279
- "magic_pdf.pipe.UNIPipe",
280
- "magic_pdf.rw.DiskReaderWriter",
281
- ]
282
- for mod_path in OBSOLETE:
283
- try:
284
- importlib.import_module(mod_path)
285
- record_fail(mod_path, "still importable — code may use old API", critical=False)
286
- except ImportError:
287
- ok(f"{mod_path} (correctly absent)")
288
-
289
- # ── 9. End-to-end pipeline smoke test ─────────────────────────────────────────
290
- section("9. End-to-end pipeline smoke test")
291
- try:
292
- from magic_pdf.model.doc_analyze_by_custom_model import doc_analyze # noqa: F401
293
- import ultralytics # noqa: F401
294
- from magic_pdf.data.dataset import ImageDataset # noqa: F401
295
- from magic_pdf.data.data_reader_writer import FileBasedDataReader, FileBasedDataWriter # noqa: F401
296
- ok("Pipeline imports (doc_analyze + ultralytics + ImageDataset + readers)", "all OK")
297
- except ImportError as exc:
298
- record_fail(
299
- "Pipeline smoke test",
300
- f"Full pipeline import chain failed: {exc}. "
301
- "This means POST /extract will fail on every request.",
302
- )
303
- except Exception as exc:
304
- record_fail("Pipeline smoke test", f"unexpected: {exc}")
305
-
306
- # ── 10. Config file ────────────────────────────────────────────────────────────
307
- section("10. MinerU config (magic-pdf.json)")
308
- _cfg: dict = {}
309
- if os.path.exists(CONFIG_PATH):
310
- try:
311
- with open(CONFIG_PATH) as f:
312
- _cfg = json.load(f)
313
- required_keys = ["models-dir", "device-mode"]
314
- missing_keys = [k for k in required_keys if k not in _cfg]
315
- if missing_keys:
316
- record_fail("Config keys", f"missing: {missing_keys}")
317
- else:
318
- ok("Config file", CONFIG_PATH)
319
- ok("device-mode", _cfg.get("device-mode", "?"))
320
- ok("models-dir", _cfg.get("models-dir", "?"))
321
- ok("formula-enable", str(_cfg.get("formula-config", {}).get("enable", "?")))
322
- ok("table-enable", str(_cfg.get("table-config", {}).get("enable", "?")))
323
- except json.JSONDecodeError as exc:
324
- record_fail("Config file", f"invalid JSON: {exc}")
325
- except Exception as exc:
326
- record_fail("Config file", str(exc))
327
- else:
328
- record_fail(
329
- "Config file",
330
- f"not found at {CONFIG_PATH}. "
331
- "Run download_models.py or check Docker build log.",
332
- )
333
-
334
- # ── 11. Model directory structure ─────────────────────────────────────────────
335
- section("11. Model directory structure")
336
-
337
- model_dir_checks = [
338
- ("PDF-Extract-Kit-1.0 root", os.path.join(MODELS_DIR, "PDF-Extract-Kit-1.0")),
339
- ("Layout models", os.path.join(EXTRACT_KIT_MODELS, "Layout")),
340
- ("Layout/YOLO", os.path.join(EXTRACT_KIT_MODELS, "Layout", "YOLO")),
341
- ("OCR models", os.path.join(EXTRACT_KIT_MODELS, "OCR")),
342
- ("OCR/paddleocr_torch", os.path.join(EXTRACT_KIT_MODELS, "OCR", "paddleocr_torch")),
343
- ("Table models (TabRec)", os.path.join(EXTRACT_KIT_MODELS, "TabRec")),
344
- ]
345
-
346
- for label, path in model_dir_checks:
347
- if os.path.isdir(path):
348
- try:
349
- n = sum(1 for _ in os.scandir(path))
350
- ok(label, f"{n} entries [{path}]")
351
- except OSError:
352
- ok(label, path)
353
- else:
354
- record_fail(label, f"directory not found: {path}")
355
-
356
- lr_dir = os.path.join(MODELS_DIR, "layoutreader")
357
- if os.path.isdir(lr_dir):
358
- ok("layoutreader (optional)", lr_dir)
359
- else:
360
- record_fail("layoutreader (optional)",
361
- "not found — MinerU will use fallback ordering (non-critical)",
362
- critical=False)
363
-
364
- # ── 11b. Critical model weight files ──────────────────────────────────────────
365
- section("11b. Critical model weight files")
366
- #
367
- # These are the EXACT files MinerU will try to open when processing a document
368
- # on a CPU deployment (default language = ch → forced to ch_lite on CPU).
369
- #
370
- # After Dockerfile Layer 3.5 patch, models_config.yml now references:
371
- # ch_lite.det = ch_PP-OCRv5_det_infer.pth (patched from v3 — v3 NOT in repo)
372
- # ch_lite.rec = ch_PP-OCRv5_rec_infer.pth (unchanged — always in repo)
373
- #
374
- # Layout uses doclayout_yolo (from magic-pdf.json layout-config).
375
- # Table (rapid_table) uses slanet-plus.onnx BUNDLED IN THE WHEEL — not here.
376
- # Formula is DISABLED — MFD/MFR files not required.
377
- #
378
- # Any CRITICAL failure here = service boots but crashes on first document.
379
-
380
- _ocr_dir = os.path.join(EXTRACT_KIT_MODELS, "OCR", "paddleocr_torch")
381
-
382
- CRITICAL_WEIGHT_FILES: list[tuple[str, str, str]] = [
383
- # (label, relative-to-EXTRACT_KIT_MODELS, reason)
384
- (
385
- "OCR det weight (ch_lite, default CPU lang)",
386
- os.path.join("OCR", "paddleocr_torch", "ch_PP-OCRv5_det_infer.pth"),
387
- "Patched from ch_PP-OCRv3_det_infer.pth (absent in HF repo). "
388
- "Missing = all OCR will crash at model load time."
389
- ),
390
- (
391
- "OCR rec weight (ch_lite)",
392
- os.path.join("OCR", "paddleocr_torch", "ch_PP-OCRv5_rec_infer.pth"),
393
- "Recognition model for ch_lite. "
394
- "Missing = OCR loads det but crashes at recognition."
395
- ),
396
- (
397
- "OCR cls weight (angle classifier)",
398
- os.path.join("OCR", "paddleocr_torch", "ch_ptocr_mobile_v2.0_cls_infer.pth"),
399
- "Used when use_angle_cls=True. Default is False so non-critical, "
400
- "but its absence causes crash if angle classification is enabled."
401
- ),
402
- (
403
- "Layout YOLO weight (doclayout_yolo)",
404
- os.path.join("Layout", "YOLO", "doclayout_yolo_docstructbench_imgsz1280_2501.pt"),
405
- "Layout detection model. Missing = layout detection crashes on every document."
406
- ),
407
- (
408
- "Layout LayoutLMv3 weight",
409
- os.path.join("Layout", "LayoutLMv3", "model_final.pth"),
410
- "Alternative layout model. Required even when doclayout_yolo is primary "
411
- "because model_configs.yaml always lists it."
412
- ),
413
- (
414
- "Multilingual OCR det (en/latin fallback)",
415
- os.path.join("OCR", "paddleocr_torch", "Multilingual_PP-OCRv3_det_infer.pth"),
416
- "Patched det for en and latin languages. Missing = crash if lang=en/latin."
417
- ),
418
- ]
419
-
420
- # cls weight is only critical if use_angle_cls=True (default False)
421
- NON_CRITICAL_LABELS = {"OCR cls weight (angle classifier)"}
422
-
423
- for label, rel_path, reason in CRITICAL_WEIGHT_FILES:
424
- full_path = os.path.join(EXTRACT_KIT_MODELS, rel_path)
425
- is_critical = label not in NON_CRITICAL_LABELS
426
- if os.path.isfile(full_path):
427
- size_mb = os.path.getsize(full_path) / (1024 * 1024)
428
- ok(label, f"{size_mb:.1f} MB [{full_path}]")
429
- else:
430
- record_fail(
431
- label,
432
- f"FILE NOT FOUND: {full_path}\n"
433
- f" Reason: {reason}",
434
- critical=is_critical,
435
- )
436
-
437
- # ── 11c. models_config.yml consistency check ──────────────────────────────────
438
- section("11c. models_config.yml consistency check")
439
- #
440
- # Reads the installed models_config.yml (inside magic_pdf package) and verifies
441
- # that every det/rec file it references for the default CPU language (ch_lite)
442
- # actually exists on disk in the expected location.
443
- #
444
- # This catches future version drift between the magic-pdf package and the HF repo
445
- # BEFORE the service starts, rather than mid-request.
446
-
447
- try:
448
- import magic_pdf
449
- import yaml as _yaml
450
- from pathlib import Path as _Path
451
-
452
- _pkg = _Path(magic_pdf.__file__).parent
453
- _mcfg = _pkg / 'model/sub_modules/ocr/paddleocr2pytorch/pytorchocr/utils/resources/models_config.yml'
454
-
455
- if not _mcfg.exists():
456
- record_fail("models_config.yml", f"not found at expected path: {_mcfg}")
457
- else:
458
- with open(_mcfg) as _f:
459
- _mc = _yaml.safe_load(_f)
460
-
461
- _ocr_torch = os.path.join(EXTRACT_KIT_MODELS, "OCR", "paddleocr_torch")
462
-
463
- # Check the two languages actually used on this CPU deployment
464
- _check_langs = ["ch_lite", "ch"]
465
- _mc_ok = True
466
- for _lang in _check_langs:
467
- _entry = _mc.get("lang", {}).get(_lang, {})
468
- for _field in ("det", "rec"):
469
- _fname = _entry.get(_field)
470
- if not _fname:
471
- continue
472
- _fpath = os.path.join(_ocr_torch, _fname)
473
- if os.path.isfile(_fpath):
474
- ok(f"models_config[{_lang}].{_field}", _fname)
475
- else:
476
- record_fail(
477
- f"models_config[{_lang}].{_field}",
478
- f"Config references '{_fname}' but file not found at:\n"
479
- f" {_fpath}\n"
480
- f" Dockerfile Layer 3.5 patch may not have run, "
481
- f"or HF repo changed its file structure again.",
482
- critical=True,
483
- )
484
- _mc_ok = False
485
-
486
- if _mc_ok:
487
- ok("models_config.yml consistency", "all referenced det/rec files exist on disk")
488
-
489
- except Exception as _exc:
490
- record_fail("models_config.yml consistency check", f"unexpected error: {_exc}", critical=False)
491
-
492
- # ── 11d. Bundled wheel resources ──────────────────────────────────────────────
493
- section("11d. Bundled wheel resources (inside magic_pdf package)")
494
- #
495
- # These files are shipped inside the magic-pdf wheel itself.
496
- # They do NOT come from the HF download. Their absence means the wheel
497
- # installed incorrectly or was corrupted.
498
-
499
- try:
500
- import magic_pdf as _mp
501
- from pathlib import Path as _P
502
-
503
- _pkg_root = _P(_mp.__file__).parent
504
- _bundled = [
505
- ("slanet-plus.onnx (table model)",
506
- _pkg_root / "resources" / "slanet_plus" / "slanet-plus.onnx"),
507
- ("fasttext langdetect model",
508
- _pkg_root / "resources" / "fasttext-langdetect" / "lid.176.ftz"),
509
- ("YOLO langdetect model",
510
- _pkg_root / "resources" / "yolov11-langdetect" / "yolo_v11_ft.pt"),
511
- ("model_configs.yaml (weight path map)",
512
- _pkg_root / "resources" / "model_config" / "model_configs.yaml"),
513
- ]
514
- for _lbl, _p in _bundled:
515
- if _p.exists():
516
- _sz = _p.stat().st_size / (1024 * 1024)
517
- ok(_lbl, f"{_sz:.2f} MB")
518
- else:
519
- record_fail(_lbl, f"expected inside wheel at {_p} — magic-pdf install may be corrupted")
520
-
521
- except Exception as _exc:
522
- record_fail("Bundled wheel resources check", f"unexpected: {_exc}", critical=False)
523
-
524
- # ── 12. Temp storage ───────────────────────────────────────────────────────────
525
- section("12. Temp storage")
526
- try:
527
- td = tempfile.mkdtemp(prefix="mineru_validate_")
528
- test_file = os.path.join(td, "write_test.bin")
529
- with open(test_file, "wb") as f:
530
- f.write(b"x" * 4096)
531
- assert os.path.getsize(test_file) == 4096
532
- shutil.rmtree(td)
533
- ok("Temp write + delete", tempfile.gettempdir())
534
- except Exception as exc:
535
- record_fail("Temp storage", str(exc))
536
-
537
- # ── 13. System memory (cgroups) ────────────────────────────────────────────────
538
- section("13. System memory (cgroups)")
539
- mem_source = "unknown"
540
- total_mb = used_mb = 0
541
-
542
- try:
543
- with open("/sys/fs/cgroup/memory.max") as f:
544
- raw = f.read().strip()
545
- if raw != "max":
546
- total_mb = int(raw) // (1024 * 1024)
547
- with open("/sys/fs/cgroup/memory.current") as f:
548
- used_mb = int(f.read().strip()) // (1024 * 1024)
549
- mem_source = "cgroups v2"
550
- except (FileNotFoundError, ValueError):
551
- pass
552
-
553
- if total_mb == 0:
554
- try:
555
- with open("/sys/fs/cgroup/memory/memory.limit_in_bytes") as f:
556
- limit = int(f.read().strip())
557
- with open("/sys/fs/cgroup/memory/memory.usage_in_bytes") as f:
558
- used_bytes = int(f.read().strip())
559
- if limit < 128 * 1024 * 1024 * 1024:
560
- total_mb = limit // (1024 * 1024)
561
- used_mb = used_bytes // (1024 * 1024)
562
- mem_source = "cgroups v1"
563
- except (FileNotFoundError, ValueError):
564
- pass
565
-
566
- if total_mb == 0:
567
- try:
568
- info: dict[str, int] = {}
569
- with open("/proc/meminfo") as f:
570
- for line in f:
571
- parts = line.split()
572
- if len(parts) >= 2:
573
- info[parts[0].rstrip(":")] = int(parts[1])
574
- total_mb = info.get("MemTotal", 0) // 1024
575
- used_mb = (info.get("MemTotal", 0) - info.get("MemAvailable", 0)) // 1024
576
- mem_source = "/proc/meminfo (may show host RAM)"
577
- except Exception:
578
- pass
579
-
580
- ok("Memory source", mem_source)
581
- ok("Total memory", f"{total_mb} MB")
582
- ok("Used memory", f"{used_mb} MB")
583
- ok("Free memory", f"{total_mb - used_mb} MB")
584
-
585
- if total_mb > 32 * 1024:
586
- record_fail(
587
- "Memory total",
588
- f"{total_mb} MB seems too large — cgroups may not be available; "
589
- "/proc/meminfo showing host RAM. Memory guard in main.py will be conservative.",
590
- critical=False,
591
- )
592
-
593
- # ── 14. /proc/meminfo sanity ───────────────────────────────────────────────────
594
- section("14. /proc/meminfo (reference)")
595
- try:
596
- with open("/proc/meminfo") as f:
597
- lines = f.readlines()[:5]
598
- for line in lines:
599
- parts = line.split()
600
- if len(parts) >= 2:
601
- kb = int(parts[1])
602
- ok(parts[0].rstrip(":"), f"{kb // 1024} MB")
603
- except Exception as exc:
604
- record_fail("/proc/meminfo", str(exc), critical=False)
605
-
606
- # ═══════════════════════════════════════════════════════════════════════════════
607
- # Summary
608
- # ═══════════════════════════════════════════════════════════════════════════════
609
- print("\n" + "═" * 60, flush=True)
610
- print(" Validation Summary", flush=True)
611
- print("═" * 60, flush=True)
612
-
613
- if warnings:
614
- print(f"\n ⚠ {len(warnings)} warning(s):", flush=True)
615
- for label, detail in warnings:
616
- print(f" • {label}: {detail}", flush=True)
617
-
618
- if failures:
619
- print(f"\n ✗ {len(failures)} CRITICAL failure(s):", flush=True)
620
- for label, detail in failures:
621
- print(f" • {label}: {detail}", flush=True)
622
- print("\n Service will NOT start until these are resolved.", flush=True)
623
- print(" Check Dockerfile pip layers and Docker build log.", flush=True)
624
- print("═" * 60 + "\n", flush=True)
625
- if not SOFT_MODE:
626
- sys.exit(1)
627
- else:
628
- print(f"\n ✓ All critical checks passed", flush=True)
629
- if warnings:
630
- print(f" ⚠ {len(warnings)} non-critical warning(s) — see above", flush=True)
631
- print("\n Service is ready to start.", flush=True)
632
- print("═" * 60 + "\n", flush=True)
633
- sys.exit(0)