Upload 7 files

Dockerfile

@@ -0,0 +1,84 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# MinerU OCR Service — Hugging Face Docker Space (CPU / pipeline backend)
+#
+# Optimized for FREE tier: 2 vCPU · 16 GB RAM · 50 GB Disk · No GPU
+#
+# System packages — what was removed and why:
+#   libreoffice      — 1.5 GB installed; caused build timeouts/OOM.
+#   libsm6 libxext6
+#   libxrender-dev   — X11 display stubs; only needed for cv2.imshow() GUI.
+#                       Headless server never opens a display.
+#   libmagic1        — Only needed by python-magic, which is not used.
+#   wget curl        — Runtime testing tools, not needed inside container.
+#
+# System packages — what was kept and why:
+#   libgl1           — OpenCV requires libGL.so.1 for all image ops (not GUI).
+#   libglib2.0-0     — GLib; required by OpenCV and many C extensions.
+#   libgomp1         — OpenMP; required by ONNX Runtime and YOLO inference.
+#   poppler-utils    — pdfinfo / pdftoppm used by MinerU PDF pre-processing.
+#
+# Pip strategy — TWO separate RUN layers for cache granularity:
+#   Layer 1: small/fast packages + opencv-python-headless.
+#            opencv-python-headless MUST be in this layer so that when
+#            magic-pdf resolves cv2 in layer 2, the headless wheel is already
+#            present and pip keeps it (avoids pulling in the full X11 build).
+#   Layer 2: magic-pdf[full-cpu] — large, slow, custom wheel index.
+#            Separate layer so code-only rebuilds don't re-download 2+ GB.
+# ─────────────────────────────────────────────────────────────────────────────
+
+FROM python:3.10-slim
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PORT=7860
+ENV MINERU_DEVICE_MODE=cpu
+ENV MINERU_BACKEND=pipeline
+
+# ── System dependencies (minimal confirmed set) ───────────────────────────────
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        libgl1 \
+        libglib2.0-0 \
+        libgomp1 \
+        poppler-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# ── Layer 1: small + opencv-headless (cached unless versions change) ──────────
+# opencv-python-headless installed HERE so layer-2 magic-pdf install sees cv2
+# already satisfied and does not pull in the full X11-dependent opencv-python.
+RUN pip install --no-cache-dir --timeout 300 \
+        "fastapi>=0.115.0" \
+        "uvicorn[standard]>=0.32.0" \
+        "python-multipart>=0.0.12" \
+        "Pillow>=10.0.0" \
+        "pillow-heif>=0.18.0" \
+        "huggingface_hub>=0.25.0" \
+        "opencv-python-headless>=4.8.0"
+
+# ── Layer 2: magic-pdf (large; cached unless version pin changes) ─────────────
+RUN pip install --no-cache-dir --timeout 300 \
+        --extra-index-url https://myhloli.github.io/wheels/ \
+        "magic-pdf[full-cpu]==1.3.12"
+
+# ── Application code ──────────────────────────────────────────────────────────
+COPY download_models.py .
+COPY validate.py .
+COPY main.py .
+COPY entrypoint.sh .
+RUN chmod +x entrypoint.sh
+
+# ── Download models at build time ─────────────────────────────────────────────
+# Baked into image = zero cold-start download delay.
+# Skip-if-exists logic in download_models.py gives Docker layer-cache reuse:
+# code-only rebuilds skip the 15-minute model download entirely.
+# MFR (formula recognition, ~1-2 GB) is excluded — disabled in config.
+RUN python download_models.py
+
+# Persist config; entrypoint.sh restores it if /root is wiped on restart.
+RUN mkdir -p /app/config && cp /root/magic-pdf.json /app/config/magic-pdf.json
+
+# ── Runtime ───────────────────────────────────────────────────────────────────
+EXPOSE 7860
+ENTRYPOINT ["/app/entrypoint.sh"]

README.md

@@ -0,0 +1,131 @@
+---
+title: MinerU OCR Service
+emoji: 📄
+colorFrom: blue
+colorTo: green
+sdk: docker
+app_port: 7860
+pinned: false
+---
+
+# MinerU OCR & Document Extraction Service
+
+Production-quality OCR and document extraction API powered by **MinerU** (`magic-pdf` package), running on Hugging Face Docker Spaces — free CPU tier.
+
+---
+
+## Root Cause Analysis — Previous Build Failures
+
+| Cause | Detail | Fix Applied |
+|---|---|---|
+| `libreoffice` in apt | ~1.5 GB installed; caused disk/OOM during build | **Removed** — DOCX/PPTX/XLSX dropped |
+| X11 libs (`libsm6`, `libxext6`, `libxrender-dev`) | Only needed for OpenCV GUI windows (cv2.imshow); headless server never uses a display | **Removed** |
+| `libmagic1` | C dep for python-magic, which was never imported | **Removed** |
+| `python-magic` in pip | Listed in requirements but never used in code | **Removed** |
+| `wget` / `curl` in apt | Runtime testing tools, not needed in container | **Removed** |
+| Single pip layer | Any failure re-downloads all packages including 2 GB magic-pdf | **Split into 2 cached layers** |
+| MFR models downloaded | Formula recognition (unimernet, ~1-2 GB) downloaded even though disabled | **Excluded via ignore_patterns** |
+| `ModuleNotFoundError: magic_pdf.pipe.UNIPipe` | API removed in magic-pdf >= 1.0 | **Replaced** with current API |
+
+---
+
+## Supported File Types
+
+| Category | Extensions |
+|---|---|
+| PDF | `.pdf` (searchable, scanned, mixed) |
+| Images | `.jpg` `.jpeg` `.png` `.webp` `.bmp` `.tiff` `.tif` `.gif` `.heic` `.heif` `.avif` |
+
+> DOCX / PPTX / XLSX were removed because they required LibreOffice, which caused build failures on free-tier hardware. Add them back once the base deployment is stable.
+
+---
+
+## API Endpoints
+
+### `GET /health`
+```json
+{ "status": "healthy" }
+```
+
+### `GET /status`
+```json
+{
+  "status": "healthy",
+  "provider": "mineru",
+  "version": "1.3.12",
+  "modelsLoaded": true,
+  "uptimeSeconds": 3742,
+  "memoryUsedMB": 5200,
+  "memoryTotalMB": 16384,
+  "activeRequests": 0,
+  "cacheEntries": 3
+}
+```
+
+### `POST /extract`
+```json
+{
+  "success": true,
+  "filename": "invoice.pdf",
+  "docType": "invoice",
+  "pageCount": 2,
+  "confidence": 0.95,
+  "markdown": "# Invoice\n\n...",
+  "metadata": {
+    "parseMethod": "txt",
+    "backend": "pipeline",
+    "cached": false,
+    "processingTimeMs": 4200,
+    "docTypeClassification": "invoice",
+    "imageCount": 0,
+    "tableCount": 1,
+    "formulaCount": 0
+  }
+}
+```
+
+### `POST /batch`
+```json
+{
+  "success": true,
+  "processed": 3,
+  "results": [...]
+}
+```
+
+---
+
+## Deployment
+
+1. Create a new Hugging Face Space → **Docker** SDK
+2. Upload all files from this directory to the Space repo root
+3. Space builds automatically (~20–30 min for first build; subsequent code-only rebuilds are faster due to Docker layer cache)
+4. Test with curl:
+
+```bash
+curl https://<your-space>.hf.space/health
+curl -X POST https://<your-space>.hf.space/extract -F "file=@doc.pdf"
+curl -X POST https://<your-space>.hf.space/batch \
+  -F "files=@a.pdf" -F "files=@b.jpg" -F "files=@c.png"
+```
+
+---
+
+## Resource Budget (free CPU Basic)
+
+| Resource | Budget | Usage |
+|---|---|---|
+| Disk | 50 GB | ~8–10 GB (packages ~3 GB + models ~4–5 GB) |
+| RAM | 16 GB | ~6–8 GB at load; peaks ~10 GB during extraction |
+| vCPU | 2 | Single uvicorn worker; sequential batch processing |
+
+---
+
+## Known Limitations
+
+1. **No DOCX/PPTX/XLSX** — dropped to make the build reliable. Re-add with LibreOffice on paid hardware.
+2. **Formula OCR disabled** — MFR models excluded to save ~1-2 GB. Set `"enable": true` in `~/magic-pdf.json` and re-download MFR models to enable.
+3. **First-request latency** — models are lazy-loaded into RAM on the first request (~30–60 s).
+4. **30 MB file size limit** — enforced on both `/extract` and `/batch`.
+5. **In-process cache** — SHA256 cache lives in RAM, cleared on container restart.
+6. **HEIC/HEIF** — requires `pillow-heif` (included); may fail on unusual encoder variants.

download_models.py

@@ -0,0 +1,130 @@
+"""
+Download MinerU pipeline models from Hugging Face Hub.
+
+Called once during Docker build: python download_models.py
+
+Optimizations vs original:
+  - Skip-if-exists: if models are already present (Docker layer cache reuse),
+    the entire download is skipped without re-downloading anything.
+  - MFR excluded: formula recognition models (unimernet, ~1-2 GB) are not
+    downloaded because formula recognition is disabled in the config. Excluding
+    them cuts download time and image size significantly.
+  - layoutreader optional: if download fails (network issue, repo unavailable),
+    the script logs a warning and continues. MinerU falls back to its built-in
+    layout ordering without layoutreader.
+
+Models saved to /app/models/
+Config written to /root/magic-pdf.json
+"""
+
+import json
+import os
+import sys
+
+MODELS_DIR = "/app/models"
+EXTRACT_KIT_DIR = os.path.join(MODELS_DIR, "PDF-Extract-Kit-1.0")
+LAYOUTREADER_DIR = os.path.join(MODELS_DIR, "layoutreader")
+
+
+def _models_present() -> bool:
+    """Return True if the key layout-model directory already exists."""
+    marker = os.path.join(EXTRACT_KIT_DIR, "models", "Layout")
+    return os.path.isdir(marker)
+
+
+def _write_config(layoutreader_dir: str) -> None:
+    config = {
+        "bucket_info": {},
+        "models-dir": os.path.join(EXTRACT_KIT_DIR, "models"),
+        "layoutreader-model-dir": layoutreader_dir,
+        "device-mode": "cpu",
+        "layout-config": {
+            "model": "doclayout_yolo"
+        },
+        "formula-config": {
+            "mfd_model": "yolo_v8_mfd",
+            "mfr_model": "unimernet_small",
+            "enable": False
+        },
+        "table-config": {
+            "model": "rapid_table",
+            "enable": True,
+            "max_time": 400
+        },
+        "backend": "pipeline"
+    }
+    config_path = os.path.expanduser("~/magic-pdf.json")
+    with open(config_path, "w") as f:
+        json.dump(config, f, indent=2)
+    print(f"Config written → {config_path}")
+    return config_path
+
+
+def download() -> None:
+    try:
+        from huggingface_hub import snapshot_download
+    except ImportError:
+        print("ERROR: huggingface_hub not installed", file=sys.stderr)
+        sys.exit(1)
+
+    # ── Skip-if-exists ────────────────────────────────────────────────────────
+    if _models_present():
+        print("Models already present — skipping download (Docker layer cache).")
+        # Config may still need writing if this is a fresh container from cached layer
+        lr_dir = LAYOUTREADER_DIR if os.path.isdir(LAYOUTREADER_DIR) else ""
+        _write_config(lr_dir)
+        return
+
+    os.makedirs(MODELS_DIR, exist_ok=True)
+
+    # ── PDF-Extract-Kit-1.0 ───────────────────────────────────────────────────
+    # Excluded via ignore_patterns:
+    #   models/MFR — formula recognition (unimernet). Disabled in config.
+    #                Saves ~1-2 GB of download and disk.
+    print("=" * 60)
+    print("Downloading PDF-Extract-Kit-1.0 ...")
+    print("  (MFR/formula-recognition excluded — disabled in config)")
+    print("=" * 60)
+    snapshot_download(
+        repo_id="opendatalab/PDF-Extract-Kit-1.0",
+        local_dir=EXTRACT_KIT_DIR,
+        ignore_patterns=[
+            "*.git*",
+            ".gitattributes",
+            "models/MFR*",
+            "models/MFR/*",
+        ],
+    )
+    print(f"  → {EXTRACT_KIT_DIR}")
+
+    # ── layoutreader (optional) ───────────────────────────────────────────────
+    # Improves reading-order accuracy. If unavailable, MinerU uses fallback.
+    layoutreader_dir = ""
+    print("=" * 60)
+    print("Downloading layoutreader (optional, improves reading order) ...")
+    print("=" * 60)
+    try:
+        snapshot_download(
+            repo_id="hantian/layoutreader",
+            local_dir=LAYOUTREADER_DIR,
+            ignore_patterns=["*.git*", ".gitattributes"],
+        )
+        layoutreader_dir = LAYOUTREADER_DIR
+        print(f"  → {LAYOUTREADER_DIR}")
+    except Exception as exc:
+        print(f"  WARNING: layoutreader download failed ({exc})")
+        print("  Continuing without layoutreader — MinerU will use fallback ordering.")
+
+    # ── Write config ──────────────────────────────────────────────────────────
+    _write_config(layoutreader_dir)
+
+    print("\n✓ Model setup complete.")
+    print(f"  models-dir          : {os.path.join(EXTRACT_KIT_DIR, 'models')}")
+    print(f"  layoutreader-dir    : {layoutreader_dir or '(not available)'}")
+    print(f"  device-mode         : cpu")
+    print(f"  formula recognition : disabled (MFR models excluded)")
+    print(f"  table recognition   : enabled")
+
+
+if __name__ == "__main__":
+    download()

entrypoint.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────────────────────
+# MinerU OCR Service — container entrypoint
+#
+# Sequence:
+#   1. Restore magic-pdf.json if wiped (HF container restart)
+#   2. Run validate.py (pre-flight dependency check)
+#      → exits 1 and kills container if any critical dep is missing
+#      → this surfaces a clear error in HF logs instead of a silent bad start
+#   3. Start uvicorn (single worker — CPU Basic, no RAM contention)
+# ─────────────────────────────────────────────────────────────────────────────
+set -e
+
+CONFIG_FILE="${HOME}/magic-pdf.json"
+
+# ── 1. Restore config if /root was wiped (HF container restart) ───────────────
+if [ ! -f "$CONFIG_FILE" ]; then
+    echo "[entrypoint] magic-pdf.json missing — restoring from baked copy..."
+    cp /app/config/magic-pdf.json "$CONFIG_FILE"
+fi
+
+echo "[entrypoint] Config   : $CONFIG_FILE"
+echo "[entrypoint] Port     : ${PORT:-7860}"
+
+# ── 2. Pre-flight validation ──────────────────────────────────────────────────
+# validate.py exits 0 on pass, exits 1 on any CRITICAL failure.
+# 'set -e' above ensures a non-zero exit from validate.py aborts this script,
+# preventing a broken uvicorn from starting and appearing healthy.
+echo "[entrypoint] Running pre-flight validation..."
+python /app/validate.py
+echo "[entrypoint] Validation passed."
+
+# ── 3. Start API server ───────────────────────────────────────────────────────
+echo "[entrypoint] Starting uvicorn on 0.0.0.0:${PORT:-7860}..."
+exec uvicorn main:app \
+    --host 0.0.0.0 \
+    --port "${PORT:-7860}" \
+    --workers 1 \
+    --timeout-keep-alive 120 \
+    --log-level info

main.py

@@ -0,0 +1,885 @@
+"""
+MinerU OCR & Document Extraction Service
+FastAPI application for Hugging Face Docker Space (CPU / pipeline backend)
+
+Correct imports for magic-pdf >= 1.0.x (magic_pdf module):
+
+    from magic_pdf.data.data_reader_writer import FileBasedDataReader, FileBasedDataWriter
+    from magic_pdf.data.dataset import PymuDocDataset, ImageDataset
+    from magic_pdf.model.doc_analyze_by_custom_model import doc_analyze
+    from magic_pdf.config.enums import SupportedPdfParseMethod
+
+OBSOLETE imports (removed in magic-pdf >= 1.0, do not use):
+    from magic_pdf.pipe.UNIPipe import UNIPipe              ← removed
+    from magic_pdf.rw.DiskReaderWriter import ...           ← removed
+    from magic_pdf.data.read_api import read_local_images   ← NOT used here;
+        function expects a single string path in 1.x but crashes with
+        "stat: ... not list" if given a list. Use FileBasedDataReader instead.
+
+Removed features vs original:
+    - DOCX / PPTX / XLSX (required LibreOffice; caused build OOM/timeout)
+    - subprocess (was only used for LibreOffice conversion)
+    - python-magic / libmagic (listed in requirements but never imported)
+
+Endpoints:
+    GET  /health   — liveness (always fast, no dependency check)
+    GET  /status   — full node status including memory (via cgroups), uptime,
+                     cache, active requests, lastModelLoadMs
+    POST /extract  — single file (PDF or image) with SHA256 cache + memory guard
+    POST /batch    — up to BATCH_MAX_FILES files; extras silently ignored
+
+Structured error format (all non-2xx responses from /extract and /batch):
+    {
+        "success":   false,
+        "stage":     "upload" | "validation" | "decode" | "ocr" | "markdown" | "unknown",
+        "errorCode": "UNSUPPORTED_TYPE" | "FILE_TOO_LARGE" | "EMPTY_FILE" |
+                     "LOW_MEMORY" | "IMAGE_DECODE_FAILED" | "OCR_PIPELINE_FAILED" |
+                     "MARKDOWN_FAILED" | "INTERNAL_ERROR",
+        "message":   "<human-readable detail>"
+    }
+"""
+
+import hashlib
+import io
+import os
+import re
+import shutil
+import sys
+import tempfile
+import threading
+import time
+import traceback
+import logging
+from importlib.metadata import version as pkg_version
+from typing import Any
+
+import fitz  # PyMuPDF — bundled with magic-pdf[full-cpu]
+from PIL import Image
+
+from fastapi import FastAPI, File, UploadFile, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+
+# ── Logging ───────────────────────────────────────────────────────────────────
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s  %(levelname)-8s  %(name)s  %(message)s",
+)
+logger = logging.getLogger("mineru-service")
+
+# ── Start time ────────────────────────────────────────────────────────────────
+_START_TIME: float = time.time()
+
+# ── Upload / batch constants ──────────────────────────────────────────────────
+MAX_UPLOAD_BYTES = 30 * 1024 * 1024  # 30 MB
+BATCH_MAX_FILES = 8
+
+# ── Supported file types ──────────────────────────────────────────────────────
+PDF_EXTENSIONS = {"pdf"}
+
+# Natively supported by ImageDataset via FileBasedDataReader
+NATIVE_IMAGE_EXTENSIONS = {"jpg", "jpeg", "png"}
+
+# Need Pillow conversion to PNG before feeding to MinerU
+PILLOW_IMAGE_EXTENSIONS = {"webp", "bmp", "tiff", "tif", "gif", "heic", "heif", "avif"}
+
+IMAGE_EXTENSIONS = NATIVE_IMAGE_EXTENSIONS | PILLOW_IMAGE_EXTENSIONS
+ALLOWED_EXTENSIONS = PDF_EXTENSIONS | IMAGE_EXTENSIONS
+
+# ── Memory safety thresholds ──────────────────────────────────────────────────
+BYTES_PER_OCR_PAGE = 100 * 1024 * 1024  # ~100 MB / page (conservative)
+IMAGE_MEMORY_FACTOR = 4                  # decoded pixels × 4 for pipeline buffers
+MEM_SAFETY_FLOOR_MB = 1024              # always keep 1 GB free
+
+# ── SHA256 extraction cache (in-process, bounded by available RAM) ────────────
+_cache: dict[str, dict[str, Any]] = {}
+_cache_lock = threading.Lock()
+
+# ── Active-request counter ────────────────────────────────────────────────────
+_active_requests: int = 0
+_active_lock = threading.Lock()
+
+# ── Model load timing ─────────────────────────────────────────────────────────
+_model_load_ms: int = 0
+
+# ── Startup self-test results (populated by startup handler) ──────────────────
+_startup_issues: list[str] = []
+_startup_done: bool = False
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Structured error exception
+# ─────────────────────────────────────────────────────────────────────────────
+class ExtractionError(Exception):
+    """
+    Raised anywhere in the extraction pipeline to produce a structured
+    JSON error response with stage + errorCode instead of a generic 500.
+    """
+    def __init__(
+        self,
+        stage: str,
+        code: str,
+        message: str,
+        http_status: int = 422,
+    ) -> None:
+        self.stage = stage
+        self.code = code
+        self.message = message
+        self.http_status = http_status
+        super().__init__(message)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "success": False,
+            "stage": self.stage,
+            "errorCode": self.code,
+            "message": self.message,
+        }
+
+
+def _err(stage: str, code: str, msg: str, status: int = 422) -> ExtractionError:
+    """Shorthand constructor."""
+    return ExtractionError(stage, code, msg, status)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Active request helpers
+# ─────────────────────────────────────────────────────────────────────────────
+def _inc_active() -> None:
+    global _active_requests
+    with _active_lock:
+        _active_requests += 1
+
+
+def _dec_active() -> None:
+    global _active_requests
+    with _active_lock:
+        _active_requests = max(0, _active_requests - 1)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Pipeline readiness (lazy, first-request check)
+# ─────────────────────────────────────────────────────────────────────────────
+_pipeline_ready: bool = False
+_pipeline_lock = threading.Lock()
+
+
+def _ensure_pipeline() -> None:
+    """
+    Verify MinerU is importable and its config is present.
+    Sets _pipeline_ready on first success; raises ExtractionError on failure.
+    Checks are done under a lock so concurrent first-requests don't race.
+    """
+    global _pipeline_ready, _model_load_ms
+    if _pipeline_ready:
+        return
+
+    with _pipeline_lock:
+        if _pipeline_ready:  # double-checked locking
+            return
+
+        config_path = os.path.expanduser("~/magic-pdf.json")
+        if not os.path.exists(config_path):
+            raise _err(
+                "model_load", "CONFIG_MISSING",
+                f"magic-pdf.json not found at {config_path}. "
+                "Check Docker build log for download_models.py output.",
+                503,
+            )
+
+        # Trigger a lightweight import to confirm the package is usable.
+        t0 = time.perf_counter()
+        try:
+            from magic_pdf.data.dataset import PymuDocDataset, ImageDataset  # noqa: F401
+            from magic_pdf.data.data_reader_writer import (  # noqa: F401
+                FileBasedDataReader, FileBasedDataWriter,
+            )
+        except ImportError as exc:
+            raise _err(
+                "model_load", "IMPORT_FAILED",
+                f"magic_pdf not importable: {exc}. Check Dockerfile pip layers.",
+                503,
+            ) from exc
+
+        _model_load_ms = int((time.perf_counter() - t0) * 1000)
+        _pipeline_ready = True
+        logger.info("Pipeline ready (import check: %d ms).", _model_load_ms)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# FastAPI app
+# ─────────────────────────────────────────────────────────────────────────────
+app = FastAPI(
+    title="MinerU OCR Service",
+    description=(
+        "OCR and document extraction via MinerU pipeline backend (CPU). "
+        "Supports PDF and image files up to 30 MB."
+    ),
+    version="1.1.0",
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["GET", "POST"],
+    allow_headers=["*"],
+)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Startup self-test
+# ─────────────────────────────────────────────────────────────────────────────
+@app.on_event("startup")
+async def startup_self_test() -> None:
+    """
+    Run at container startup. Verifies all critical dependencies are present.
+    Never crashes the server — issues are stored in _startup_issues and
+    surfaced via GET /status so operators can diagnose without SSH access.
+    """
+    global _startup_done
+    issues: list[str] = []
+
+    # 1. cv2 — most common missing dependency
+    try:
+        import cv2  # noqa: F401
+        logger.info("startup ✓  cv2 available (version %s)", cv2.__version__)
+    except ImportError as exc:
+        msg = (
+            f"cv2 not importable: {exc}. "
+            "Add 'opencv-python-headless>=4.8.0' to pip layer 1 in Dockerfile."
+        )
+        issues.append(msg)
+        logger.critical("startup FAIL  %s", msg)
+
+    # 2. magic_pdf core imports
+    try:
+        from magic_pdf.data.dataset import PymuDocDataset, ImageDataset  # noqa: F401
+        from magic_pdf.data.data_reader_writer import (  # noqa: F401
+            FileBasedDataReader, FileBasedDataWriter,
+        )
+        from magic_pdf.model.doc_analyze_by_custom_model import doc_analyze  # noqa: F401
+        from magic_pdf.config.enums import SupportedPdfParseMethod  # noqa: F401
+        logger.info("startup ✓  magic_pdf imports OK")
+    except ImportError as exc:
+        msg = f"magic_pdf not importable: {exc}"
+        issues.append(msg)
+        logger.critical("startup FAIL  %s", msg)
+
+    # 3. MinerU config
+    config_path = os.path.expanduser("~/magic-pdf.json")
+    if os.path.exists(config_path):
+        logger.info("startup ✓  magic-pdf.json found at %s", config_path)
+    else:
+        msg = f"magic-pdf.json missing at {config_path} — run download_models.py"
+        issues.append(msg)
+        logger.critical("startup FAIL  %s", msg)
+
+    # 4. Model files
+    models_dir = "/app/models/PDF-Extract-Kit-1.0/models"
+    if os.path.isdir(models_dir):
+        logger.info("startup ✓  models directory found at %s", models_dir)
+    else:
+        msg = f"Models directory missing at {models_dir} — run download_models.py"
+        issues.append(msg)
+        logger.critical("startup FAIL  %s", msg)
+
+    # 5. Temp storage writable
+    try:
+        td = tempfile.mkdtemp(prefix="mineru_selftest_")
+        shutil.rmtree(td)
+        logger.info("startup ✓  temp storage writable")
+    except Exception as exc:
+        msg = f"Temp storage not writable: {exc}"
+        issues.append(msg)
+        logger.critical("startup FAIL  %s", msg)
+
+    _startup_issues.extend(issues)
+    _startup_done = True
+
+    if not issues:
+        logger.info("=" * 60)
+        logger.info("Startup self-test PASSED — service ready.")
+        logger.info("=" * 60)
+    else:
+        logger.error("=" * 60)
+        logger.error("Startup self-test FAILED — %d issue(s). See above.", len(issues))
+        logger.error("Service will start but /extract will fail until fixed.")
+        logger.error("=" * 60)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# GET /health
+# ─────────────────────────────────────────────────────────────────────────────
+@app.get("/health")
+def health() -> dict[str, Any]:
+    """
+    Liveness probe. Always returns 200 so HF Space marks the container as
+    running. Use GET /status to check whether the OCR pipeline is ready.
+    """
+    return {"status": "healthy"}
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# GET /status
+# ─────────────────────────────────────────────────────────────────────────────
+@app.get("/status")
+def status() -> dict[str, Any]:
+    """
+    Full readiness report. Memory is read from cgroups (not /proc/meminfo)
+    so the container's actual allocation is reported — not the host's RAM.
+    /proc/meminfo inside a Docker container on HF shows the host machine's
+    RAM (e.g. 123 GB) which is misleading. Cgroups v2 → v1 → /proc fallback.
+    """
+    used_mb, total_mb = _mem_mb()
+    return {
+        "status": "healthy" if not _startup_issues else "degraded",
+        "provider": "mineru",
+        "version": _mineru_version(),
+        "modelsLoaded": _pipeline_ready,
+        "startupIssues": _startup_issues,
+        "uptimeSeconds": int(time.time() - _START_TIME),
+        "memoryUsedMB": used_mb,
+        "memoryTotalMB": total_mb,
+        "activeRequests": _active_requests,
+        "cacheEntries": len(_cache),
+        "lastModelLoadMs": _model_load_ms,
+    }
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /extract — single file
+# ─────────────────────────────────────────────────────────────────────────────
+@app.post("/extract")
+async def extract(file: UploadFile = File(...)) -> JSONResponse:
+    try:
+        _ensure_pipeline()
+        raw, filename, ext = await _read_upload(file)
+        result = _run_extraction(raw, filename, ext)
+        return JSONResponse(content=result)
+    except ExtractionError as exc:
+        logger.warning(
+            "/extract structured error [%s/%s]: %s",
+            exc.stage, exc.code, exc.message,
+        )
+        return JSONResponse(status_code=exc.http_status, content=exc.to_dict())
+    except HTTPException:
+        raise
+    except Exception as exc:
+        logger.exception("/extract unhandled error")
+        return JSONResponse(
+            status_code=500,
+            content={
+                "success": False,
+                "stage": "unknown",
+                "errorCode": "INTERNAL_ERROR",
+                "message": str(exc),
+                "traceback": traceback.format_exc()[-2000:],
+            },
+        )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /batch — up to 8 files; extras silently ignored
+# ─────────────────────────────────────────────────────────────────────────────
+@app.post("/batch")
+async def batch(files: list[UploadFile] = File(...)) -> JSONResponse:
+    """
+    Policy:
+    - 1–8 files  → process all.
+    - > 8 files  → silently process only files[0:8].
+    Sequential processing to stay within CPU Basic memory limits.
+    Per-file failures use the structured error format; one failure never
+    aborts the rest of the batch.
+    """
+    try:
+        _ensure_pipeline()
+    except ExtractionError as exc:
+        return JSONResponse(status_code=exc.http_status, content=exc.to_dict())
+
+    candidates = files[:BATCH_MAX_FILES]
+    results: list[dict[str, Any]] = []
+
+    for upload in candidates:
+        try:
+            raw, filename, ext = await _read_upload(upload)
+            result = _run_extraction(raw, filename, ext)
+        except ExtractionError as exc:
+            result = exc.to_dict()
+            result["filename"] = _sanitize_filename(upload.filename or "upload")
+        except Exception as exc:
+            fname = _sanitize_filename(upload.filename or "upload")
+            logger.exception("Batch item failed: %s", fname)
+            result = {
+                "success": False,
+                "filename": fname,
+                "stage": "unknown",
+                "errorCode": "INTERNAL_ERROR",
+                "message": str(exc),
+            }
+        results.append(result)
+
+    return JSONResponse(content={
+        "success": True,
+        "processed": len(results),
+        "results": results,
+    })
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Upload reader (shared by /extract and /batch)
+# ─────────────────────────────────────────────────────────────────────────────
+async def _read_upload(upload: UploadFile) -> tuple[bytes, str, str]:
+    """Validate and read one upload. Returns (raw_bytes, filename, ext)."""
+    filename = _sanitize_filename(upload.filename or "upload")
+    ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
+
+    if ext not in ALLOWED_EXTENSIONS:
+        raise _err(
+            "validation", "UNSUPPORTED_TYPE",
+            f"Unsupported file type '.{ext}'. "
+            f"Supported: {sorted(ALLOWED_EXTENSIONS)}",
+            415,
+        )
+
+    raw = await upload.read(MAX_UPLOAD_BYTES + 1)
+
+    if len(raw) > MAX_UPLOAD_BYTES:
+        raise _err(
+            "upload", "FILE_TOO_LARGE",
+            f"'{filename}' exceeds the {MAX_UPLOAD_BYTES // 1024 // 1024} MB limit.",
+            413,
+        )
+    if len(raw) == 0:
+        raise _err("upload", "EMPTY_FILE", f"'{filename}' is empty.", 400)
+
+    return raw, filename, ext
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Extraction dispatcher (shared by /extract and /batch)
+# ─────────────────────────────────────────────────────────────────────────────
+def _run_extraction(raw: bytes, filename: str, ext: str) -> dict[str, Any]:
+    """
+    1. SHA256 cache lookup  → return immediately on hit (cached: true)
+    2. Memory safety guard  → raise ExtractionError(LOW_MEMORY) if OOM likely
+    3. Dispatch to PDF or image processor
+    4. Cache successful result
+    5. Return with timing metadata
+    """
+    # ── SHA256 cache ──────────────────────────────────────────────────────────
+    file_hash = hashlib.sha256(raw).hexdigest()
+    with _cache_lock:
+        cached = _cache.get(file_hash)
+    if cached is not None:
+        logger.info("Cache hit  %s  sha256=%.12s…", filename, file_hash)
+        result = {**cached}
+        result["metadata"] = {**cached["metadata"], "cached": True, "processingTimeMs": 0}
+        return result
+
+    # ── Memory safety guard ───────────────────────────────────────────────────
+    _assert_memory_safe(raw, ext)
+
+    # ── Process ───────────────────────────────────────────────────────────────
+    _inc_active()
+    work_dir = tempfile.mkdtemp(prefix="mineru_")
+    try:
+        t0 = time.perf_counter()
+
+        if ext in PDF_EXTENSIONS:
+            result = _process_pdf(raw, filename, work_dir)
+        else:
+            result = _process_image(raw, filename, ext, work_dir)
+
+        elapsed_ms = int((time.perf_counter() - t0) * 1000)
+        result["metadata"] = {
+            **result["metadata"],
+            "processingTimeMs": elapsed_ms,
+            "cached": False,
+        }
+
+        # Store without timing so cache entries stay lean
+        entry = {**result, "metadata": {k: v for k, v in result["metadata"].items()
+                                         if k not in ("processingTimeMs", "cached")}}
+        with _cache_lock:
+            _cache[file_hash] = entry
+
+        return result
+
+    except ExtractionError:
+        raise
+    except Exception as exc:
+        logger.exception("Extraction failed for %s", filename)
+        raise _err(
+            "unknown", "INTERNAL_ERROR",
+            f"Unexpected error during extraction: {exc}",
+            500,
+        ) from exc
+    finally:
+        _dec_active()
+        shutil.rmtree(work_dir, ignore_errors=True)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Memory safety guard
+# ─────────────────────────────────────────────────────────────────────────────
+def _assert_memory_safe(raw: bytes, ext: str) -> None:
+    """
+    Estimate peak memory the pipeline needs and reject with LOW_MEMORY if
+    available would drop below MEM_SAFETY_FLOOR_MB.
+    """
+    used_mb, total_mb = _mem_mb()
+    if total_mb == 0:
+        return  # cgroups and /proc both unavailable — skip guard
+
+    available_mb = total_mb - used_mb
+
+    if ext in PDF_EXTENSIONS:
+        page_count = max(1, _pdf_page_count(raw))
+        estimated_mb = (page_count * BYTES_PER_OCR_PAGE) // (1024 * 1024)
+    else:
+        estimated_mb = _image_memory_estimate(raw, ext) // (1024 * 1024)
+
+    free_after = available_mb - estimated_mb
+    logger.info(
+        "Memory check: avail=%d MB  est=%d MB  free_after=%d MB",
+        available_mb, estimated_mb, free_after,
+    )
+
+    if free_after < MEM_SAFETY_FLOOR_MB:
+        raise _err(
+            "validation", "LOW_MEMORY",
+            f"Insufficient memory. "
+            f"Available: {available_mb} MB, "
+            f"Estimated needed: {estimated_mb} MB, "
+            f"Safety floor: {MEM_SAFETY_FLOOR_MB} MB. "
+            "Try a smaller file or wait for active requests to complete.",
+            507,
+        )
+
+
+def _image_memory_estimate(raw: bytes, ext: str) -> int:
+    try:
+        if ext in {"heic", "heif"}:
+            try:
+                from pillow_heif import register_heif_opener
+                register_heif_opener()
+            except ImportError:
+                pass
+        img = Image.open(io.BytesIO(raw))
+        w, h = img.size
+        channels = len(img.getbands()) or 3
+        img.close()
+        return w * h * channels * IMAGE_MEMORY_FACTOR
+    except Exception:
+        return len(raw) * 20  # conservative fallback
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# PDF processor
+# ─────────────────────────────────────────────────────────────────────────────
+def _process_pdf(raw: bytes, filename: str, work_dir: str) -> dict[str, Any]:
+    from magic_pdf.data.data_reader_writer import FileBasedDataWriter
+    from magic_pdf.data.dataset import PymuDocDataset
+    from magic_pdf.model.doc_analyze_by_custom_model import doc_analyze
+    from magic_pdf.config.enums import SupportedPdfParseMethod
+
+    images_dir = os.path.join(work_dir, "images")
+    os.makedirs(images_dir, exist_ok=True)
+
+    try:
+        image_writer = FileBasedDataWriter(images_dir)
+    except Exception as exc:
+        raise _err("decode", "PDF_WRITER_FAILED", f"Could not create image writer: {exc}") from exc
+
+    page_count = _pdf_page_count(raw)
+
+    try:
+        ds = PymuDocDataset(raw)
+        method = ds.classify()
+    except Exception as exc:
+        raise _err("decode", "PDF_PARSE_FAILED", f"Could not parse PDF: {exc}") from exc
+
+    try:
+        if method == SupportedPdfParseMethod.TXT:
+            infer_result = ds.apply(doc_analyze, ocr=False)
+            pipe_result = infer_result.pipe_txt_mode(image_writer)
+            parse_method = "txt"
+            confidence = 0.95
+        else:
+            infer_result = ds.apply(doc_analyze, ocr=True)
+            pipe_result = infer_result.pipe_ocr_mode(image_writer)
+            parse_method = "ocr"
+            confidence = 0.85
+    except Exception as exc:
+        raise _err("ocr", "OCR_PIPELINE_FAILED", f"doc_analyze/pipe failed: {exc}") from exc
+
+    try:
+        markdown = pipe_result.get_markdown(images_dir)
+    except Exception as exc:
+        raise _err("markdown", "MARKDOWN_FAILED", f"get_markdown failed: {exc}") from exc
+
+    content_list = _safe_content_list(pipe_result, images_dir)
+    doc_type = _classify_document(markdown, filename)
+
+    return {
+        "success": True,
+        "filename": filename,
+        "docType": doc_type,
+        "pageCount": page_count,
+        "confidence": confidence,
+        "markdown": markdown,
+        "metadata": {
+            "parseMethod": parse_method,
+            "backend": "pipeline",
+            "docTypeClassification": doc_type,
+            "imageCount": _count_images(content_list),
+            "tableCount": _count_tables(content_list),
+            "formulaCount": _count_formulas(content_list),
+        },
+    }
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Image processor
+# ─────────────────────────────────────────────────────────────────────────────
+def _process_image(raw: bytes, filename: str, ext: str, work_dir: str) -> dict[str, Any]:
+    """
+    NOTE: read_local_images() is intentionally NOT used here.
+    In magic-pdf 1.x it expects a single path string; passing a list causes:
+        "stat: path should be string, bytes, os.PathLike or integer, not list"
+    We use FileBasedDataReader + ImageDataset directly — explicit and safe.
+    """
+    from magic_pdf.data.data_reader_writer import FileBasedDataReader, FileBasedDataWriter
+    from magic_pdf.data.dataset import ImageDataset
+    from magic_pdf.model.doc_analyze_by_custom_model import doc_analyze
+
+    images_dir = os.path.join(work_dir, "images")
+    os.makedirs(images_dir, exist_ok=True)
+
+    try:
+        image_writer = FileBasedDataWriter(images_dir)
+    except Exception as exc:
+        raise _err("decode", "IMAGE_WRITER_FAILED", f"Could not create image writer: {exc}") from exc
+
+    # Convert non-native formats to PNG before feeding to MinerU
+    try:
+        if ext in PILLOW_IMAGE_EXTENSIONS:
+            raw = _convert_image_to_png(raw, ext)
+            save_ext = "png"
+        else:
+            save_ext = ext
+    except ExtractionError:
+        raise
+    except Exception as exc:
+        raise _err("decode", "IMAGE_DECODE_FAILED", f"Could not decode image: {exc}") from exc
+
+    img_filename = f"input.{save_ext}"
+    img_path = os.path.join(work_dir, img_filename)
+    try:
+        with open(img_path, "wb") as fh:
+            fh.write(raw)
+    except OSError as exc:
+        raise _err("decode", "WRITE_FAILED", f"Could not write temp image: {exc}") from exc
+
+    try:
+        # FileBasedDataReader(base_dir).read(relative_name) → bytes
+        reader = FileBasedDataReader(work_dir)
+        image_bytes = reader.read(img_filename)
+        ds = ImageDataset(image_bytes)
+    except Exception as exc:
+        raise _err("decode", "IMAGE_DATASET_FAILED",
+                   f"Could not build ImageDataset: {exc}") from exc
+
+    try:
+        infer_result = ds.apply(doc_analyze, ocr=True)
+        pipe_result = infer_result.pipe_ocr_mode(image_writer)
+    except Exception as exc:
+        raise _err("ocr", "OCR_PIPELINE_FAILED", f"doc_analyze/pipe failed: {exc}") from exc
+
+    try:
+        markdown = pipe_result.get_markdown(images_dir)
+    except Exception as exc:
+        raise _err("markdown", "MARKDOWN_FAILED", f"get_markdown failed: {exc}") from exc
+
+    content_list = _safe_content_list(pipe_result, images_dir)
+    doc_type = _classify_document(markdown, filename)
+
+    return {
+        "success": True,
+        "filename": filename,
+        "docType": doc_type,
+        "pageCount": 1,
+        "confidence": 0.85,
+        "markdown": markdown,
+        "metadata": {
+            "parseMethod": "ocr",
+            "backend": "pipeline",
+            "docTypeClassification": doc_type,
+            "imageCount": _count_images(content_list),
+            "tableCount": _count_tables(content_list),
+            "formulaCount": _count_formulas(content_list),
+        },
+    }
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Utility helpers
+# ─────────────────────────────────────────────────────────────────────────────
+def _sanitize_filename(name: str) -> str:
+    name = os.path.basename(name)
+    name = re.sub(r"[^\w.\-]", "_", name)
+    return name[:200] or "upload"
+
+
+def _pdf_page_count(raw: bytes) -> int:
+    try:
+        doc = fitz.open(stream=raw, filetype="pdf")
+        count = doc.page_count
+        doc.close()
+        return count
+    except Exception:
+        return 0
+
+
+def _convert_image_to_png(raw: bytes, ext: str) -> bytes:
+    if ext in {"heic", "heif"}:
+        try:
+            from pillow_heif import register_heif_opener
+            register_heif_opener()
+        except ImportError:
+            raise _err(
+                "decode", "HEIF_NOT_SUPPORTED",
+                "HEIC/HEIF support requires pillow-heif (not installed).",
+                415,
+            )
+    try:
+        img = Image.open(io.BytesIO(raw)).convert("RGB")
+        buf = io.BytesIO()
+        img.save(buf, format="PNG")
+        return buf.getvalue()
+    except Exception as exc:
+        raise _err("decode", "IMAGE_DECODE_FAILED", f"Pillow could not open image: {exc}") from exc
+
+
+def _safe_content_list(pipe_result: Any, images_dir: str) -> list[dict]:
+    try:
+        return pipe_result.get_content_list(images_dir) or []
+    except Exception:
+        return []
+
+
+def _count_images(content_list: list[dict]) -> int:
+    return sum(1 for item in content_list if item.get("type") == "image")
+
+
+def _count_tables(content_list: list[dict]) -> int:
+    return sum(1 for item in content_list if item.get("type") == "table")
+
+
+def _count_formulas(content_list: list[dict]) -> int:
+    return sum(
+        1 for item in content_list
+        if item.get("type") in {"equation", "formula", "interline_equation"}
+    )
+
+
+def _classify_document(markdown: str, filename: str) -> str:
+    """Keyword-based document type heuristic over extracted Markdown + filename."""
+    text = (markdown + " " + filename).lower()
+
+    rules: list[tuple[str, list[str]]] = [
+        ("invoice",        ["invoice", "bill to", "invoice number", "invoice #",
+                            "due date", "amount due", "subtotal", "tax invoice"]),
+        ("receipt",        ["receipt", "thank you for your purchase", "order total",
+                            "payment received", "transaction id", "cash receipt"]),
+        ("marksheet",      ["marksheet", "mark sheet", "grade sheet", "scorecard",
+                            "score card", "cgpa", "sgpa", "semester result",
+                            "result sheet", "marks obtained"]),
+        ("resume",         ["curriculum vitae", "cv", "resume", "work experience",
+                            "education", "skills", "references", "objective",
+                            "professional summary"]),
+        ("research paper", ["abstract", "introduction", "methodology", "conclusion",
+                            "references", "keywords", "doi:", "arxiv", "journal",
+                            "proceedings"]),
+        ("form",           ["please fill", "signature", "date of birth", "applicant",
+                            "application form", "form no", "checkbox", "tick", "field"]),
+        ("contract",       ["agreement", "hereby", "whereas", "terms and conditions",
+                            "party of the first", "signed by", "witnesseth",
+                            "indemnify", "governing law"]),
+        ("screenshot",     ["screenshot", "screen capture", "url:", "http://",
+                            "https://", "browser", "toolbar", "desktop"]),
+    ]
+
+    scores: dict[str, int] = {}
+    for doc_type, keywords in rules:
+        score = sum(1 for kw in keywords if kw in text)
+        if score:
+            scores[doc_type] = score
+
+    return max(scores, key=lambda k: scores[k]) if scores else "generic document"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Memory — cgroup-aware (fixes "105 GB / 123 GB" /proc/meminfo host bleed)
+# ─────────────────────────────────────────────────────────────────────────────
+def _mem_mb() -> tuple[int, int]:
+    """
+    Return (used_mb, total_mb) for the CONTAINER, not the host.
+
+    Priority:
+      1. cgroups v2  /sys/fs/cgroup/memory.max + memory.current
+      2. cgroups v1  /sys/fs/cgroup/memory/memory.limit_in_bytes + usage_in_bytes
+      3. /proc/meminfo fallback (may show host memory in Docker — known inaccuracy)
+
+    /proc/meminfo is last resort because HF Docker containers typically do NOT
+    have cgroup memory limits mapped into /proc, so it shows the physical host
+    RAM (e.g. 123 GB on a 128 GB bare-metal host), misleading the memory guard.
+    """
+    # ── cgroups v2 (preferred — modern Docker / HF Spaces) ───────────────────
+    try:
+        with open("/sys/fs/cgroup/memory.max") as f:
+            raw_max = f.read().strip()
+        if raw_max != "max":
+            limit_bytes = int(raw_max)
+            with open("/sys/fs/cgroup/memory.current") as f:
+                used_bytes = int(f.read().strip())
+            if limit_bytes > 0:
+                return used_bytes // (1024 * 1024), limit_bytes // (1024 * 1024)
+    except (FileNotFoundError, ValueError, OSError):
+        pass
+
+    # ── cgroups v1 ────────────────────────────────────────────────────────────
+    try:
+        with open("/sys/fs/cgroup/memory/memory.limit_in_bytes") as f:
+            limit_bytes = int(f.read().strip())
+        with open("/sys/fs/cgroup/memory/memory.usage_in_bytes") as f:
+            used_bytes = int(f.read().strip())
+        # Unconstrained cgroup reports a sentinel > 1 PB; skip it
+        if limit_bytes < 128 * 1024 * 1024 * 1024:
+            return used_bytes // (1024 * 1024), limit_bytes // (1024 * 1024)
+    except (FileNotFoundError, ValueError, OSError):
+        pass
+
+    # ── /proc/meminfo fallback ────────────────────────────────────────────────
+    try:
+        info: dict[str, int] = {}
+        with open("/proc/meminfo") as f:
+            for line in f:
+                parts = line.split()
+                if len(parts) >= 2:
+                    info[parts[0].rstrip(":")] = int(parts[1])  # values are in kB
+        total_kb = info.get("MemTotal", 0)
+        avail_kb = info.get("MemAvailable", 0)
+        used_kb = total_kb - avail_kb
+        return used_kb // 1024, total_kb // 1024
+    except Exception:
+        return 0, 0
+
+
+def _mineru_version() -> str:
+    for pkg in ("magic-pdf", "mineru"):
+        try:
+            return pkg_version(pkg)
+        except Exception:
+            continue
+    return "unknown"

requirements.txt

@@ -0,0 +1,19 @@
+# Reference only — dependencies are installed directly in the Dockerfile
+# to allow two-layer pip caching (small packages / magic-pdf separately).
+#
+# ── Layer 1 — small + opencv-headless ────────────────────────────────────────
+fastapi>=0.115.0
+uvicorn[standard]>=0.32.0
+python-multipart>=0.0.12
+Pillow>=10.0.0
+pillow-heif>=0.18.0
+huggingface_hub>=0.25.0
+opencv-python-headless>=4.8.0   # MUST be layer-1 so magic-pdf sees cv2 as satisfied
+                                 # and does not pull in the full X11 opencv-python build
+#
+# ── Layer 2 — magic-pdf (requires --extra-index-url https://myhloli.github.io/wheels/) ──
+magic-pdf[full-cpu]==1.3.12
+#
+# ── Removed ───────────────────────────────────────────────────────────────────
+# python-magic  — was listed in requirements but never imported in main.py
+# libmagic1     — C dep of python-magic, also removed from system packages

validate.py

@@ -0,0 +1,315 @@
+#!/usr/bin/env python3
+"""
+Pre-flight validation script for MinerU OCR Service.
+
+Run by entrypoint.sh BEFORE uvicorn starts.
+Exits 0 if all checks pass.
+Exits 1 if any CRITICAL check fails — this crashes the container loudly
+so Hugging Face logs show an actionable error instead of a silent crash
+or a healthy-looking service that fails on every request.
+
+Usage:
+    python validate.py           # run all checks, exit 0/1
+    python validate.py --soft    # run all checks, always exit 0 (log only)
+"""
+
+import importlib
+import json
+import os
+import shutil
+import sys
+import tempfile
+import time
+import traceback
+
+SOFT_MODE = "--soft" in sys.argv  # never exit 1, just print
+
+MODELS_DIR = "/app/models"
+EXTRACT_KIT_MODELS = os.path.join(MODELS_DIR, "PDF-Extract-Kit-1.0", "models")
+LAYOUT_MARKER = os.path.join(EXTRACT_KIT_MODELS, "Layout")  # canary directory
+CONFIG_PATH = os.path.expanduser("~/magic-pdf.json")
+
+
+# ── helpers ────────────────────────────────────────────────────────────────────
+def ok(label: str, detail: str = "") -> None:
+    suffix = f"  ({detail})" if detail else ""
+    print(f"  ✓  {label}{suffix}", flush=True)
+
+
+def fail(label: str, detail: str, critical: bool = True) -> None:
+    tag = "CRITICAL" if critical else "WARNING"
+    print(f"  ✗  [{tag}] {label}: {detail}", flush=True)
+
+
+def section(title: str) -> None:
+    print(f"\n{'─' * 60}", flush=True)
+    print(f"  {title}", flush=True)
+    print(f"{'─' * 60}", flush=True)
+
+
+# ── check registry ─────────────────────────────────────────────────────────────
+failures: list[tuple[str, str]] = []   # (label, detail)
+warnings: list[tuple[str, str]] = []
+
+
+def record_fail(label: str, detail: str, critical: bool = True) -> None:
+    fail(label, detail, critical)
+    if critical:
+        failures.append((label, detail))
+    else:
+        warnings.append((label, detail))
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+print("\n" + "═" * 60, flush=True)
+print("  MinerU OCR Service — Pre-flight Validation", flush=True)
+print("═" * 60, flush=True)
+
+# ── 1. Python version ──────────────────────────────────────────────────────────
+section("1. Python runtime")
+pv = sys.version_info
+if pv >= (3, 10):
+    ok("Python version", f"{pv.major}.{pv.minor}.{pv.micro}")
+else:
+    record_fail("Python version",
+                f"{pv.major}.{pv.minor} detected — magic-pdf requires >= 3.10")
+
+# ── 2. cv2 ─────────────────────────────────────────────────────────────────────
+section("2. OpenCV (cv2)")
+try:
+    import cv2
+    ok("cv2 import", f"version {cv2.__version__}")
+
+    # Confirm headless (no X11 dep) by checking build info
+    build = cv2.getBuildInformation()
+    if "GTK" in build or "Qt" in build:
+        record_fail("cv2 build", "GUI backend detected — use opencv-python-headless",
+                    critical=False)
+    else:
+        ok("cv2 headless", "no GUI backend detected")
+except ImportError as exc:
+    record_fail(
+        "cv2 import",
+        f"{exc}. "
+        "Add 'opencv-python-headless>=4.8.0' to Dockerfile pip layer 1 "
+        "BEFORE magic-pdf install.",
+    )
+except Exception as exc:
+    record_fail("cv2 import", f"unexpected error: {exc}")
+
+# ── 3. magic_pdf core ──────────────────────────────────────────────────────────
+section("3. magic_pdf core imports")
+
+REQUIRED_IMPORTS = [
+    ("magic_pdf.data.dataset",                 ["PymuDocDataset", "ImageDataset"]),
+    ("magic_pdf.data.data_reader_writer",      ["FileBasedDataReader", "FileBasedDataWriter"]),
+    ("magic_pdf.model.doc_analyze_by_custom_model", ["doc_analyze"]),
+    ("magic_pdf.config.enums",                 ["SupportedPdfParseMethod"]),
+]
+
+for module_path, symbols in REQUIRED_IMPORTS:
+    try:
+        mod = importlib.import_module(module_path)
+        missing = [s for s in symbols if not hasattr(mod, s)]
+        if missing:
+            record_fail(f"{module_path}", f"missing symbols: {missing}")
+        else:
+            ok(module_path, ", ".join(symbols))
+    except ImportError as exc:
+        record_fail(module_path, str(exc))
+    except Exception as exc:
+        record_fail(module_path, f"unexpected: {exc}")
+
+# Confirm removed/deprecated imports are truly gone
+section("3b. Deprecated API check (should NOT exist)")
+OBSOLETE = [
+    "magic_pdf.pipe.UNIPipe",
+    "magic_pdf.rw.DiskReaderWriter",
+]
+for mod_path in OBSOLETE:
+    try:
+        importlib.import_module(mod_path)
+        record_fail(mod_path, "still importable — code may use old API", critical=False)
+    except ImportError:
+        ok(f"{mod_path} (correctly absent)")
+
+# ── 4. Config file ─────────────────────────────────────────────────────────────
+section("4. MinerU config (magic-pdf.json)")
+if os.path.exists(CONFIG_PATH):
+    try:
+        with open(CONFIG_PATH) as f:
+            cfg = json.load(f)
+        required_keys = ["models-dir", "device-mode"]
+        missing_keys = [k for k in required_keys if k not in cfg]
+        if missing_keys:
+            record_fail("Config keys", f"missing: {missing_keys}")
+        else:
+            ok("Config file", CONFIG_PATH)
+            ok("device-mode", cfg.get("device-mode", "?"))
+            ok("models-dir",  cfg.get("models-dir",  "?"))
+            ok("formula-enable", str(cfg.get("formula-config", {}).get("enable", "?")))
+            ok("table-enable",   str(cfg.get("table-config",   {}).get("enable", "?")))
+    except json.JSONDecodeError as exc:
+        record_fail("Config file", f"invalid JSON: {exc}")
+    except Exception as exc:
+        record_fail("Config file", str(exc))
+else:
+    record_fail(
+        "Config file",
+        f"not found at {CONFIG_PATH}. "
+        "Run download_models.py or check Docker build log.",
+    )
+
+# ── 5. Model files ─────────────────────────────────────────────────────────────
+section("5. Model files")
+
+model_checks = [
+    ("PDF-Extract-Kit-1.0 root",  os.path.join(MODELS_DIR, "PDF-Extract-Kit-1.0")),
+    ("Layout models (canary)",    LAYOUT_MARKER),
+    ("MFD models",                os.path.join(EXTRACT_KIT_MODELS, "MFD")),
+    ("Table models",              os.path.join(EXTRACT_KIT_MODELS, "TabRec")),
+]
+
+for label, path in model_checks:
+    if os.path.isdir(path):
+        # Count files for a sanity check
+        try:
+            n = sum(1 for _ in os.scandir(path))
+            ok(label, f"{path}  ({n} entries)")
+        except OSError:
+            ok(label, path)
+    else:
+        record_fail(label, f"directory not found: {path}")
+
+# layoutreader — optional
+lr_dir = os.path.join(MODELS_DIR, "layoutreader")
+if os.path.isdir(lr_dir):
+    ok("layoutreader (optional)", lr_dir)
+else:
+    record_fail("layoutreader (optional)",
+                "not found — MinerU will use fallback ordering (non-critical)",
+                critical=False)
+
+# Validate config models-dir points to existing path
+try:
+    with open(CONFIG_PATH) as f:
+        cfg = json.load(f)
+    cfg_models = cfg.get("models-dir", "")
+    if cfg_models and os.path.isdir(cfg_models):
+        ok("Config models-dir exists", cfg_models)
+    elif cfg_models:
+        record_fail("Config models-dir", f"points to missing path: {cfg_models}")
+except Exception:
+    pass  # already reported above
+
+# ── 6. Temp storage ────────────────────────────────────────────────────────────
+section("6. Temp storage")
+try:
+    td = tempfile.mkdtemp(prefix="mineru_validate_")
+    test_file = os.path.join(td, "write_test.bin")
+    with open(test_file, "wb") as f:
+        f.write(b"x" * 4096)
+    assert os.path.getsize(test_file) == 4096
+    shutil.rmtree(td)
+    ok("Temp write + delete", tempfile.gettempdir())
+except Exception as exc:
+    record_fail("Temp storage", str(exc))
+
+# ── 7. System memory ───────────────────────────────────────────────────────────
+section("7. System memory (cgroups)")
+mem_source = "unknown"
+total_mb = used_mb = 0
+
+try:
+    with open("/sys/fs/cgroup/memory.max") as f:
+        raw = f.read().strip()
+    if raw != "max":
+        total_mb = int(raw) // (1024 * 1024)
+        with open("/sys/fs/cgroup/memory.current") as f:
+            used_mb = int(f.read().strip()) // (1024 * 1024)
+        mem_source = "cgroups v2"
+except (FileNotFoundError, ValueError):
+    pass
+
+if total_mb == 0:
+    try:
+        with open("/sys/fs/cgroup/memory/memory.limit_in_bytes") as f:
+            limit = int(f.read().strip())
+        with open("/sys/fs/cgroup/memory/memory.usage_in_bytes") as f:
+            used_bytes = int(f.read().strip())
+        if limit < 128 * 1024 * 1024 * 1024:
+            total_mb = limit // (1024 * 1024)
+            used_mb  = used_bytes // (1024 * 1024)
+            mem_source = "cgroups v1"
+    except (FileNotFoundError, ValueError):
+        pass
+
+if total_mb == 0:
+    try:
+        info: dict[str, int] = {}
+        with open("/proc/meminfo") as f:
+            for line in f:
+                parts = line.split()
+                if len(parts) >= 2:
+                    info[parts[0].rstrip(":")] = int(parts[1])
+        total_mb = info.get("MemTotal", 0) // 1024
+        used_mb  = (info.get("MemTotal", 0) - info.get("MemAvailable", 0)) // 1024
+        mem_source = "/proc/meminfo (may show host RAM)"
+    except Exception:
+        pass
+
+ok("Memory source", mem_source)
+ok("Total memory",  f"{total_mb} MB")
+ok("Used memory",   f"{used_mb} MB")
+ok("Free memory",   f"{total_mb - used_mb} MB")
+
+if total_mb > 32 * 1024:
+    record_fail(
+        "Memory total",
+        f"{total_mb} MB seems too large for a container — "
+        "cgroups may not be available; /proc/meminfo is showing host RAM. "
+        "Memory guard in main.py will be conservative.",
+        critical=False,
+    )
+
+# ── 8. /proc/meminfo sanity ────────────────────────────────────────────────────
+section("8. /proc/meminfo (for reference)")
+try:
+    with open("/proc/meminfo") as f:
+        lines = f.readlines()[:5]
+    for line in lines:
+        parts = line.split()
+        if len(parts) >= 2:
+            kb = int(parts[1])
+            ok(parts[0].rstrip(":"), f"{kb // 1024} MB")
+except Exception as exc:
+    record_fail("/proc/meminfo", str(exc), critical=False)
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Summary
+# ═══════════════════════════════════════════════════════════════════════════════
+print("\n" + "═" * 60, flush=True)
+print("  Validation Summary", flush=True)
+print("═" * 60, flush=True)
+
+if warnings:
+    print(f"\n  ⚠  {len(warnings)} warning(s):", flush=True)
+    for label, detail in warnings:
+        print(f"     • {label}: {detail}", flush=True)
+
+if failures:
+    print(f"\n  ✗  {len(failures)} CRITICAL failure(s):", flush=True)
+    for label, detail in failures:
+        print(f"     • {label}: {detail}", flush=True)
+    print("\n  Service will NOT start until these are resolved.", flush=True)
+    print("  Check Dockerfile pip layers and Docker build log.", flush=True)
+    print("═" * 60 + "\n", flush=True)
+    if not SOFT_MODE:
+        sys.exit(1)
+else:
+    print(f"\n  ✓  All critical checks passed", flush=True)
+    if warnings:
+        print(f"  ⚠  {len(warnings)} non-critical warning(s) — see above", flush=True)
+    print("\n  Service is ready to start.", flush=True)
+    print("═" * 60 + "\n", flush=True)
+    sys.exit(0)