Patch extractor to use Mistral OCR and fix py3.14 deps

README.md

@@ -16,3 +16,15 @@ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-
 
 - Diagnostic audit trail: `smoke_signal/manifest/DIAGNOSTIC_AUDIT_TRAIL.md`
 - Bug report template: `smoke_signal/manifest/BUG_REPORT_TEMPLATE.md`
+
+## OCR Runtime (Mistral OCR)
+
+- PDF OCR is wired to Mistral OCR using model `mistral-ocr-latest`.
+- Add the Hugging Face Space secret:
+  - `MISTRAL_API_KEY`
+- The extractor sends uploaded PDFs to Mistral OCR as base64 `document_url` payloads and preserves page-order output for fingerprint metrics.
+
+## Fingerprint Data Boundary
+
+- Final author fingerprint records store derived numeric/categorical metrics only.
+- Source manuscript text is not stored in the final fingerprint library.

requirements.txt

@@ -1,5 +1,6 @@
 gradio>=4.44,<7
 huggingface_hub<1.0
+mistralai>=2.4.5
 nltk>=3.8
 openpyxl>=3.1
 openai>=1.30.0
@@ -8,7 +9,6 @@ pdfplumber>=0.10
 Pillow>=10.0.0
 plotly>=5.22
 pymupdf>=1.23
-pymupdf>=1.23.0
 pypdfium2>=4.30
 pytesseract>=0.3.10
-surya-ocr>=0.4.0
+surya-ocr>=0.4.0; python_version < "3.13"

src/codex_extractor.py

@@ -88,6 +88,7 @@ import os
 import re
 import string
 import zipfile
+from base64 import b64encode
 from collections import Counter
 from pathlib import Path
 from shutil import which
@@ -113,6 +114,18 @@ try:
 except Exception:
     PYTESSERACT_AVAILABLE = False
 
+try:
+    from mistralai import Mistral
+    MISTRAL_AVAILABLE = True
+except Exception:
+    try:
+        # Back-compat import path used by older SDK docs.
+        from mistralai.client import Mistral  # type: ignore
+        MISTRAL_AVAILABLE = True
+    except Exception:
+        Mistral = None  # type: ignore
+        MISTRAL_AVAILABLE = False
+
 try:
     import nltk
     import socket as _socket
@@ -299,6 +312,137 @@ def _word_count(text: str) -> int:
     return len(SIMPLE_TOKENISE_PATTERN.findall((text or "").lower()))
 
 
+def _mistral_ocr_ready() -> bool:
+    return MISTRAL_AVAILABLE and bool(os.getenv("MISTRAL_API_KEY"))
+
+
+def _mistral_client() -> Any:
+    api_key = os.getenv("MISTRAL_API_KEY", "").strip()
+    if not api_key:
+        raise RuntimeError(
+            "MISTRAL_API_KEY is missing. Add it as a Hugging Face Space secret."
+        )
+    if not MISTRAL_AVAILABLE:
+        raise RuntimeError(
+            "mistralai SDK is not installed. Add `mistralai` to requirements.txt."
+        )
+    return Mistral(api_key=api_key)
+
+
+def _mistral_page_markdown(page: Any) -> str:
+    if isinstance(page, dict):
+        return str(page.get("markdown") or "")
+    return str(getattr(page, "markdown", "") or "")
+
+
+def _mistral_page_index(page: Any, default_idx: int) -> int:
+    raw_idx: Any
+    if isinstance(page, dict):
+        raw_idx = page.get("index", default_idx)
+    else:
+        raw_idx = getattr(page, "index", default_idx)
+    try:
+        return int(raw_idx)
+    except Exception:
+        return default_idx
+
+
+def extract_text_from_pdf_mistral(
+    pdf_path: str | Path,
+    start_page: int | None = None,
+    end_page: int | None = None,
+) -> tuple[str, str, list[dict]]:
+    """
+    OCR extraction using Mistral OCR (`mistral-ocr-latest`).
+
+    Returns (story_text, raw_text, page_trace).
+    page_trace follows the same shape used by the rest of the pipeline.
+    """
+    client = _mistral_client()
+    max_pages = int(os.getenv("OCR_MAX_PAGES", "300"))
+    include_image_base64 = os.getenv("MISTRAL_OCR_INCLUDE_IMAGE_BASE64", "0").strip().lower() in {
+        "1", "true", "yes",
+    }
+
+    pdf_bytes = Path(pdf_path).read_bytes()
+    encoded_pdf = b64encode(pdf_bytes).decode("utf-8")
+    data_url = f"data:application/pdf;base64,{encoded_pdf}"
+
+    ocr_kwargs: dict[str, Any] = {
+        "model": "mistral-ocr-latest",
+        "document": {
+            "type": "document_url",
+            "document_url": data_url,
+        },
+        "include_image_base64": include_image_base64,
+    }
+
+    # If user selected page range, send it directly to OCR (0-based indices).
+    if start_page is not None or end_page is not None:
+        idx_start = max(0, (start_page - 1) if start_page is not None else 0)
+        idx_end = (end_page - 1) if end_page is not None else idx_start
+        idx_end = max(idx_start, idx_end)
+        ocr_kwargs["pages"] = list(range(idx_start, idx_end + 1))
+
+    ocr_response = client.ocr.process(**ocr_kwargs)
+    pages = getattr(ocr_response, "pages", None)
+    if pages is None and isinstance(ocr_response, dict):
+        pages = ocr_response.get("pages")
+    if not isinstance(pages, list):
+        raise RuntimeError("Mistral OCR response did not include a page list.")
+
+    if max_pages > 0:
+        pages = pages[:max_pages]
+
+    raw_parts: list[str] = []
+    story_parts: list[str] = []
+    page_trace: list[dict] = []
+
+    for local_idx, page in enumerate(pages):
+        page_idx = _mistral_page_index(page, local_idx)
+        page_text = _mistral_page_markdown(page)
+        raw_parts.append(page_text)
+        raw_wc = _word_count(page_text)
+
+        classification = "story"
+        skip_reason = ""
+        included = False
+
+        if start_page is not None or end_page is not None:
+            idx_start = max(0, (start_page - 1) if start_page is not None else 0)
+            idx_end = (end_page - 1) if end_page is not None else idx_start
+            idx_end = max(idx_start, idx_end)
+            if idx_start <= page_idx <= idx_end:
+                story_parts.append(page_text)
+                included = True
+            else:
+                classification = "out_of_range"
+                skip_reason = f"outside user range {start_page}–{end_page}"
+        else:
+            # Keep existing cover/front-matter behavior to preserve metric quality.
+            if len(pages) > 1 and local_idx == 0:
+                classification = "cover"
+                skip_reason = "first page auto-skipped as cover"
+            elif is_front_matter_page(page_text):
+                classification = "front_matter"
+                skip_reason = "front-matter signals detected"
+            else:
+                story_parts.append(page_text)
+                included = True
+
+        cleaned_wc = _word_count(page_text) if included else 0
+        page_trace.append({
+            "page_number": page_idx + 1,
+            "raw_word_count": raw_wc,
+            "cleaned_word_count": cleaned_wc,
+            "classification": classification,
+            "included": included,
+            "skip_reason": skip_reason,
+        })
+
+    return "\n".join(story_parts), "\n".join(raw_parts), page_trace
+
+
 def _ocr_runtime_ready() -> bool:
     return PDFIUM_AVAILABLE and PYTESSERACT_AVAILABLE and which("tesseract") is not None
 
@@ -447,20 +591,50 @@ def extract_text_from_pdf(
 
     raw_text = "\n".join(raw_parts)
     story_text = "\n".join(story_parts)
+    story_wc = _word_count(story_text)
+    mistral_first = os.getenv("OCR_USE_MISTRAL", "1").strip().lower() not in {"0", "false", "no"}
+    mistral_min_words = int(os.getenv("MISTRAL_OCR_MIN_WORDS", "20"))
 
-    if _word_count(story_text) >= 20:
+    if mistral_first:
+        try:
+            mistral_story_text, mistral_raw_text, mistral_trace = extract_text_from_pdf_mistral(
+                pdf_path,
+                start_page=start_page,
+                end_page=end_page,
+            )
+            mistral_wc = _word_count(mistral_story_text)
+            if mistral_wc >= max(mistral_min_words, story_wc):
+                return mistral_story_text, mistral_raw_text, mistral_trace
+        except Exception:
+            # Keep extraction resilient if API key is absent or OCR request fails.
+            pass
+
+    if story_wc >= 20:
         return story_text, raw_text, page_trace
 
+    # If not already attempted as first-pass, try Mistral OCR before legacy OCR fallback.
+    if not mistral_first:
+        try:
+            mistral_story_text, mistral_raw_text, mistral_trace = extract_text_from_pdf_mistral(
+                pdf_path,
+                start_page=start_page,
+                end_page=end_page,
+            )
+            if _word_count(mistral_story_text) >= max(mistral_min_words, story_wc):
+                return mistral_story_text, mistral_raw_text, mistral_trace
+        except Exception:
+            pass
+
+    # Legacy local OCR fallback (kept for resilience if Mistral is unavailable).
     try:
         ocr_story_text, ocr_raw_text, ocr_trace = extract_text_from_pdf_ocr(
             pdf_path, start_page=start_page, end_page=end_page,
         )
-        if _word_count(ocr_story_text) >= max(20, _word_count(story_text)):
-            # Mark all original pages as ocr_fallback and merge OCR trace
+        if _word_count(ocr_story_text) >= max(20, story_wc):
             for entry in page_trace:
                 if entry["classification"] == "story":
                     entry["classification"] = "ocr_fallback"
-                    entry["skip_reason"] = "native text layer empty; OCR used"
+                    entry["skip_reason"] = "native text layer empty; legacy OCR used"
             return ocr_story_text, ocr_raw_text, ocr_trace
     except Exception:
         pass
@@ -1847,11 +2021,11 @@ def process_upload(
         )
 
         if not story_text or len(story_text.split()) < 20:
-            if str(file_path).lower().endswith(".pdf") and not _ocr_runtime_ready():
+            if str(file_path).lower().endswith(".pdf") and not _mistral_ocr_ready() and not _ocr_runtime_ready():
                 return (
                     "ERROR: No usable text extracted from file and OCR runtime is unavailable. "
-                    "Install OCR dependencies (`pypdfium2`, `pytesseract`) and system package "
-                    "`tesseract-ocr` in the Space build.",
+                    "Set Hugging Face secret `MISTRAL_API_KEY` for Mistral OCR (`mistral-ocr-latest`) "
+                    "or install legacy OCR dependencies (`pypdfium2`, `pytesseract`, `tesseract-ocr`).",
                     {},
                 )
             return (