Merge pull request #4 from r0m4k/feat/offline-backend-and-ui

.gitignore

@@ -12,11 +12,13 @@ train/data/
 eval/data/synth*
 runs/
 
-# Model artifacts (bundle the final GGUF via git-lfs deliberately, not by accident)
+# Model artifacts — never commit downloaded/large weights by accident.
 *.safetensors
 *.bin
+*.gguf
 adapters/
-# NOTE: the shipped GGUF + mmproj are tracked via .gitattributes (git-lfs), not ignored.
+models/
+# NOTE: if you ever ship a GGUF inside the repo, track it deliberately via git-lfs.
 
 # OS / editor
 .DS_Store

app.py

@@ -12,6 +12,11 @@ from src.extraction import build_extractor
 
 load_local_env()
 
+# The hosted-API key field is only relevant when the API backend is active; in offline
+# (local) modes it's dead weight, so we only show it when EXTRACTOR_BACKEND == "api".
+_API_MODE = os.getenv("EXTRACTOR_BACKEND", "auto").strip().lower() == "api"
+
+
 def extract_lab_values(
     uploaded_file: str | None,
     api_key_override: str,
@@ -2302,6 +2307,11 @@ button.bte-action *,
 
 
 with gr.Blocks(title="Blood Test Explainer") as demo:
+    # Flatten the report container so only the inner .bte-report card shows (no double box).
+    gr.HTML(
+        "<style>.bte-report-panel,.bte-report-panel>*{background:transparent !important;"
+        "border:0 !important;box-shadow:none !important;padding:0 !important;}</style>"
+    )
     with gr.Row(equal_height=True, elem_classes=["bte-title"]):
         with gr.Column(scale=6, min_width=420, elem_classes=["bte-title-copy"]):
             gr.HTML(
@@ -2313,7 +2323,7 @@ with gr.Blocks(title="Blood Test Explainer") as demo:
                 </div>
                 """
             )
-        with gr.Column(scale=4, min_width=360):
+        with gr.Column(scale=4, min_width=360, visible=_API_MODE):
             with gr.Group(elem_classes=["bte-api-key-panel"]):
                 api_key_override = gr.Textbox(
                     label="OpenBMB API key",
@@ -2379,7 +2389,7 @@ with gr.Blocks(title="Blood Test Explainer") as demo:
         show_progress="hidden",
     )
 
-    with gr.Group(visible=False) as report_panel:
+    with gr.Group(visible=False, elem_classes=["bte-report-panel"]) as report_panel:
         report = gr.HTML(empty_report_html())
 
     run_button.click(

src/extraction/factory.py

@@ -1,11 +1,12 @@
 """Backend selection.
 
 `EXTRACTOR_BACKEND` env:
-  - `auto` (default): local if a GGUF is configured + importable, otherwise the API backend.
-    This keeps the app working today (API) and flips to fully-offline the moment the fine-tuned
-    GGUF is bundled — no code change.
-  - `local`: force the offline MiniCPM-V backend (errors if not configured).
-  - `api`:   force the hosted OpenBMB endpoint (dev fallback only).
+  - `auto` (default): local-server if LLAMA_SERVER_URL is set; else the in-process llama.cpp
+    backend if a GGUF + llama-cpp-python are available; else the hosted API.
+  - `local` / `server`: the offline **llama-server** backend (the path that works for MiniCPM-V
+    4.6). Run `llama-server -m model.gguf --mmproj mmproj.gguf --port 8080` alongside the app.
+  - `llamacpp`: in-process llama-cpp-python (only once it supports the model build).
+  - `api`: the hosted OpenBMB endpoint (dev fallback only).
 """
 
 from __future__ import annotations
@@ -14,6 +15,7 @@ import os
 
 from src.extraction.base import Extractor
 from src.extraction.local_minicpmv import LocalMiniCPMVExtractor
+from src.extraction.local_server import LocalServerExtractor
 from src.openbmb_client import OpenBMBExtractor
 
 
@@ -26,20 +28,23 @@ def build_extractor(
 
     if backend == "api":
         return OpenBMBExtractor(api_url=api_url, model=model, api_key=api_key)
-
-    if backend == "local":
+    if backend in ("local", "server", "local-server"):
+        return LocalServerExtractor()
+    if backend == "llamacpp":
         return LocalMiniCPMVExtractor()
 
     # auto
-    if _local_available():
+    if os.getenv("LLAMA_SERVER_URL"):
+        return LocalServerExtractor()
+    if _llamacpp_available():
         try:
             return LocalMiniCPMVExtractor()
         except Exception:
-            pass  # fall through to API
+            pass
     return OpenBMBExtractor(api_url=api_url, model=model, api_key=api_key)
 
 
-def _local_available() -> bool:
+def _llamacpp_available() -> bool:
     if not (os.getenv("LOCAL_MODEL_PATH") and os.getenv("LOCAL_MMPROJ_PATH")):
         return False
     try:

src/extraction/local_server.py

@@ -0,0 +1,105 @@
+"""Offline extraction via a local llama.cpp server (llama-server).
+
+This is the off-grid backend that actually works for MiniCPM-V 4.6. The pip `llama-cpp-python`
+bundles an llama.cpp too old to load 4.6, but the current `llama-server` (brew / release build)
+runs it fine. We POST to a llama-server on localhost with the document image plus:
+  - our **GBNF grammar**, so the output is always the `{tests, notes}` schema, and
+  - `enable_thinking: false`, so the model doesn't spend its whole token budget on a `<think>`
+    ramble (the cause of the "could not be converted into a report" failure).
+
+localhost = the model running on this machine, so it is still fully off-grid (no external call).
+
+Run the server next to the app:
+    llama-server -m model.gguf --mmproj mmproj.gguf --port 8080
+
+Config (env):
+    LLAMA_SERVER_URL    default http://127.0.0.1:8080/v1/chat/completions
+    LLAMA_SERVER_MODEL  default "minicpm-v"
+    LLAMA_SERVER_GRAMMAR set to "1" to send the GBNF grammar (OFF by default: the current
+                        llama-server build rejects our grammar, and `enable_thinking:false`
+                        plus the tolerant parser already yield clean {tests,notes} output)
+"""
+
+from __future__ import annotations
+
+import os
+
+import requests
+
+from src.document_processing import document_to_payload_parts
+from src.grammar import extraction_grammar
+from src.openbmb_client import (
+    EXTRACTION_PROMPT,
+    ExtractionResult,
+    _normalize_notes,
+    _normalize_tests,
+    _parse_json_response,
+)
+
+DEFAULT_SERVER_URL = "http://127.0.0.1:8080/v1/chat/completions"
+
+
+class LocalServerExtractor:
+    """Implements the `Extractor` protocol against a local llama-server."""
+
+    def __init__(
+        self,
+        url: str | None = None,
+        model: str | None = None,
+        timeout_seconds: int = 180,
+    ) -> None:
+        self.url = (url or os.getenv("LLAMA_SERVER_URL") or DEFAULT_SERVER_URL).strip()
+        self.model = (model or os.getenv("LLAMA_SERVER_MODEL") or "minicpm-v").strip()
+        self.timeout_seconds = timeout_seconds
+        self.use_grammar = os.getenv("LLAMA_SERVER_GRAMMAR", "0") == "1"
+
+    def extract(self, file_path: str, max_pages: int = 3) -> ExtractionResult:
+        parts = document_to_payload_parts(file_path, max_pages=max_pages)
+        payload = {
+            "model": self.model,
+            "messages": [
+                {"role": "user", "content": [{"type": "text", "text": EXTRACTION_PROMPT}, *parts]}
+            ],
+            "temperature": 0,
+            "max_tokens": 2048,
+            # Stop the model from emitting a <think> reasoning block (it otherwise burns the
+            # whole token budget before producing JSON). Unknown fields are ignored by the server.
+            "chat_template_kwargs": {"enable_thinking": False},
+        }
+        if self.use_grammar:
+            # Grammar-constrained decoding: output can only be our {tests, notes} schema.
+            payload["grammar"] = extraction_grammar()
+
+        response = requests.post(
+            self.url,
+            json=payload,
+            headers={"Content-Type": "application/json"},
+            timeout=self.timeout_seconds,
+        )
+        response.raise_for_status()
+
+        raw = _message_content(response.json())
+        parsed = _parse_json_response(raw)
+        return ExtractionResult(
+            tests=_normalize_tests(parsed.get("tests", [])),
+            notes=_normalize_notes(parsed.get("notes", [])),
+            raw_response=raw,
+            request_summary={
+                "backend": "local-server",
+                "url": self.url,
+                "document_parts": len(parts),
+                "max_pages": max_pages,
+                "grammar": self.use_grammar,
+            },
+        )
+
+
+def _message_content(payload: dict) -> str:
+    try:
+        message = payload["choices"][0]["message"]
+    except (KeyError, IndexError, TypeError) as error:
+        raise ValueError("llama-server response did not include choices[0].message.") from error
+    content = message.get("content") or ""
+    if isinstance(content, list):
+        content = "\n".join(p.get("text", "") for p in content if isinstance(p, dict))
+    return content.strip()

src/openbmb_client.py

@@ -160,11 +160,15 @@ def _normalize_api_key(value: str | None) -> str | None:
 
 
 def _parse_json_response(text: str) -> dict[str, Any]:
-    cleaned = _strip_code_fence(text)
+    cleaned = _strip_think(_strip_code_fence(text))
     parsed = _loads_model_json(cleaned)
 
+    # Some models (e.g. MiniCPM-V in "thinking" mode) return a bare array of tests
+    # instead of the {tests, notes} object. Wrap it so the rest of the app is unchanged.
+    if isinstance(parsed, list):
+        return {"tests": parsed, "notes": []}
     if not isinstance(parsed, dict):
-        raise ValueError("Model response JSON must be an object.")
+        raise ValueError("Model response JSON must be an object or array.")
     return parsed
 
 
@@ -175,9 +179,9 @@ def _loads_model_json(text: str) -> Any:
         try:
             return json.loads(text, strict=False)
         except json.JSONDecodeError:
-            match = re.search(r"\{.*\}", text, flags=re.DOTALL)
+            match = re.search(r"\[.*\]|\{.*\}", text, flags=re.DOTALL)
             if not match:
-                raise ValueError("Model response did not contain a JSON object.")
+                raise ValueError("Model response did not contain JSON.")
             snippet = match.group(0)
             try:
                 return json.loads(snippet)
@@ -196,6 +200,14 @@ def _strip_code_fence(text: str) -> str:
     return stripped.strip()
 
 
+_THINK_RE = re.compile(r"<think(?:ing)?>.*?</think(?:ing)?>", flags=re.DOTALL | re.IGNORECASE)
+
+
+def _strip_think(text: str) -> str:
+    """Drop <think>...</think> reasoning blocks some models emit before the JSON."""
+    return _THINK_RE.sub("", text).strip()
+
+
 def _normalize_tests(value: Any) -> list[dict[str, Any]]:
     if not isinstance(value, list):
         return []

tests/test_parsing.py

@@ -0,0 +1,37 @@
+"""Parser must survive MiniCPM-V's thinking mode + bare-array output."""
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from src.openbmb_client import _normalize_tests, _parse_json_response  # noqa: E402
+
+
+def test_think_block_then_bare_array():
+    raw = (
+        "<think>\nWe need to extract the tests...\n</think>\n\n"
+        '[{"marker": "Glucose", "value": "95", "unit": "mg/dL", "reference_range": "70-99"}]'
+    )
+    parsed = _parse_json_response(raw)
+    assert set(parsed) == {"tests", "notes"}
+    tests = _normalize_tests(parsed["tests"])
+    assert len(tests) == 1 and tests[0]["marker"] == "Glucose"
+
+
+def test_plain_object_still_works():
+    raw = '{"tests": [{"marker": "ALT", "value": "30"}], "notes": ["ok"]}'
+    parsed = _parse_json_response(raw)
+    assert parsed["notes"] == ["ok"]
+    assert _normalize_tests(parsed["tests"])[0]["marker"] == "ALT"
+
+
+def test_code_fenced_array():
+    raw = '```json\n[{"marker": "TSH", "value": "2.1", "unit": "mIU/L"}]\n```'
+    tests = _normalize_tests(_parse_json_response(raw)["tests"])
+    assert tests[0]["marker"] == "TSH"
+
+
+def test_prose_then_object_is_recovered():
+    raw = 'Here are the results:\n{"tests": [{"marker": "HDL", "value": "55"}], "notes": []}'
+    assert _normalize_tests(_parse_json_response(raw)["tests"])[0]["marker"] == "HDL"