Fall back from Transformers to llama.cpp

README.md

@@ -51,7 +51,8 @@ This is the active deployment path.
 
 With `EXTRACTOR_BACKEND=auto`, the app checks CUDA availability at runtime. If CUDA is visible, it
 uses the official OpenBMB MiniCPM-V 4.6 Transformers path. If CUDA is not visible, it falls back to
-the CPU `llama.cpp` GGUF path. The deterministic knowledge-graph enrichment and UI rendering stay
+the CPU `llama.cpp` GGUF path. If the CUDA worker fails, `auto` retries with CPU llama.cpp unless
+`AUTO_FALLBACK_TO_LLAMACPP=0`. The deterministic knowledge-graph enrichment and UI rendering stay
 in normal Gradio/Python code.
 
 This workflow should not be further changed back to Docker unless the project intentionally gives up

RUNBOOK.md

@@ -24,7 +24,7 @@ Do not switch the Space back to Docker unless the project intentionally gives up
 
 `EXTRACTOR_BACKEND`:
 
-- `auto`: uses the Transformers backend when `torch.cuda.is_available()` is true; otherwise uses the CPU llama.cpp backend.
+- `auto`: uses the Transformers backend when `torch.cuda.is_available()` is true; otherwise uses the CPU llama.cpp backend. If the Transformers worker fails, it retries with CPU llama.cpp unless `AUTO_FALLBACK_TO_LLAMACPP=0`.
 - `llamacpp-gpu`: force the GGUF llama.cpp backend.
 - `zerogpu` / `transformers`: force the OpenBMB Transformers backend.
 - `api`: hosted OpenBMB endpoint for development fallback only.

src/extraction/auto.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import os
+
 from src.extraction.base import Extractor
 from src.extraction.llamacpp_gpu import LlamaCppGPUExtractor
 from src.extraction.zerogpu_transformers import ZeroGPUTransformersExtractor
@@ -15,7 +17,20 @@ class AutoExtractor:
         self._selected: Extractor | None = None
 
     def extract(self, file_path: str, max_pages: int = 3):
-        return self._backend().extract(file_path, max_pages=max_pages)
+        backend = self._backend()
+        try:
+            return backend.extract(file_path, max_pages=max_pages)
+        except Exception as exc:
+            if not isinstance(backend, ZeroGPUTransformersExtractor) or not _fallback_enabled():
+                raise
+
+            print(
+                "[Blood Test Explainer] CUDA Transformers backend failed; "
+                f"falling back to CPU llama.cpp. Inner error: {type(exc).__name__}: {exc}",
+                flush=True,
+            )
+            self._selected = LlamaCppGPUExtractor()
+            return self._selected.extract(file_path, max_pages=max_pages)
 
     def _backend(self) -> Extractor:
         if self._selected is None:
@@ -32,6 +47,10 @@ def cuda_available() -> bool:
     except Exception:
         return False
 
+
+def _fallback_enabled() -> bool:
+    return os.getenv("AUTO_FALLBACK_TO_LLAMACPP", "1").strip().lower() not in {"0", "false", "no"}
+
     try:
         return bool(torch.cuda.is_available())
     except Exception:

src/extraction/zerogpu_transformers.py

@@ -143,31 +143,37 @@ def _run_zerogpu_generation(
 ) -> str:
     import torch
 
-    processor, model = _get_model(model_id)
-    inputs = processor.apply_chat_template(
-        messages,
-        tokenize=True,
-        add_generation_prompt=True,
-        return_dict=True,
-        return_tensors="pt",
-        downsample_mode=downsample_mode,
-        max_slice_nums=9,
-    ).to(model.device)
-
-    with torch.inference_mode():
-        generated_ids = model.generate(
-            **inputs,
+    try:
+        processor, model = _get_model(model_id)
+        inputs = processor.apply_chat_template(
+            messages,
+            tokenize=True,
+            add_generation_prompt=True,
+            return_dict=True,
+            return_tensors="pt",
             downsample_mode=downsample_mode,
-            max_new_tokens=max_new_tokens,
-            do_sample=False,
+            max_slice_nums=9,
+        ).to(model.device)
+
+        with torch.inference_mode():
+            generated_ids = model.generate(
+                **inputs,
+                downsample_mode=downsample_mode,
+                max_new_tokens=max_new_tokens,
+                do_sample=False,
+            )
+
+        generated_ids_trimmed = [
+            out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids, strict=False)
+        ]
+        output_text = processor.batch_decode(
+            generated_ids_trimmed,
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=False,
         )
-
-    generated_ids_trimmed = [
-        out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids, strict=False)
-    ]
-    output_text = processor.batch_decode(
-        generated_ids_trimmed,
-        skip_special_tokens=True,
-        clean_up_tokenization_spaces=False,
-    )
-    return str(output_text[0]).strip() if output_text else ""
+        return str(output_text[0]).strip() if output_text else ""
+    except Exception as exc:
+        raise RuntimeError(
+            "OpenBMB Transformers generation failed on the CUDA/ZeroGPU lane. "
+            f"Inner error: {type(exc).__name__}: {exc}"
+        ) from exc