Fall back to CPU when the GPU path fails, and surface tracebacks

/translate 500d in 5s with mt.error still null, so the failure was in the
ZeroGPU call rather than model loading. Try the decorated function, then
retry the same work on CPU instead of failing the request. Reading Space
logs needs a token, so unhandled errors now come back in the response body.

app.py

@@ -58,6 +58,23 @@ DEVICE = os.environ.get("ASR_DEVICE", "cuda" if _ON_ZEROGPU else "cpu")
 # at a glance whether the models have finished loading).
 app = FastAPI()
 
+
+@app.exception_handler(Exception)
+def _unhandled(request, exc):
+    """
+    Return the traceback in the response body. Reading a Space's logs needs an
+    HF token, so an opaque 500 is genuinely hard to diagnose from the client
+    that is calling this. Nothing here handles user data worth hiding.
+    """
+    import traceback
+    from fastapi.responses import JSONResponse
+    tb = traceback.format_exc()
+    print(tb, flush=True)
+    return JSONResponse(status_code=500,
+                        content={"error": f"{type(exc).__name__}: {exc}",
+                                 "traceback": tb.splitlines()[-12:],
+                                 "device": DEVICE})
+
 # --------------------------------------------------------------- MODEL LOAD ---
 # Loading takes minutes on a cold Space (weights download + CPU init). Doing it
 # at import time would make the Space fail its health check and restart-loop, so
@@ -172,8 +189,24 @@ def _words_from_chunks(chunks, audio_secs: float):
     return words
 
 
-@_GPU
-def _asr_run(audio, gen):
+def _device_fallback() -> bool:
+    """
+    Drop to CPU and force a reload. ZeroGPU only grants a device inside its
+    decorated functions, and the grant can fail for reasons that have nothing
+    to do with the model — quota, a worker thread, a cold pool. Falling back is
+    slower but keeps the service answering instead of 500ing.
+    """
+    global DEVICE
+    if DEVICE == "cpu":
+        return False
+    print(f"GPU path failed on {DEVICE} — reloading on CPU", flush=True)
+    DEVICE = "cpu"
+    _models["asr"] = _models["mt"] = None
+    _errors["asr"] = _errors["mt"] = None
+    return True
+
+
+def _asr_impl(audio, gen):
     """Decoding, on the GPU when ZeroGPU has granted one."""
     pipe = _load("asr")
     try:
@@ -186,6 +219,19 @@ def _asr_run(audio, gen):
         return pipe(audio.copy(), return_timestamps=True, generate_kwargs=gen), False
 
 
+_asr_gpu = _GPU(_asr_impl)
+
+
+def _asr_run(audio, gen):
+    try:
+        return _asr_gpu(audio, gen)
+    except Exception as e:
+        print(f"asr on {DEVICE} failed ({type(e).__name__}: {e})", flush=True)
+        if not _device_fallback():
+            raise
+        return _asr_impl(audio, gen)
+
+
 @app.post("/asr")
 async def asr(request: Request, x_service_token: str = Header(default=None)):
     _auth(x_service_token)
@@ -221,11 +267,23 @@ class TranslateIn(BaseModel):
     texts: list[str]
 
 
-@_GPU
-def _mt_run(texts):
+def _mt_impl(texts):
     return _load("mt")(texts, batch_size=MT_BATCH, truncation=True)
 
 
+_mt_gpu = _GPU(_mt_impl)
+
+
+def _mt_run(texts):
+    try:
+        return _mt_gpu(texts)
+    except Exception as e:
+        print(f"mt on {DEVICE} failed ({type(e).__name__}: {e})", flush=True)
+        if not _device_fallback():
+            raise
+        return _mt_impl(texts)
+
+
 @app.post("/translate")
 def translate(body: TranslateIn, x_service_token: str = Header(default=None)):
     _auth(x_service_token)