fix: harden modal render status polling

backend/modal_client.py

@@ -18,12 +18,14 @@ class ModalSynthesisClient:
         base_url: Optional[str],
         auth_token: Optional[str] = None,
         timeout_seconds: float = 300.0,
-        poll_interval_seconds: float = 1.0,
+        poll_interval_seconds: float = 3.0,
+        max_status_retries: int = 4,
     ) -> None:
         self.base_url = base_url.rstrip("/") if base_url else None
         self.auth_token = auth_token
         self.timeout_seconds = timeout_seconds
         self.poll_interval_seconds = poll_interval_seconds
+        self.max_status_retries = max_status_retries
 
     @classmethod
     def from_env(cls) -> "ModalSynthesisClient":
@@ -31,7 +33,8 @@ class ModalSynthesisClient:
             base_url=os.getenv("SCRIPTORIUM_MODAL_BASE_URL"),
             auth_token=os.getenv("SCRIPTORIUM_MODAL_AUTH_TOKEN"),
             timeout_seconds=float(os.getenv("SCRIPTORIUM_MODAL_TIMEOUT_SECONDS", "300")),
-            poll_interval_seconds=float(os.getenv("SCRIPTORIUM_MODAL_POLL_INTERVAL_SECONDS", "1")),
+            poll_interval_seconds=float(os.getenv("SCRIPTORIUM_MODAL_POLL_INTERVAL_SECONDS", "3")),
+            max_status_retries=int(os.getenv("SCRIPTORIUM_MODAL_STATUS_RETRIES", "4")),
         )
 
     def is_configured(self) -> bool:
@@ -110,8 +113,26 @@ class ModalSynthesisClient:
         voice_config: VoiceConfig,
     ) -> Iterable[Dict[str, object]]:
         cursor = 0
+        consecutive_status_failures = 0
         while True:
-            data = self._get_json(f"/renders/{job_id}", params={"cursor": cursor})
+            try:
+                data = self._get_json(f"/renders/{job_id}", params={"cursor": cursor})
+                consecutive_status_failures = 0
+            except ValueError as exc:
+                consecutive_status_failures += 1
+                if consecutive_status_failures > self.max_status_retries:
+                    raise
+                yield {
+                    "type": "log",
+                    "session_id": session_id,
+                    "backend": MODAL_BACKEND,
+                    "model": voice_config.model,
+                    "message": (
+                        f"Transient Modal status error ({consecutive_status_failures}/{self.max_status_retries}): {exc}"
+                    ),
+                }
+                time.sleep(self.poll_interval_seconds)
+                continue
             events = list(data.get("events") or [])
             cursor += len(events)
             for event in events:
@@ -178,7 +199,11 @@ class ModalSynthesisClient:
                 "Try again, or increase SCRIPTORIUM_MODAL_TIMEOUT_SECONDS."
             ) from exc
         except requests.RequestException as exc:
-            raise ValueError(f"Modal request failed: {exc}") from exc
+            detail = ""
+            response = getattr(exc, "response", None)
+            if response is not None and response.text:
+                detail = f" | response={response.text[:400]}"
+            raise ValueError(f"Modal request failed: {exc}{detail}") from exc
 
     def _get_json(self, path: str, *, params: Dict[str, object]) -> Dict[str, object]:
         self._ensure_configured()
@@ -197,7 +222,11 @@ class ModalSynthesisClient:
                 "Try again, or increase SCRIPTORIUM_MODAL_TIMEOUT_SECONDS."
             ) from exc
         except requests.RequestException as exc:
-            raise ValueError(f"Modal request failed: {exc}") from exc
+            detail = ""
+            response = getattr(exc, "response", None)
+            if response is not None and response.text:
+                detail = f" | response={response.text[:400]}"
+            raise ValueError(f"Modal request failed: {exc}{detail}") from exc
 
     def _absolute_url(self, path: str) -> str:
         if path.startswith("http://") or path.startswith("https://"):

backend/synthesis_service.py

@@ -150,7 +150,9 @@ class SynthesisService:
             render_dir=render_dir,
             voice_config=voice_config,
         ):
-            self._modal_status[session_id] = str(event.get("type", "running"))
+            event_type = str(event.get("type", "running"))
+            if event_type not in {"log"}:
+                self._modal_status[session_id] = event_type
             yield event
         if self._modal_status.get(session_id) not in {"failed", "cancelled"}:
             self._modal_status[session_id] = "completed"

modal_app.py

@@ -88,16 +88,23 @@ def render_preview(payload: Dict[str, Any]) -> Dict[str, Any]:
 
 
 def _job_state(job_id: str) -> Dict[str, Any]:
-    return dict(
-        jobs.get(
-            job_id,
-            {
-                "status": "pending",
-                "events": [],
-                "cancel_requested": False,
-            },
-        )
+    raw = jobs.get(
+        job_id,
+        {
+            "status": "pending",
+            "events": [],
+            "cancel_requested": False,
+        },
     )
+    if raw is None:
+        return {
+            "status": "pending",
+            "events": [],
+            "cancel_requested": False,
+        }
+    if not isinstance(raw, dict):
+        raise RuntimeError(f"Corrupt job state for {job_id}: expected dict, got {type(raw).__name__}")
+    return dict(raw)
 
 
 def _save_job_state(job_id: str, state: Dict[str, Any]) -> None:
@@ -228,13 +235,16 @@ def render_status_endpoint(
     authorization: str | None = Header(default=None),
 ) -> Dict[str, Any]:
     _check_auth(authorization)
-    state = _job_state(job_id)
-    events = list(state.get("events", []))
-    return {
-        "job_id": job_id,
-        "status": state.get("status", "pending"),
-        "events": events[cursor:],
-    }
+    try:
+        state = _job_state(job_id)
+        events = list(state.get("events", []))
+        return {
+            "job_id": job_id,
+            "status": state.get("status", "pending"),
+            "events": events[cursor:],
+        }
+    except Exception as exc:
+        raise HTTPException(status_code=500, detail=f"Unable to fetch render status for {job_id}: {exc}") from exc
 
 
 @web_app.post("/renders/{job_id}/cancel")

tests/test_modal_client.py

@@ -28,11 +28,13 @@ def test_modal_client_defaults_to_longer_timeout() -> None:
 def test_modal_client_uses_longer_default_timeout_from_env(monkeypatch) -> None:
     monkeypatch.delenv("SCRIPTORIUM_MODAL_TIMEOUT_SECONDS", raising=False)
     monkeypatch.delenv("SCRIPTORIUM_MODAL_POLL_INTERVAL_SECONDS", raising=False)
+    monkeypatch.delenv("SCRIPTORIUM_MODAL_STATUS_RETRIES", raising=False)
 
     client = ModalSynthesisClient.from_env()
 
     assert client.timeout_seconds == 300.0
-    assert client.poll_interval_seconds == 1.0
+    assert client.poll_interval_seconds == 3.0
+    assert client.max_status_retries == 4
 
 
 def test_modal_client_surfaces_read_timeout_as_user_facing_error(monkeypatch, tmp_path) -> None:
@@ -51,3 +53,35 @@ def test_modal_client_surfaces_read_timeout_as_user_facing_error(monkeypatch, tm
             diffusion_steps=32,
             speed=1.0,
         )
+
+
+def test_modal_client_retries_transient_status_failures(monkeypatch, tmp_path) -> None:
+    client = ModalSynthesisClient(
+        base_url="https://scriptorium-tts--mattkevan.modal.run",
+        poll_interval_seconds=0,
+        max_status_retries=2,
+    )
+
+    calls = {"count": 0}
+
+    def fake_get_json(path, *, params):
+        calls["count"] += 1
+        if calls["count"] == 1:
+            raise ValueError("Modal request failed: 500 Server Error")
+        return {"status": "completed", "events": []}
+
+    monkeypatch.setattr(client, "_get_json", fake_get_json)
+    monkeypatch.setattr("backend.modal_client.time.sleep", lambda *_args, **_kwargs: None)
+
+    events = list(
+        client.render(
+            session_id="session-a",
+            job_id="job-123",
+            render_dir=tmp_path,
+            voice_config=type("Voice", (), {"model": "omnivoice"})(),
+        )
+    )
+
+    assert calls["count"] == 2
+    assert events[0]["type"] == "log"
+    assert "Transient Modal status error" in events[0]["message"]