fix: add retry logic, error handling, and startup diagnostics for deployed pipeline

- Add startup config logging in main.py (masked secrets, warnings if empty)
- Add retry with exponential backoff in MedGemma API (3 retries, handles 503 cold-start)
- Broaden patient_parser exception handler (catch all, not just ValueError)
- Stop pipeline on critical step failure (skip remaining instead of cascading errors)
- Add /api/health/config diagnostic endpoint

src/backend/app/agent/orchestrator.py

@@ -122,6 +122,9 @@ class Orchestrator:
         This is the main entry point. Each step is executed sequentially,
         with state flowing from one step to the next. Steps that don't
         depend on each other (drug check + guidelines) run in parallel.
+
+        If a critical step (parse, reason) fails, subsequent dependent
+        steps are marked as SKIPPED to avoid cascading errors.
         """
         case_id = str(uuid.uuid4())[:8]
         steps = self._create_steps(case)
@@ -134,10 +137,23 @@ class Orchestrator:
 
         try:
             # ── Step 1: Parse patient data ──
-            yield await self._run_step("parse", self._step_parse, case.patient_text)
+            step = await self._run_step("parse", self._step_parse, case.patient_text)
+            yield step
+
+            if step.status == AgentStepStatus.FAILED:
+                # Can't continue without patient profile — skip remaining steps
+                yield from self._skip_remaining_steps("parse")
+                self._state.completed_at = datetime.utcnow()
+                return
 
             # ── Step 2: Clinical reasoning ──
-            yield await self._run_step("reason", self._step_reason)
+            step = await self._run_step("reason", self._step_reason)
+            yield step
+
+            if step.status == AgentStepStatus.FAILED:
+                yield from self._skip_remaining_steps("reason")
+                self._state.completed_at = datetime.utcnow()
+                return
 
             # ── Step 3 & 4: Drug check + Guidelines (parallel) ──
             parallel_tasks = []
@@ -175,6 +191,20 @@ class Orchestrator:
                     step.error = f"Pipeline aborted: {str(e)}"
             raise
 
+    def _skip_remaining_steps(self, after_step_id: str) -> list[AgentStep]:
+        """Mark all steps after after_step_id as skipped. Returns them for yielding."""
+        skipped = []
+        found = False
+        for step in self._state.steps:
+            if step.step_id == after_step_id:
+                found = True
+                continue
+            if found and step.status == AgentStepStatus.PENDING:
+                step.status = AgentStepStatus.SKIPPED
+                step.error = f"Skipped: prerequisite step '{after_step_id}' failed"
+                skipped.append(step)
+        return skipped
+
     async def _run_step(self, step_id: str, fn, *args) -> AgentStep:
         """Execute a single step, tracking status and timing."""
         step = self._get_step(step_id)

src/backend/app/api/health.py

@@ -1,9 +1,26 @@
 """Health check endpoint."""
+import logging
+
 from fastapi import APIRouter
 
+from app.config import settings
+
+logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
 @router.get("/health")
 async def health_check():
     return {"status": "ok", "service": "CDS Agent"}
+
+
+@router.get("/api/health/config")
+async def config_check():
+    """Diagnostic endpoint: shows whether critical env vars are configured (no secrets)."""
+    return {
+        "medgemma_base_url_set": bool(settings.medgemma_base_url),
+        "medgemma_api_key_set": bool(settings.medgemma_api_key),
+        "medgemma_model_id": settings.medgemma_model_id,
+        "hf_token_set": bool(settings.hf_token),
+        "medgemma_max_tokens": settings.medgemma_max_tokens,
+    }

src/backend/app/main.py

@@ -1,12 +1,17 @@
 """
 Clinical Decision Support Agent — FastAPI Backend
 """
+import logging
+
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
 from app.api import cases, health, ws
 from app.config import settings
 
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s")
+logger = logging.getLogger(__name__)
+
 app = FastAPI(
     title="Clinical Decision Support Agent",
     description="Agentic clinical decision support powered by MedGemma (HAI-DEF)",
@@ -31,6 +36,24 @@ app.include_router(ws.router, prefix="/ws", tags=["websocket"])
 @app.on_event("startup")
 async def startup():
     """Initialize services on startup."""
-    # TODO: Initialize MedGemma model / connection
-    # TODO: Initialize RAG vector store
-    pass
+    # Log configuration (mask secrets)
+    def _mask(val: str) -> str:
+        if not val:
+            return "(empty)"
+        if len(val) <= 8:
+            return "***"
+        return val[:4] + "..." + val[-4:]
+
+    logger.info("=== CDS Agent Backend Starting ===")
+    logger.info(f"  medgemma_base_url : {settings.medgemma_base_url or '(empty)'}")
+    logger.info(f"  medgemma_model_id : {settings.medgemma_model_id}")
+    logger.info(f"  medgemma_api_key  : {_mask(settings.medgemma_api_key)}")
+    logger.info(f"  hf_token          : {_mask(settings.hf_token)}")
+    logger.info(f"  medgemma_max_tokens: {settings.medgemma_max_tokens}")
+    logger.info(f"  cors_origins      : {settings.cors_origins}")
+    logger.info(f"  chroma_persist_dir: {settings.chroma_persist_dir}")
+
+    if not settings.medgemma_base_url:
+        logger.warning("MEDGEMMA_BASE_URL is empty -- MedGemma API calls will fail!")
+    if not settings.medgemma_api_key:
+        logger.warning("MEDGEMMA_API_KEY is empty -- MedGemma API calls will fail!")

src/backend/app/services/medgemma.py

@@ -10,6 +10,7 @@ All tools that need MedGemma go through this service.
 """
 from __future__ import annotations
 
+import asyncio
 import json
 import logging
 from typing import Any, Optional, Type, TypeVar
@@ -22,6 +23,10 @@ logger = logging.getLogger(__name__)
 
 T = TypeVar("T", bound=BaseModel)
 
+# Retry configuration for transient API errors (cold-start / 503)
+MAX_API_RETRIES = 3
+RETRY_BASE_DELAY = 5.0  # seconds, doubles on each retry
+
 
 class MedGemmaService:
     """
@@ -146,6 +151,9 @@ class MedGemmaService:
         happens to be plain Gemma on Google AI Studio (which rejects the system
         role), we automatically fall back to folding the system prompt into the
         user message.
+
+        Includes retry with exponential backoff for transient errors (503 cold
+        start, connection errors, timeouts).
         """
         client = await self._get_client()
 
@@ -154,29 +162,59 @@ class MedGemmaService:
             messages.append({"role": "system", "content": system_prompt})
         messages.append({"role": "user", "content": prompt})
 
-        try:
-            response = await client.chat.completions.create(
-                model=settings.medgemma_model_id,
-                messages=messages,
-                max_tokens=max_tokens,
-                temperature=temperature,
-            )
-            return response.choices[0].message.content
-        except Exception as e:
-            # Fallback: fold system prompt into user message (Google AI Studio compat)
-            if system_prompt and "system" in str(e).lower():
-                logger.warning("Backend rejected system role — folding into user message.")
-                fallback_messages = [
-                    {"role": "user", "content": f"{system_prompt}\n\n{prompt}"}
-                ]
+        last_error: Optional[Exception] = None
+
+        for attempt in range(MAX_API_RETRIES):
+            try:
                 response = await client.chat.completions.create(
                     model=settings.medgemma_model_id,
-                    messages=fallback_messages,
+                    messages=messages,
                     max_tokens=max_tokens,
                     temperature=temperature,
                 )
                 return response.choices[0].message.content
-            raise
+            except Exception as e:
+                error_str = str(e).lower()
+                last_error = e
+
+                # Detect system-role rejection (Google AI Studio) — immediate fallback, no retry
+                if system_prompt and "system" in error_str:
+                    logger.warning("Backend rejected system role -- folding into user message.")
+                    fallback_messages = [
+                        {"role": "user", "content": f"{system_prompt}\n\n{prompt}"}
+                    ]
+                    try:
+                        response = await client.chat.completions.create(
+                            model=settings.medgemma_model_id,
+                            messages=fallback_messages,
+                            max_tokens=max_tokens,
+                            temperature=temperature,
+                        )
+                        return response.choices[0].message.content
+                    except Exception as e2:
+                        last_error = e2
+                        error_str = str(e2).lower()
+
+                # Retry on transient errors (503, 502, 429, connection, timeout)
+                is_transient = any(
+                    keyword in error_str
+                    for keyword in ["503", "502", "429", "service unavailable", "overloaded",
+                                    "connection", "timeout", "timed out", "temporarily"]
+                )
+                if is_transient and attempt < MAX_API_RETRIES - 1:
+                    delay = RETRY_BASE_DELAY * (2 ** attempt)
+                    logger.warning(
+                        f"MedGemma API transient error (attempt {attempt + 1}/{MAX_API_RETRIES}): "
+                        f"{e}. Retrying in {delay:.0f}s..."
+                    )
+                    await asyncio.sleep(delay)
+                    continue
+
+                # Non-transient or final attempt — log and raise
+                logger.error(f"MedGemma API error (attempt {attempt + 1}/{MAX_API_RETRIES}): {e}")
+                break
+
+        raise last_error
 
     async def _generate_local(
         self, prompt: str, system_prompt: Optional[str], max_tokens: int, temperature: float

src/backend/app/tools/patient_parser.py

@@ -67,11 +67,11 @@ class PatientParserTool:
             logger.info(f"Parsed patient profile: {profile.chief_complaint}")
             return profile
 
-        except ValueError:
-            # Fallback: If structured parsing fails, do basic extraction
-            logger.warning("Structured parsing failed, attempting basic extraction")
+        except Exception as e:
+            # Fallback: If any error occurs (API, parsing, etc.), do basic extraction
+            logger.warning(f"Patient parsing failed ({type(e).__name__}: {e}), using basic extraction")
             return PatientProfile(
                 chief_complaint=patient_text[:200],
                 history_of_present_illness=patient_text,
-                additional_notes="Auto-extracted from raw text (structured parsing failed)",
+                additional_notes=f"Auto-extracted from raw text (structured parsing failed: {type(e).__name__})",
             )