Update app.py

app.py

@@ -3,17 +3,18 @@
 Elderly HealthWatch AI Backend (FastAPI)
 Pipeline:
  - receive images
- - run VLM (remote HF Space via gradio_api/call/chat style) -> raw text + meta
+ - run VLM (remote gradio / chat_fn) -> JSON feature vector + raw text + meta
  - run LLM (remote gradio /chat) -> structured risk JSON (per requested schema)
  - continue rest of processing and store results
 
 Notes:
- - Add gradio_client==1.13.2 (or another compatible 1.x) to requirements.txt for LLM calls
- - Add httpx to requirements.txt for VLM POST/GET flow
+ - Add gradio_client==1.13.2 (or another compatible 1.x) to requirements.txt
  - If VLM/LLM Spaces are private, set HF_TOKEN in the environment for authentication.
  - This variant:
-    * sends the face image to the HF Space using the POST/GET event flow (tries JSON data-uri first, then multipart fallback)
-    * returns raw VLM output and meta (no VLM-side JSON extraction)
+    * logs raw VLM responses,
+    * always returns raw VLM output in API responses,
+    * extracts JSON from VLM via regex when possible, and
+    * sends only the face image to the VLM (not the eye image).
 """
 
 import io
@@ -25,7 +26,6 @@ import logging
 import traceback
 import re
 import time
-import base64
 from typing import Dict, Any, Optional, Tuple
 from datetime import datetime
 
@@ -36,10 +36,7 @@ from PIL import Image
 import numpy as np
 import cv2  # opencv-python-headless expected installed
 
-# HTTP client for the POST/GET event style VLM calls
-import httpx
-
-# Optional gradio client (for LLM calls)
+# Optional gradio client (for VLM + LLM calls)
 try:
     from gradio_client import Client, handle_file  # type: ignore
     GRADIO_AVAILABLE = True
@@ -249,243 +246,156 @@ def extract_json_via_regex(raw_text: str) -> Dict[str, Any]:
     return out
 
 # -----------------------
-# VLM helper using HF Spaces POST/GET event flow (gradio_api/call/chat)
-# Robust: try JSON (data-uri) POST first; if 5xx, fall back to multipart/form-data file upload.
+# Gradio / VLM helper (sends only face image, returns meta)
 # -----------------------
+def get_gradio_client_for_space(space: str) -> Client:
+    if not GRADIO_AVAILABLE:
+        raise RuntimeError("gradio_client not installed in this environment. Add gradio_client to requirements.txt.")
+    if HF_TOKEN:
+        return Client(space, hf_token=HF_TOKEN)
+    return Client(space)
+
 def run_vlm_and_get_features(face_path: str, eye_path: Optional[str] = None, prompt: Optional[str] = None,
                              raise_on_file_delivery_failure: bool = False
                              ) -> Tuple[Optional[Dict[str, Any]], str, Dict[str, Any]]:
     """
-    VLM caller using the HF Spaces 'gradio_api/call/chat' style:
-      1) POST -> returns an EVENT_ID
-      2) GET /gradio_api/call/chat/{EVENT_ID} -> fetch result
-
-    Behavior:
-      - Try JSON payload with data URI (fast path)
-      - If JSON POST yields server error (5xx), retry with multipart/form-data attaching the face image
-      - Poll GET endpoint a few times for result
-      - Return (parsed_features_or_None, raw_text, meta)
-      - parsed_features is None (we avoid parsing JSON here)
+    Synchronous call to remote VLM (gradio /chat_fn). Sends ONLY the face image file.
+    Returns tuple: (parsed_features_dict_or_None, raw_text_response_str, meta)
+    meta includes:
+      - vlm_file_delivery_ok (bool)  # expects ≥1 file acknowledged (face)
+      - vlm_files_seen (int or None)
+      - vlm_raw_len (int)
+      - vlm_out_object (short repr)
     """
     prompt = prompt or DEFAULT_VLM_PROMPT
 
+    
     if not os.path.exists(face_path):
         raise FileNotFoundError(f"Face image not found at: {face_path}")
+    if not os.path.exists(eye_path):
+        raise FileNotFoundError(f"Eye image not found at: {eye_path}")
+    
+    face_size = os.path.getsize(face_path)
+    eye_size = os.path.getsize(eye_path)
+    logger.info(f"VLM input files - Face: {face_size} bytes, Eye: {eye_size} bytes")
+    
+    if face_size == 0 or eye_size == 0:
+        raise ValueError("One or both images are empty (0 bytes)")
+    
+    if not GRADIO_AVAILABLE:
+        raise RuntimeError("gradio_client not available in this environment.")
 
-    with open(face_path, "rb") as f:
-        face_bytes = f.read()
-    if not face_bytes:
-        raise ValueError("Face image is empty (0 bytes)")
+    client = get_gradio_client_for_space(GRADIO_VLM_SPACE)
+    
+    # Verify files can be opened as images
+    try:
+        Image.open(face_path).verify()
+        Image.open(eye_path).verify()
+        logger.info("Both images verified as valid")
+    except Exception as e:
+        raise ValueError(f"Invalid image file(s): {e}")
+    
+    message = {"text": prompt, "files": [handle_file(face_path), handle_file(eye_path)]}
+    
+    logger.info(f"Calling VLM with message structure: text={len(prompt)} chars, files=2")
+    client = get_gradio_client_for_space(GRADIO_VLM_SPACE)
+    # NOTE: only send face image to the Space
 
-    face_b64 = base64.b64encode(face_bytes).decode("ascii")
-    face_data_uri = f"data:image/jpeg;base64,{face_b64}"
+    message = {"text": prompt, "files": [handle_file(face_path)]}
 
-    payload_json = {
-        "data": [
-            {
-                "text": prompt,
-                "files": [face_data_uri]
-            }
-        ]
-    }
+    meta: Dict[str, Any] = {"vlm_file_delivery_ok": False, "vlm_files_seen": None, "vlm_raw_len": 0, "vlm_out_object": None}
 
-    # Prepare endpoint(s)
-    SPACE_HOST = os.getenv("VLM_SPACE_HOST")  # optional full host override
-    if SPACE_HOST:
-        base_url = SPACE_HOST.rstrip("/")
+    # SINGLE CALL (no retries)
+    try:
+        logger.info("Calling VLM Space %s with 1 file (face only)", GRADIO_VLM_SPACE)
+        result = client.predict(message=message, history=[], api_name="/chat_fn")
+    except Exception as e:
+        logger.exception("VLM call failed (no retries)")
+        raise RuntimeError(f"VLM call failed: {e}")
+
+    # Normalize result
+    raw_text = ""
+    out = None
+    if not result:
+        logger.warning("VLM returned empty result object")
     else:
-        # Many public HF spaces map to {owner}-{space}.hf.space when used in hostnames.
-        if "/" in GRADIO_VLM_SPACE:
-            base_url = f"https://{GRADIO_VLM_SPACE.replace('/', '-')}.hf.space"
+        if isinstance(result, (list, tuple)):
+            out = result[0]
+        elif isinstance(result, dict):
+            out = result
         else:
-            base_url = f"https://{GRADIO_VLM_SPACE}.hf.space"
+            out = {"text": str(result)}
 
-    post_url = f"{base_url}/gradio_api/call/chat"
-    get_url_template = f"{base_url}/gradio_api/call/chat/{{event_id}}"
-
-    headers_json = {"Content-Type": "application/json"}
-    if HF_TOKEN:
-        headers_json["Authorization"] = f"Bearer {HF_TOKEN}"
-
-    meta: Dict[str, Any] = {
-        "vlm_file_delivery_ok": False,
-        "vlm_files_seen": None,
-        "vlm_raw_len": 0,
-        "vlm_out_object": None,
-        "post_url": post_url,
-        "attempts": []
-    }
-
-    def _extract_event_id(resp_text: str, resp_json: Optional[Dict[str, Any]]) -> Optional[str]:
-        if isinstance(resp_json, dict):
-            for k in ("event_id", "id", "job"):
-                if k in resp_json and resp_json[k]:
-                    return resp_json[k]
-        # try a quoted token heuristic (like the awk approach)
-        m = re.search(r'"([^"]{8,})"', resp_text or "")
-        if m:
-            return m.group(1)
-        parts = re.split(r'"', resp_text or "")
-        if len(parts) >= 5:
-            candidate = parts[3].strip()
-            if candidate:
-                return candidate
-        return None
-
-    with httpx.Client(timeout=30.0) as client:
-        # Attempt 1: JSON data-uri POST
+        text_out = out.get("text") or out.get("output") or ""
+        raw_text = text_out or ""
+        meta["vlm_raw_len"] = len(raw_text or "")
         try:
-            logger.info("VLM POST (JSON data-uri) -> %s (prompt len=%d)", post_url, len(prompt))
-            resp = client.post(post_url, headers=headers_json, json=payload_json)
-            resp.raise_for_status()
-            meta["attempts"].append({"mode": "json", "status_code": resp.status_code})
-            try:
-                resp_json = resp.json()
-            except Exception:
-                resp_json = None
-
-            event_id = _extract_event_id(resp.text, resp_json)
-            if not event_id:
-                raise RuntimeError(f"Failed to obtain EVENT_ID from VLM POST (json) response: {resp.text[:1000]}")
-            meta["event_id"] = event_id
-
-        except httpx.HTTPStatusError as he:
-            # Log attempt and fallback to multipart if server-side error
-            status = he.response.status_code if he.response is not None else None
-            body_excerpt = (he.response.text[:1000] if he.response is not None else str(he))
-            logger.warning("VLM JSON POST failed (status=%s). Response excerpt: %s", status, body_excerpt[:400])
-            meta["attempts"].append({"mode": "json", "status_code": status, "error": body_excerpt})
-            if status is None or 500 <= status < 600:
-                # Try multipart fallback
-                try:
-                    logger.info("Attempting multipart/form-data fallback to %s", post_url)
-                    # Some Spaces expect 'data' field to be JSON array describing inputs and files to be referenced.
-                    # We'll send 'data' as JSON string with a placeholder for file indices, and attach the file in 'file' part.
-                    data_field = json.dumps([{"text": prompt, "files": [None]}])
-                    files = {
-                        "data": (None, data_field, "application/json"),
-                        "file": (os.path.basename(face_path), face_bytes, "image/jpeg")
-                    }
-                    # Authorization header only; content-type will be set by httpx for multipart
-                    headers_mp = {}
-                    if HF_TOKEN:
-                        headers_mp["Authorization"] = f"Bearer {HF_TOKEN}"
-
-                    resp2 = client.post(post_url, headers=headers_mp, files=files)
-                    resp2.raise_for_status()
-                    meta["attempts"].append({"mode": "multipart", "status_code": resp2.status_code})
-                    try:
-                        resp2_json = resp2.json()
-                    except Exception:
-                        resp2_json = None
-                    event_id = _extract_event_id(resp2.text, resp2_json)
-                    if not event_id:
-                        raise RuntimeError(f"Failed to obtain EVENT_ID from VLM POST (multipart) response: {resp2.text[:1000]}")
-                    meta["event_id"] = event_id
-                except Exception as e_mp:
-                    logger.exception("Multipart fallback failed")
-                    meta["attempts"].append({"mode": "multipart", "error": str(e_mp)})
-                    raise RuntimeError(f"VLM POST failed (json then multipart): {body_excerpt[:1000]} | multipart error: {str(e_mp)}")
-            else:
-                # Non-5xx error — surface it
-                raise RuntimeError(f"VLM POST failed with status {status}: {body_excerpt[:1000]}")
-        except Exception as e:
-            logger.exception("VLM POST unexpected failure")
-            meta["attempts"].append({"mode": "json", "error": str(e)})
-            raise RuntimeError(f"VLM POST failed: {e}")
-
-        # If we have event_id, poll GET endpoint for result
-        event_id = meta.get("event_id")
-        if not event_id:
-            raise RuntimeError("No event_id obtained from VLM POST (unexpected)")
-
-        get_url = get_url_template.format(event_id=event_id)
-        logger.info("Polling VLM event result at %s", get_url)
-
-        max_polls = 8
-        poll_delay = 0.5
-        final_text = ""
-        last_response_json = None
-        for attempt in range(max_polls):
-            try:
-                r2 = client.get(get_url, timeout=30.0)
-            except Exception as e_get:
-                logger.warning("GET attempt %d failed: %s", attempt + 1, str(e_get))
-                time.sleep(poll_delay)
-                continue
+            meta["vlm_out_object"] = str(out)[:2000]
+        except Exception:
+            meta["vlm_out_object"] = "<unreprable>"
 
-            if r2.status_code == 204 or not (r2.text and r2.text.strip()):
-                time.sleep(poll_delay)
-                continue
+        logger.info("VLM response object (debug snippet): %s", meta["vlm_out_object"])
 
-            try:
-                r2j = r2.json()
-                last_response_json = r2j
-            except Exception:
-                r2j = None
-
-            text_out = ""
-            if isinstance(r2j, dict):
-                if "data" in r2j and isinstance(r2j["data"], list) and len(r2j["data"]) > 0:
-                    first = r2j["data"][0]
-                    if isinstance(first, dict):
-                        text_out = first.get("text") or first.get("output") or json.dumps(first)
-                    elif isinstance(first, str):
-                        text_out = first
-                text_out = text_out or r2j.get("text") or r2j.get("msg") or r2j.get("output", "") or ""
+    # --- Check whether the remote acknowledged receiving files (expect 1) ---
+    files_seen = None
+    try:
+        if isinstance(out, dict):
+            for key in ("files", "output_files", "files_sent", "uploaded_files", "received_files"):
+                if key in out and isinstance(out[key], (list, tuple)):
+                    files_seen = len(out[key])
+                    break
+
+        if files_seen is None and raw_text:
+            ext_matches = re.findall(r"\.(?:jpg|jpeg|png|bmp|gif)\b", raw_text, flags=re.IGNORECASE)
+            if ext_matches:
+                files_seen = len(ext_matches)
             else:
-                text_out = r2.text or ""
+                matches = re.findall(r"\b(?:uploaded|received|file)\b", raw_text, flags=re.IGNORECASE)
+                if matches:
+                    files_seen = max(1, len(matches))
 
-            if text_out and text_out.strip():
-                final_text = text_out
-                meta["attempts"].append({"mode": "get", "status_code": r2.status_code})
-                break
-            else:
-                time.sleep(poll_delay)
-                continue
+        meta["vlm_files_seen"] = files_seen
+        meta["vlm_file_delivery_ok"] = (files_seen is not None and files_seen >= 1)
+    except Exception:
+        meta["vlm_files_seen"] = None
+        meta["vlm_file_delivery_ok"] = False
 
-        if not final_text:
-            final_text = (r2.text or "").strip()
-            meta["attempts"].append({"mode": "get_last", "status_code": r2.status_code if 'r2' in locals() and r2 is not None else None, "raw": final_text[:500]})
+    if raise_on_file_delivery_failure and not meta["vlm_file_delivery_ok"]:
+        logger.error("VLM did not acknowledge receiving the face file. meta=%s", meta)
+        raise RuntimeError("VLM Space did not acknowledge receiving the face image")
 
-        meta["vlm_raw_len"] = len(final_text)
-        meta["vlm_out_object"] = (final_text[:2000] + "...") if len(final_text) > 2000 else final_text
+    # Log raw VLM output for debugging/auditing
+    logger.info("VLM raw output (length=%d):\n%s", len(raw_text or ""), (raw_text[:1000] + "...") if raw_text and len(raw_text) > 1000 else (raw_text or "<EMPTY>"))
 
-        # Best-effort: detect whether server mentions receiving a file
-        files_seen = None
+    # Try to parse JSON first (fast path)
+    parsed_features = None
+    try:
+        parsed_features = json.loads(raw_text) if raw_text and raw_text.strip() else None
+        if parsed_features is not None and not isinstance(parsed_features, dict):
+            parsed_features = None
+    except Exception:
+        parsed_features = None
+
+    # If json.loads failed or returned None, try regex-based extraction
+    if parsed_features is None and raw_text and raw_text.strip():
         try:
-            if isinstance(last_response_json, dict):
-                for key in ("files", "output_files", "files_sent", "uploaded_files", "received_files"):
-                    if key in last_response_json and isinstance(last_response_json[key], (list, tuple)):
-                        files_seen = len(last_response_json[key])
-                        break
-            if files_seen is None and final_text:
-                ext_matches = re.findall(r"\.(?:jpg|jpeg|png|bmp|gif)\b", final_text, flags=re.IGNORECASE)
-                if ext_matches:
-                    files_seen = len(ext_matches)
-                else:
-                    matches = re.findall(r"\b(?:uploaded|received|file)\b", final_text, flags=re.IGNORECASE)
-                    if matches:
-                        files_seen = max(1, len(matches))
-        except Exception:
-            files_seen = None
+            parsed_features = extract_json_via_regex(raw_text)
+            logger.info("VLM regex-extracted features:\n%s", json.dumps(parsed_features, indent=2, ensure_ascii=False))
+        except Exception as e:
+            logger.info("VLM regex extraction failed or found nothing: %s", str(e))
+            parsed_features = None
 
-        meta["vlm_files_seen"] = files_seen
-        meta["vlm_file_delivery_ok"] = (files_seen is not None and files_seen >= 1)
+    if parsed_features is None:
+        logger.info("VLM parsed features: None (will fallback to sending '{}' or raw string to LLM).")
+    else:
+        logger.info("VLM parsed features (final): %s", json.dumps(parsed_features, ensure_ascii=False))
 
-        parsed_features = None
-        return parsed_features, (final_text or ""), meta
+    # Always return parsed_features (or None), raw_text (string), and meta dict
+    return parsed_features, (raw_text or ""), meta
 
 # -----------------------
 # Gradio / LLM helper (defensive, with retry + clamps)
 # -----------------------
-def get_gradio_client_for_space(space: str) -> Client:
-    if not GRADIO_AVAILABLE:
-        raise RuntimeError("gradio_client not installed in this environment. Add gradio_client to requirements.txt.")
-    if HF_TOKEN:
-        return Client(space, hf_token=HF_TOKEN)
-    return Client(space)
-
 def run_llm_on_vlm(vlm_features_or_raw: Any,
                    max_new_tokens: int = 1024,
                    temperature: float = 0.0,
@@ -640,6 +550,16 @@ def run_llm_on_vlm(vlm_features_or_raw: Any,
 
             return parsed
 
+        except AppError as app_e:
+            logger.exception("LLM AppError (remote validation failed) on attempt %d: %s", attempt, str(app_e))
+            last_exc = app_e
+            if attempt == 1:
+                predict_kwargs["temperature"] = 0.2
+                predict_kwargs["max_new_tokens"] = float(512)
+                logger.info("Retrying LLM call with temperature=0.2 and max_new_tokens=512")
+                continue
+            else:
+                raise RuntimeError(f"LLM call failed (AppError): {app_e}")
         except Exception as e:
             logger.exception("LLM call failed on attempt %d: %s", attempt, str(e))
             last_exc = e
@@ -670,10 +590,9 @@ async def health_check():
     return {
         "status": "healthy",
         "detector": impl,
-        "vlm_available": True,  # we use HTTP POST/GET for VLM
+        "vlm_available": GRADIO_AVAILABLE,
         "vlm_space": GRADIO_VLM_SPACE,
-        "llm_space": LLM_GRADIO_SPACE,
-        "gradio_client_for_llm": GRADIO_AVAILABLE
+        "llm_space": LLM_GRADIO_SPACE
     }
 
 @app.post("/api/v1/validate-eye-photo")
@@ -705,7 +624,7 @@ async def validate_eye_photo(image: UploadFile = File(...)):
                 is_valid = eye_openness_score >= 0.3
                 return {"valid": bool(is_valid), "face_detected": True, "eye_openness_score": round(eye_openness_score, 2),
                         "message_english": "Photo looks good! Eyes are properly open." if is_valid else "Eyes appear to be closed or partially closed. Please open your eyes wide and try again.",
-                        "message_hindi": "फोटो अच्छी ���ै! आंखें ठीक से खुली हैं।" if is_valid else "आंखें बंद या आंशिक रूप से बंद दिखाई दे रही हैं। कृपया अपनी आंखें चौड़ी खोलें और पुनः प्रयास करें।",
+                        "message_hindi": "फोटो अच्छी है! आंखें ठीक से खुली हैं।" if is_valid else "आंखें बंद या आंशिक रूप से बंद दिखाई दे रही हैं। कृपया अपनी आंखें चौड़ी खोलें और पुनः प्रयास करें。",
                         "eye_landmarks": {"left_eye": left_eye, "right_eye": right_eye}}
             except Exception:
                 traceback.print_exc()
@@ -729,7 +648,7 @@ async def validate_eye_photo(image: UploadFile = File(...)):
             is_valid = eye_openness_score >= 0.3
             return {"valid": bool(is_valid), "face_detected": True, "eye_openness_score": round(eye_openness_score, 2),
                     "message_english": "Photo looks good! Eyes are properly open." if is_valid else "Eyes appear to be closed or partially closed. Please open your eyes wide and try again.",
-                    "message_hindi": "फोटो अच्छी है! आंखें ठीक से खुली हैं।" if is_valid else "आंखें बंद या आंशिक रूप से बंद दिखाई दे रही हैं। कृपया अपनी आंखें चौड़ी खोलें और पुनः प्रयास करें।",
+                    "message_hindi": "फोटो अच्छी है! आंखें ठीक से खुली हैं।" if is_valid else "आंखें बंद या आंशिक रूप से बंद दिखाई दे रही हैं। कृपया अपनी आंखें चौड़ी खोलें और पुनः प्रयास करें。",
                     "eye_landmarks": {"left_eye": left_eye, "right_eye": right_eye}}
 
         if isinstance(mtcnn, dict) and mtcnn.get("impl") == "opencv":
@@ -756,7 +675,7 @@ async def validate_eye_photo(image: UploadFile = File(...)):
                     left_eye = {"x": cx, "y": cy}
                 return {"valid": bool(is_valid), "face_detected": True, "eye_openness_score": round(eye_openness_score, 2),
                         "message_english": "Photo looks good! Eyes are detected." if is_valid else "Eyes not detected. Please open your eyes wide and try again.",
-                        "message_hindi": "फोटो अच्छी है! आंखें मिलीं।" if is_valid else "आंखें नहीं मिलीं। कृपया अपनी आंखें चौड़ी खोलें और पुनः प्रयास करें।",
+                        "message_hindi": "फोटो अच्छी है! आंखें मिलीं।" if is_valid else "आंखें नहीं मिलीं। कृपया अपनी आंखें चौड़ी खोलें और पुनः प्रयास करें。",
                         "eye_landmarks": {"left_eye": left_eye, "right_eye": right_eye}}
             except Exception:
                 traceback.print_exc()
@@ -858,8 +777,7 @@ async def get_vitals_from_upload(
     Note: VLM will receive only the face image (not the eye image).
     """
     if not GRADIO_AVAILABLE:
-        # LLM call requires gradio_client to be installed. If not present, user must install it.
-        raise HTTPException(status_code=500, detail="LLM client (gradio_client) not available in this deployment.")
+        raise HTTPException(status_code=500, detail="VLM/LLM client not available in this deployment.")
 
     # save files to a temp directory
     try:
@@ -970,6 +888,8 @@ class ImageUrls(BaseModel):
     face_image_url: HttpUrl
     eye_image_url: HttpUrl
 
+import httpx  # make sure to add httpx to requirements
+
 # helper: download URL to file with safety checks
 async def download_image_to_path(url: str, dest_path: str, max_bytes: int = 5_000_000, timeout_seconds: int = 10) -> None:
     """
@@ -1011,8 +931,7 @@ async def get_vitals_from_urls(payload: ImageUrls = Body(...)):
     Body: { "face_image_url": "...", "eye_image_url": "..." }
     """
     if not GRADIO_AVAILABLE:
-        # LLM call requires gradio_client to be installed
-        raise HTTPException(status_code=500, detail="LLM client (gradio_client) not available in this deployment.")
+        raise HTTPException(status_code=500, detail="VLM/LLM client not available in this deployment.")
 
     # prepare tmp paths
     try:
@@ -1256,4 +1175,4 @@ async def process_screening(screening_id: str):
 # -----------------------
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run("app:app", host="0.0.0.0", port=7860, reload=False)
+    uvicorn.run("app:app", host="0.0.0.0", port=7860, reload=False)