Update app.py

app.py

@@ -3,18 +3,17 @@
 Elderly HealthWatch AI Backend (FastAPI)
 Pipeline:
  - receive images
- - run VLM (remote gradio / chat_fn) -> JSON feature vector + raw text + meta
+ - run VLM (remote HF Space via gradio_api/call/chat style) -> raw text + meta
  - run LLM (remote gradio /chat) -> structured risk JSON (per requested schema)
  - continue rest of processing and store results
 
 Notes:
- - Add gradio_client==1.13.2 (or another compatible 1.x) to requirements.txt
+ - Add gradio_client==1.13.2 (or another compatible 1.x) to requirements.txt for LLM calls
+ - Add httpx to requirements.txt for VLM POST/GET flow
  - If VLM/LLM Spaces are private, set HF_TOKEN in the environment for authentication.
  - This variant:
-    * logs raw VLM responses,
-    * always returns raw VLM output in API responses,
-    * extracts JSON from VLM via regex when possible, and
-    * sends only the face image to the VLM (not the eye image).
+    * sends the face image to the HF Space using the POST/GET event flow
+    * returns raw VLM output and meta (no VLM-side JSON extraction)
 """
 
 import io
@@ -26,6 +25,7 @@ import logging
 import traceback
 import re
 import time
+import base64
 from typing import Dict, Any, Optional, Tuple
 from datetime import datetime
 
@@ -36,7 +36,10 @@ from PIL import Image
 import numpy as np
 import cv2  # opencv-python-headless expected installed
 
-# Optional gradio client (for VLM + LLM calls)
+# HTTP client for the POST/GET event style VLM calls
+import httpx
+
+# Optional gradio client (for LLM calls)
 try:
     from gradio_client import Client, handle_file  # type: ignore
     GRADIO_AVAILABLE = True
@@ -246,156 +249,193 @@ def extract_json_via_regex(raw_text: str) -> Dict[str, Any]:
     return out
 
 # -----------------------
-# Gradio / VLM helper (sends only face image, returns meta)
+# VLM helper using HF Spaces POST/GET event flow (gradio_api/call/chat)
 # -----------------------
-def get_gradio_client_for_space(space: str) -> Client:
-    if not GRADIO_AVAILABLE:
-        raise RuntimeError("gradio_client not installed in this environment. Add gradio_client to requirements.txt.")
-    if HF_TOKEN:
-        return Client(space, hf_token=HF_TOKEN)
-    return Client(space)
-
 def run_vlm_and_get_features(face_path: str, eye_path: Optional[str] = None, prompt: Optional[str] = None,
                              raise_on_file_delivery_failure: bool = False
                              ) -> Tuple[Optional[Dict[str, Any]], str, Dict[str, Any]]:
     """
-    Synchronous call to remote VLM (gradio /chat_fn). Sends ONLY the face image file.
-    Returns tuple: (parsed_features_dict_or_None, raw_text_response_str, meta)
-    meta includes:
-      - vlm_file_delivery_ok (bool)  # expects ≥1 file acknowledged (face)
-      - vlm_files_seen (int or None)
-      - vlm_raw_len (int)
-      - vlm_out_object (short repr)
+    VLM caller using the HF Spaces 'gradio_api/call/chat' style:
+      1) POST -> returns an EVENT_ID
+      2) GET /gradio_api/call/chat/{EVENT_ID} -> fetch result
+
+    This function:
+      - Loads face image, encodes as base64 and embeds in JSON payload as a single file
+      - POSTs to the Space endpoint to create an event
+      - GETs the event result and extracts text/output
+      - Returns (parsed_features_or_None, raw_text, meta)
+    NOTE: The function returns parsed_features=None (no JSON extraction here) and raw_text for LLM downstream.
     """
     prompt = prompt or DEFAULT_VLM_PROMPT
 
-    
     if not os.path.exists(face_path):
         raise FileNotFoundError(f"Face image not found at: {face_path}")
-    if not os.path.exists(eye_path):
-        raise FileNotFoundError(f"Eye image not found at: {eye_path}")
-    
-    face_size = os.path.getsize(face_path)
-    eye_size = os.path.getsize(eye_path)
-    logger.info(f"VLM input files - Face: {face_size} bytes, Eye: {eye_size} bytes")
-    
-    if face_size == 0 or eye_size == 0:
-        raise ValueError("One or both images are empty (0 bytes)")
-    
-    if not GRADIO_AVAILABLE:
-        raise RuntimeError("gradio_client not available in this environment.")
-
-    client = get_gradio_client_for_space(GRADIO_VLM_SPACE)
-    
-    # Verify files can be opened as images
-    try:
-        Image.open(face_path).verify()
-        Image.open(eye_path).verify()
-        logger.info("Both images verified as valid")
-    except Exception as e:
-        raise ValueError(f"Invalid image file(s): {e}")
-    
-    message = {"text": prompt, "files": [handle_file(face_path), handle_file(eye_path)]}
-    
-    logger.info(f"Calling VLM with message structure: text={len(prompt)} chars, files=2")
-    client = get_gradio_client_for_space(GRADIO_VLM_SPACE)
-    # NOTE: only send face image to the Space
 
-    message = {"text": prompt, "files": [handle_file(face_path)]}
+    # Read and base64-encode the face image for embedding in JSON
+    with open(face_path, "rb") as f:
+        face_bytes = f.read()
+    if not face_bytes:
+        raise ValueError("Face image is empty (0 bytes)")
 
-    meta: Dict[str, Any] = {"vlm_file_delivery_ok": False, "vlm_files_seen": None, "vlm_raw_len": 0, "vlm_out_object": None}
+    face_b64 = base64.b64encode(face_bytes).decode("ascii")
+    # prefix with MIME type (assume jpeg)
+    face_data_uri = f"data:image/jpeg;base64,{face_b64}"
 
-    # SINGLE CALL (no retries)
-    try:
-        logger.info("Calling VLM Space %s with 1 file (face only)", GRADIO_VLM_SPACE)
-        result = client.predict(message=message, history=[], api_name="/chat_fn")
-    except Exception as e:
-        logger.exception("VLM call failed (no retries)")
-        raise RuntimeError(f"VLM call failed: {e}")
+    # Build the JSON payload consistent with gradio multimodal style:
+    payload = {
+        "data": [
+            {
+                "text": prompt,
+                "files": [face_data_uri]
+            }
+        ]
+    }
 
-    # Normalize result
-    raw_text = ""
-    out = None
-    if not result:
-        logger.warning("VLM returned empty result object")
+    # Prepare endpoint(s)
+    SPACE_HOST = os.getenv("VLM_SPACE_HOST")  # optional full host override
+    if SPACE_HOST:
+        base_url = SPACE_HOST.rstrip("/")
     else:
-        if isinstance(result, (list, tuple)):
-            out = result[0]
-        elif isinstance(result, dict):
-            out = result
+        # infer from GRADIO_VLM_SPACE if it's of form "owner/space-name"
+        # Many public HF spaces also map to {owner}-{space}.hf.space
+        if "/" in GRADIO_VLM_SPACE:
+            base_url = f"https://{GRADIO_VLM_SPACE.replace('/', '-')}.hf.space"
         else:
-            out = {"text": str(result)}
+            base_url = f"https://{GRADIO_VLM_SPACE}.hf.space"
 
-        text_out = out.get("text") or out.get("output") or ""
-        raw_text = text_out or ""
-        meta["vlm_raw_len"] = len(raw_text or "")
-        try:
-            meta["vlm_out_object"] = str(out)[:2000]
-        except Exception:
-            meta["vlm_out_object"] = "<unreprable>"
+    post_url = f"{base_url}/gradio_api/call/chat"
+    get_url_template = f"{base_url}/gradio_api/call/chat/{{event_id}}"
 
-        logger.info("VLM response object (debug snippet): %s", meta["vlm_out_object"])
+    headers = {"Content-Type": "application/json"}
+    if HF_TOKEN:
+        headers["Authorization"] = f"Bearer {HF_TOKEN}"
+
+    meta: Dict[str, Any] = {
+        "vlm_file_delivery_ok": False,
+        "vlm_files_seen": None,
+        "vlm_raw_len": 0,
+        "vlm_out_object": None,
+        "post_url": post_url
+    }
 
-    # --- Check whether the remote acknowledged receiving files (expect 1) ---
-    files_seen = None
     try:
-        if isinstance(out, dict):
-            for key in ("files", "output_files", "files_sent", "uploaded_files", "received_files"):
-                if key in out and isinstance(out[key], (list, tuple)):
-                    files_seen = len(out[key])
-                    break
+        logger.info("VLM POST -> %s (payload text len=%d, files=1)", post_url, len(prompt))
+        with httpx.Client(timeout=30.0) as client:
+            resp = client.post(post_url, headers=headers, json=payload)
+            resp.raise_for_status()
 
-        if files_seen is None and raw_text:
-            ext_matches = re.findall(r"\.(?:jpg|jpeg|png|bmp|gif)\b", raw_text, flags=re.IGNORECASE)
-            if ext_matches:
-                files_seen = len(ext_matches)
-            else:
-                matches = re.findall(r"\b(?:uploaded|received|file)\b", raw_text, flags=re.IGNORECASE)
-                if matches:
-                    files_seen = max(1, len(matches))
+            # Try to robustly extract an event id from the POST response
+            event_id = None
+            try:
+                rj = resp.json()
+            except Exception:
+                rj = {}
+
+            if isinstance(rj, dict):
+                event_id = rj.get("event_id") or rj.get("id") or rj.get("job")
+            if not event_id:
+                # try to extract using regex from resp.text
+                m = re.search(r'"([^"]{8,})"', resp.text or "")
+                if m:
+                    event_id = m.group(1)
+            if not event_id:
+                parts = re.split(r'"', resp.text or "")
+                if len(parts) >= 5:
+                    event_id_candidate = parts[3].strip()
+                    if event_id_candidate:
+                        event_id = event_id_candidate
+            if not event_id:
+                raise RuntimeError(f"Failed to obtain EVENT_ID from VLM POST response: {resp.text[:1000]}")
+
+            meta["event_id"] = event_id
+            logger.info("VLM event created: %s", event_id)
+
+            # Poll the GET result endpoint
+            get_url = get_url_template.format(event_id=event_id)
+            logger.info("Polling VLM event result at %s", get_url)
+
+            max_polls = 6
+            poll_delay = 0.5
+            final_text = ""
+            last_response_json = None
+            for attempt in range(max_polls):
+                r2 = client.get(get_url, headers=headers, timeout=30.0)
+                if r2.status_code == 204 or not (r2.text and r2.text.strip()):
+                    time.sleep(poll_delay)
+                    continue
+                try:
+                    r2j = r2.json()
+                    last_response_json = r2j
+                except Exception:
+                    r2j = None
+
+                text_out = ""
+                if isinstance(r2j, dict):
+                    if "data" in r2j and isinstance(r2j["data"], list) and len(r2j["data"]) > 0:
+                        first = r2j["data"][0]
+                        if isinstance(first, dict):
+                            text_out = first.get("text") or first.get("output") or json.dumps(first)
+                        elif isinstance(first, str):
+                            text_out = first
+                    text_out = text_out or r2j.get("text") or r2j.get("msg") or r2j.get("output", "") or ""
+                else:
+                    text_out = r2.text or ""
 
-        meta["vlm_files_seen"] = files_seen
-        meta["vlm_file_delivery_ok"] = (files_seen is not None and files_seen >= 1)
-    except Exception:
-        meta["vlm_files_seen"] = None
-        meta["vlm_file_delivery_ok"] = False
+                if text_out and text_out.strip():
+                    final_text = text_out
+                    break
+                else:
+                    time.sleep(poll_delay)
+                    continue
 
-    if raise_on_file_delivery_failure and not meta["vlm_file_delivery_ok"]:
-        logger.error("VLM did not acknowledge receiving the face file. meta=%s", meta)
-        raise RuntimeError("VLM Space did not acknowledge receiving the face image")
+            if not final_text:
+                final_text = (r2.text or "").strip()
 
-    # Log raw VLM output for debugging/auditing
-    logger.info("VLM raw output (length=%d):\n%s", len(raw_text or ""), (raw_text[:1000] + "...") if raw_text and len(raw_text) > 1000 else (raw_text or "<EMPTY>"))
+            meta["vlm_raw_len"] = len(final_text)
+            meta["vlm_out_object"] = (final_text[:2000] + "...") if len(final_text) > 2000 else final_text
 
-    # Try to parse JSON first (fast path)
-    parsed_features = None
-    try:
-        parsed_features = json.loads(raw_text) if raw_text and raw_text.strip() else None
-        if parsed_features is not None and not isinstance(parsed_features, dict):
-            parsed_features = None
-    except Exception:
-        parsed_features = None
+            # Best-effort: detect whether server mentions receiving a file
+            files_seen = None
+            try:
+                if isinstance(last_response_json, dict):
+                    for key in ("files", "output_files", "files_sent", "uploaded_files", "received_files"):
+                        if key in last_response_json and isinstance(last_response_json[key], (list, tuple)):
+                            files_seen = len(last_response_json[key])
+                            break
+                if files_seen is None and final_text:
+                    ext_matches = re.findall(r"\.(?:jpg|jpeg|png|bmp|gif)\b", final_text, flags=re.IGNORECASE)
+                    if ext_matches:
+                        files_seen = len(ext_matches)
+                    else:
+                        matches = re.findall(r"\b(?:uploaded|received|file)\b", final_text, flags=re.IGNORECASE)
+                        if matches:
+                            files_seen = max(1, len(matches))
+            except Exception:
+                files_seen = None
 
-    # If json.loads failed or returned None, try regex-based extraction
-    if parsed_features is None and raw_text and raw_text.strip():
-        try:
-            parsed_features = extract_json_via_regex(raw_text)
-            logger.info("VLM regex-extracted features:\n%s", json.dumps(parsed_features, indent=2, ensure_ascii=False))
-        except Exception as e:
-            logger.info("VLM regex extraction failed or found nothing: %s", str(e))
-            parsed_features = None
+            meta["vlm_files_seen"] = files_seen
+            meta["vlm_file_delivery_ok"] = (files_seen is not None and files_seen >= 1)
 
-    if parsed_features is None:
-        logger.info("VLM parsed features: None (will fallback to sending '{}' or raw string to LLM).")
-    else:
-        logger.info("VLM parsed features (final): %s", json.dumps(parsed_features, ensure_ascii=False))
+            parsed_features = None
+            return parsed_features, (final_text or ""), meta
 
-    # Always return parsed_features (or None), raw_text (string), and meta dict
-    return parsed_features, (raw_text or ""), meta
+    except httpx.HTTPStatusError as he:
+        logger.exception("VLM HTTP error")
+        raise RuntimeError(f"VLM http error: {he.response.status_code} {str(he)}")
+    except Exception as e:
+        logger.exception("VLM call (httpx) failed")
+        raise RuntimeError(f"VLM call failed: {e}")
 
 # -----------------------
 # Gradio / LLM helper (defensive, with retry + clamps)
 # -----------------------
+def get_gradio_client_for_space(space: str) -> Client:
+    if not GRADIO_AVAILABLE:
+        raise RuntimeError("gradio_client not installed in this environment. Add gradio_client to requirements.txt.")
+    if HF_TOKEN:
+        return Client(space, hf_token=HF_TOKEN)
+    return Client(space)
+
 def run_llm_on_vlm(vlm_features_or_raw: Any,
                    max_new_tokens: int = 1024,
                    temperature: float = 0.0,
@@ -550,16 +590,6 @@ def run_llm_on_vlm(vlm_features_or_raw: Any,
 
             return parsed
 
-        except AppError as app_e:
-            logger.exception("LLM AppError (remote validation failed) on attempt %d: %s", attempt, str(app_e))
-            last_exc = app_e
-            if attempt == 1:
-                predict_kwargs["temperature"] = 0.2
-                predict_kwargs["max_new_tokens"] = float(512)
-                logger.info("Retrying LLM call with temperature=0.2 and max_new_tokens=512")
-                continue
-            else:
-                raise RuntimeError(f"LLM call failed (AppError): {app_e}")
         except Exception as e:
             logger.exception("LLM call failed on attempt %d: %s", attempt, str(e))
             last_exc = e
@@ -590,9 +620,10 @@ async def health_check():
     return {
         "status": "healthy",
         "detector": impl,
-        "vlm_available": GRADIO_AVAILABLE,
+        "vlm_available": True,  # we use HTTP POST/GET for VLM
         "vlm_space": GRADIO_VLM_SPACE,
-        "llm_space": LLM_GRADIO_SPACE
+        "llm_space": LLM_GRADIO_SPACE,
+        "gradio_client_for_llm": GRADIO_AVAILABLE
     }
 
 @app.post("/api/v1/validate-eye-photo")
@@ -624,7 +655,7 @@ async def validate_eye_photo(image: UploadFile = File(...)):
                 is_valid = eye_openness_score >= 0.3
                 return {"valid": bool(is_valid), "face_detected": True, "eye_openness_score": round(eye_openness_score, 2),
                         "message_english": "Photo looks good! Eyes are properly open." if is_valid else "Eyes appear to be closed or partially closed. Please open your eyes wide and try again.",
-                        "message_hindi": "फोटो अच्छी है! आंखें ठीक से खुली हैं।" if is_valid else "आंखें बंद या आंशिक रूप से बंद दिखाई दे रही हैं। कृपया अपनी आंखें चौड़ी खोलें और पुनः प्रयास करें。",
+                        "message_hindi": "फोटो अच्छी है! आंखें ठीक से खुली हैं।" if is_valid else "आंखें बंद या आंशिक रूप से बंद दिखाई दे रही हैं। कृपया अपनी आंखें चौड़ी खोलें और पुनः प्रयास करें।",
                         "eye_landmarks": {"left_eye": left_eye, "right_eye": right_eye}}
             except Exception:
                 traceback.print_exc()
@@ -648,7 +679,7 @@ async def validate_eye_photo(image: UploadFile = File(...)):
             is_valid = eye_openness_score >= 0.3
             return {"valid": bool(is_valid), "face_detected": True, "eye_openness_score": round(eye_openness_score, 2),
                     "message_english": "Photo looks good! Eyes are properly open." if is_valid else "Eyes appear to be closed or partially closed. Please open your eyes wide and try again.",
-                    "message_hindi": "फोटो अच्छी है! आंखें ठीक से खुली हैं।" if is_valid else "आंखें बंद या आंशिक रूप से बंद दिखाई दे रही हैं। कृपया अपनी आंखें चौड़ी खोलें और पुनः प्रयास करें。",
+                    "message_hindi": "फोटो अच्छी है! आंखें ठीक से खुली हैं।" if is_valid else "आंखें बंद या आंशिक रूप से बंद दिखाई दे रही हैं। कृपया अपनी आंखें चौड़ी खोलें और पुनः प्रयास करें।",
                     "eye_landmarks": {"left_eye": left_eye, "right_eye": right_eye}}
 
         if isinstance(mtcnn, dict) and mtcnn.get("impl") == "opencv":
@@ -675,7 +706,7 @@ async def validate_eye_photo(image: UploadFile = File(...)):
                     left_eye = {"x": cx, "y": cy}
                 return {"valid": bool(is_valid), "face_detected": True, "eye_openness_score": round(eye_openness_score, 2),
                         "message_english": "Photo looks good! Eyes are detected." if is_valid else "Eyes not detected. Please open your eyes wide and try again.",
-                        "message_hindi": "फोटो अच्छी है! आंखें मिलीं।" if is_valid else "आंखें नहीं मिलीं। कृपया अपनी आंखें चौड़ी खोलें और पुनः प्रयास करें。",
+                        "message_hindi": "फोटो अच्छी है! आंखें मिलीं।" if is_valid else "आंखें नहीं मिलीं। कृपया अपनी आंखें चौड़ी खोलें और पुनः प्रयास करें।",
                         "eye_landmarks": {"left_eye": left_eye, "right_eye": right_eye}}
             except Exception:
                 traceback.print_exc()
@@ -777,7 +808,8 @@ async def get_vitals_from_upload(
     Note: VLM will receive only the face image (not the eye image).
     """
     if not GRADIO_AVAILABLE:
-        raise HTTPException(status_code=500, detail="VLM/LLM client not available in this deployment.")
+        # LLM call requires gradio_client to be installed. If not present, user must install it.
+        raise HTTPException(status_code=500, detail="LLM client (gradio_client) not available in this deployment.")
 
     # save files to a temp directory
     try:
@@ -888,8 +920,6 @@ class ImageUrls(BaseModel):
     face_image_url: HttpUrl
     eye_image_url: HttpUrl
 
-import httpx  # make sure to add httpx to requirements
-
 # helper: download URL to file with safety checks
 async def download_image_to_path(url: str, dest_path: str, max_bytes: int = 5_000_000, timeout_seconds: int = 10) -> None:
     """
@@ -931,7 +961,8 @@ async def get_vitals_from_urls(payload: ImageUrls = Body(...)):
     Body: { "face_image_url": "...", "eye_image_url": "..." }
     """
     if not GRADIO_AVAILABLE:
-        raise HTTPException(status_code=500, detail="VLM/LLM client not available in this deployment.")
+        # LLM call requires gradio_client to be installed
+        raise HTTPException(status_code=500, detail="LLM client (gradio_client) not available in this deployment.")
 
     # prepare tmp paths
     try: