Update app.py

app.py

@@ -4,6 +4,7 @@ Elderly HealthWatch AI Backend (FastAPI)
 This variant uses:
 - facenet-pytorch or mtcnn if available
 - otherwise falls back to OpenCV Haar cascades (fast, CPU-only, lightweight)
+- integrates a remote VLM via gradio_client to get JSON feature vectors
 """
 
 import io
@@ -18,6 +19,28 @@ import numpy as np
 import os
 import traceback
 import cv2  # opencv-python-headless expected installed
+import json
+import logging
+
+# Optional gradio client import (for VLM)
+try:
+    from gradio_client import Client, handle_file
+    GRADIO_AVAILABLE = True
+except Exception:
+    GRADIO_AVAILABLE = False
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+
+# Configuration for remote VLM (change to your target Space)
+GRADIO_SPACE = os.getenv("GRADIO_SPACE", "developer0hye/Qwen3-VL-8B-Instruct")
+# If your space is private, set HF_TOKEN as a secret / env var in Spaces
+HF_TOKEN = os.getenv("HF_TOKEN", None)
+DEFAULT_VLM_PROMPT = (
+    "From the provided face/eye images, compute the required screening features "
+    "(pallor, sclera yellowness, redness, mobility metrics, quality checks) "
+    "and output a clean JSON feature vector only."
+)
 
 # Attempt to import facenet-pytorch MTCNN first (recommended)
 try:
@@ -102,6 +125,92 @@ def estimate_eye_openness_from_detection(confidence: float) -> float:
     except Exception:
         return 0.0
 
+# --------------------------
+# VLM client helper
+# --------------------------
+def get_gradio_client():
+    """Return a configured gradio Client or raise if not available."""
+    if not GRADIO_AVAILABLE:
+        raise RuntimeError("gradio_client not installed in this environment.")
+    if HF_TOKEN:
+        return Client(GRADIO_SPACE, hf_token=HF_TOKEN)
+    return Client(GRADIO_SPACE)
+
+def run_vlm_and_get_features(face_path: str, eye_path: str, prompt: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Synchronous call to the remote VLM (gradio / chat_fn).
+    Expects the VLM to return a JSON string only (we parse it).
+    On success returns a dict (parsed JSON).
+    On failure raises RuntimeError or ValueError.
+    """
+    prompt = prompt or DEFAULT_VLM_PROMPT
+
+    if not os.path.exists(face_path) or not os.path.exists(eye_path):
+        raise FileNotFoundError("Face or eye image path missing for VLM call.")
+
+    if not GRADIO_AVAILABLE:
+        raise RuntimeError("gradio_client is not available in this environment.")
+
+    client = get_gradio_client()
+
+    message = {
+        "text": prompt,
+        "files": [handle_file(face_path), handle_file(eye_path)]
+    }
+
+    # Call the remote API
+    try:
+        logging.info("Calling remote VLM at %s", GRADIO_SPACE)
+        # result is typically a tuple: (output_dict, new_history)
+        result = client.predict(message=message, history=[], api_name="/chat_fn")
+    except Exception as e:
+        logging.exception("VLM call failed")
+        raise RuntimeError(f"VLM call failed: {e}")
+
+    # Extract text output
+    if not result or not isinstance(result, (list, tuple)):
+        # some spaces return just a dict too
+        if isinstance(result, dict):
+            out = result
+        else:
+            raise RuntimeError("Unexpected VLM response shape")
+    else:
+        out = result[0]
+
+    if not isinstance(out, dict):
+        raise RuntimeError("Unexpected VLM output format (expected dict with 'text' key)")
+
+    text_out = out.get("text") or out.get("output") or None
+    if not text_out:
+        raise RuntimeError("VLM returned empty text output")
+
+    # The model was instructed to return JSON only. Try to parse it.
+    try:
+        features = json.loads(text_out)
+    except Exception:
+        # attempt a forgiving extraction: find the first { ... } block
+        try:
+            s = text_out
+            first = s.find("{")
+            last = s.rfind("}")
+            if first != -1 and last != -1 and last > first:
+                maybe = s[first:last+1]
+                features = json.loads(maybe)
+            else:
+                raise
+        except Exception as e:
+            logging.exception("Failed to parse JSON from VLM output")
+            raise ValueError(f"Failed to parse JSON from VLM output: {e}\nRaw output: {text_out}")
+
+    if not isinstance(features, dict):
+        raise ValueError("Parsed VLM output is not a JSON object/dict")
+
+    return features
+
+# --------------------------
+# End VLM helper
+# --------------------------
+
 @app.get("/")
 async def read_root():
     return {"message": "Elderly HealthWatch AI Backend"}
@@ -115,7 +224,7 @@ async def health_check():
         impl = "opencv_haar_fallback"
     else:
         impl = _MTCNN_IMPL
-    return {"status": "healthy", "detector": impl}
+    return {"status": "healthy", "detector": impl, "vlm_available": GRADIO_AVAILABLE}
 
 @app.post("/api/v1/validate-eye-photo")
 async def validate_eye_photo(image: UploadFile = File(...)):
@@ -398,7 +507,23 @@ async def process_screening(screening_id: str):
         }
         screenings_db[screening_id]["quality_metrics"] = quality_metrics
 
-        # Simulate VLM/medical model steps (kept short)
+        # Attempt VLM call to compute multimodal features (pallor, sclera yellowness, etc.)
+        try:
+            vlm_features = run_vlm_and_get_features(face_path, eye_path)
+            # attach under ai_results
+            screenings_db[screening_id].setdefault("ai_results", {})
+            screenings_db[screening_id]["ai_results"].update({
+                "vlm_features": vlm_features
+            })
+        except Exception as e:
+            # Don't fail the entire pipeline for VLM errors; record them
+            logging.exception("VLM feature extraction failed")
+            screenings_db[screening_id].setdefault("ai_results", {})
+            screenings_db[screening_id]["ai_results"].update({
+                "vlm_error": str(e)
+            })
+
+        # Simulate Medical model steps (kept short)
         await asyncio.sleep(1)
         vlm_face_desc = "Patient appears to have normal facial tone; no severe jaundice visible."
         vlm_eye_desc = "Sclera shows mild yellowing."
@@ -427,7 +552,9 @@ async def process_screening(screening_id: str):
             "medical_insights": medical_insights,
             "processing_time_ms": 1200
         }
-        screenings_db[screening_id]["ai_results"] = ai_results
+        # Merge ai_results while preserving vlm_features if present
+        screenings_db[screening_id].setdefault("ai_results", {})
+        screenings_db[screening_id]["ai_results"].update(ai_results)
 
         disease_predictions = [
             {