dpv007 commited on
Commit
9301350
·
verified ·
1 Parent(s): 196daa0

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +265 -201
app.py CHANGED
@@ -1,6 +1,6 @@
1
  """
2
- Elderly HealthWatch AI Backend (FastAPI) - Refactored with Qwen2.5-VL
3
- Updated to use Qwen2.5-VL with multiple space options and fallback
4
  """
5
 
6
  import io
@@ -12,7 +12,7 @@ import logging
12
  import traceback
13
  import re
14
  from typing import Dict, Any, Optional, Tuple
15
- from datetime import datetime
16
 
17
  from fastapi import FastAPI, UploadFile, File, BackgroundTasks, HTTPException
18
  from fastapi.middleware.cors import CORSMiddleware
@@ -20,6 +20,15 @@ from PIL import Image
20
  import numpy as np
21
  import cv2
22
 
 
 
 
 
 
 
 
 
 
23
  try:
24
  from gradio_client import Client, handle_file
25
  GRADIO_AVAILABLE = True
@@ -32,36 +41,18 @@ except Exception:
32
  logging.basicConfig(level=logging.INFO)
33
  logger = logging.getLogger("elderly_healthwatch")
34
 
35
- # Multiple VLM options - will try in order until one works
36
- VLM_SPACES = [
37
- {
38
- "space": "Qwen/Qwen2.5-VL-7B-Instruct",
39
- "model_id": "Qwen/Qwen2.5-VL-7B-Instruct",
40
- "api_name": "/model_chat",
41
- "type": "official"
42
- },
43
- {
44
- "space": "mrdbourke/Qwen2.5-VL-Instruct-Demo",
45
- "model_id": "Qwen/Qwen2.5-VL-7B-Instruct",
46
- "api_name": "/run_example",
47
- "type": "demo"
48
- }
49
- ]
50
-
51
- GRADIO_VLM_SPACE = os.getenv("GRADIO_SPACE", VLM_SPACES[0]["space"])
52
- VLM_MODEL_ID = os.getenv("VLM_MODEL_ID", "Qwen/Qwen2.5-VL-7B-Instruct")
53
  LLM_GRADIO_SPACE = os.getenv("LLM_GRADIO_SPACE", "Tonic/med-gpt-oss-20b-demo")
54
  HF_TOKEN = os.getenv("HF_TOKEN", None)
55
 
 
 
 
 
56
  DEFAULT_VLM_PROMPT = (
57
- "Analyze this person's face and eyes carefully. Look for signs of: "
58
- "1) Pallor (pale skin indicating possible anemia), "
59
- "2) Sclera yellowness (yellowing of eye whites indicating possible jaundice), "
60
- "3) Eye redness or irritation, "
61
- "4) Skin hydration and texture. "
62
- "Provide your analysis in JSON format with these keys: "
63
- "pallor_score (0-1), sclera_yellowness (0-1), redness (0-1), "
64
- "hydration_issue (0-1), overall_quality (0-1), notes."
65
  )
66
 
67
  LLM_SYSTEM_PROMPT = (
@@ -82,26 +73,70 @@ os.makedirs(TMP_DIR, exist_ok=True)
82
  # In-memory database
83
  screenings_db: Dict[str, Dict[str, Any]] = {}
84
 
85
- # Track which VLM space is working
86
- active_vlm_config = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
 
88
  # ============================================================================
89
  # Face Detection Setup
90
  # ============================================================================
91
  def setup_face_detector():
92
  """Initialize face detector (MTCNN or OpenCV fallback)"""
 
93
  try:
94
  from facenet_pytorch import MTCNN
95
  return MTCNN(keep_all=False, device="cpu"), "facenet_pytorch"
96
  except Exception:
97
  pass
98
 
 
99
  try:
100
  from mtcnn import MTCNN
101
  return MTCNN(), "mtcnn"
102
  except Exception:
103
  pass
104
 
 
105
  try:
106
  face_path = os.path.join(cv2.data.haarcascades, "haarcascade_frontalface_default.xml")
107
  eye_path = os.path.join(cv2.data.haarcascades, "haarcascade_eye.xml")
@@ -158,6 +193,7 @@ def detect_face_and_eyes(pil_img: Image.Image) -> Dict[str, Any]:
158
 
159
  img_arr = np.asarray(pil_img)
160
 
 
161
  if detector_type == "facenet_pytorch":
162
  try:
163
  boxes, probs, landmarks = face_detector.detect(pil_img, landmarks=True)
@@ -185,6 +221,7 @@ def detect_face_and_eyes(pil_img: Image.Image) -> Dict[str, Any]:
185
  return {"face_detected": False, "face_confidence": 0.0, "eye_openness_score": 0.0,
186
  "left_eye": None, "right_eye": None}
187
 
 
188
  elif detector_type == "mtcnn":
189
  try:
190
  detections = face_detector.detect_faces(img_arr)
@@ -208,6 +245,7 @@ def detect_face_and_eyes(pil_img: Image.Image) -> Dict[str, Any]:
208
  return {"face_detected": False, "face_confidence": 0.0, "eye_openness_score": 0.0,
209
  "left_eye": None, "right_eye": None}
210
 
 
211
  elif detector_type == "opencv":
212
  try:
213
  gray = cv2.cvtColor(img_arr, cv2.COLOR_RGB2GRAY)
@@ -299,7 +337,7 @@ def extract_json_from_llm_output(raw_text: str) -> Dict[str, Any]:
299
  }
300
 
301
  # ============================================================================
302
- # VLM & LLM Integration - UPDATED FOR QWEN2.5-VL with Fallback
303
  # ============================================================================
304
  def get_gradio_client(space: str) -> Client:
305
  """Get Gradio client with optional auth"""
@@ -307,128 +345,194 @@ def get_gradio_client(space: str) -> Client:
307
  raise RuntimeError("gradio_client not installed")
308
  return Client(space, hf_token=HF_TOKEN) if HF_TOKEN else Client(space)
309
 
310
- def call_vlm_single_image(client: Client, image_path: str, prompt: str, config: Dict) -> str:
311
- """Call VLM with a single image using appropriate API"""
312
- try:
313
- if config["type"] == "demo":
314
- # mrdbourke style API
315
- result = client.predict(
316
- image=handle_file(image_path),
317
- text_input=prompt,
318
- model_id=config["model_id"],
319
- api_name=config["api_name"]
320
- )
321
- # Extract text from tuple response
322
- if isinstance(result, (list, tuple)) and len(result) > 0:
323
- return str(result[0])
324
- return str(result)
325
- else:
326
- # Try official Qwen space API (if it exists)
327
- result = client.predict(
328
- query=prompt,
329
- image=handle_file(image_path),
330
- api_name=config["api_name"]
331
- )
332
- if isinstance(result, (list, tuple)) and len(result) > 0:
333
- return str(result[0])
334
- return str(result)
335
- except Exception as e:
336
- logger.error("VLM single image call failed with config %s: %s", config, str(e))
337
- raise
338
-
339
- def call_vlm(face_path: str, eye_path: str, prompt: Optional[str] = None) -> Tuple[Optional[Dict], str]:
340
- """
341
- Call Qwen2.5-VL and return (parsed_features, raw_text)
342
- Tries multiple VLM spaces until one works
343
- """
344
- global active_vlm_config
345
-
346
  prompt = prompt or DEFAULT_VLM_PROMPT
347
 
348
- if not os.path.exists(face_path) or not os.path.exists(eye_path):
349
- raise FileNotFoundError("Face or eye image path missing")
350
 
351
- logger.info("VLM Input - Face: %s (exists: %s, size: %d bytes)",
352
- face_path, os.path.exists(face_path), os.path.getsize(face_path))
353
- logger.info("VLM Input - Eye: %s (exists: %s, size: %d bytes)",
354
- eye_path, os.path.exists(eye_path), os.path.getsize(eye_path))
355
- logger.info("VLM Prompt: %s", prompt[:100])
356
 
357
- # Try active config first if we have one that worked before
358
- configs_to_try = []
359
- if active_vlm_config:
360
- configs_to_try.append(active_vlm_config)
361
- configs_to_try.extend([c for c in VLM_SPACES if c != active_vlm_config])
 
 
 
 
 
 
362
 
363
  last_error = None
364
- for config in configs_to_try:
 
365
  try:
366
- logger.info("Trying VLM Space: %s with api_name=%s", config["space"], config["api_name"])
367
- client = get_gradio_client(config["space"])
368
 
369
- # Call VLM twice - once for face, once for eyes
370
- face_text = call_vlm_single_image(
371
- client, face_path,
372
- prompt + " Focus on the face and overall skin condition.",
373
- config
374
- )
375
 
376
- eye_text = call_vlm_single_image(
377
- client, eye_path,
378
- prompt + " Focus on the eyes, sclera color, and eye health.",
379
- config
380
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
381
 
382
- # Success! Save this config
383
- active_vlm_config = config
384
 
385
- # Combine both analyses
386
- combined_text = f"Face Analysis:\n{face_text}\n\nEye Analysis:\n{eye_text}"
387
- logger.info("VLM combined text (first 500 chars): %s", combined_text[:500])
 
388
 
389
  # Try to parse JSON
390
  parsed = None
391
  try:
392
- parsed = json.loads(combined_text)
393
  if not isinstance(parsed, dict):
 
394
  parsed = None
395
  else:
396
- logger.info("VLM successfully parsed JSON with keys: %s", list(parsed.keys()))
397
  except Exception:
 
398
  try:
399
- first = combined_text.find("{")
400
- last = combined_text.rfind("}")
401
  if first != -1 and last != -1 and last > first:
402
- json_str = combined_text[first:last+1]
403
  parsed = json.loads(json_str)
404
- if not isinstance(parsed, dict):
405
- parsed = None
406
  else:
407
- logger.info("Successfully extracted JSON from text with keys: %s", list(parsed.keys()))
408
  except Exception as extract_err:
409
- logger.warning("Could not extract JSON from VLM text: %s", str(extract_err))
410
  parsed = None
411
 
412
- # If no JSON found, create structured data from text
413
- if parsed is None:
414
- logger.info("No JSON found, creating structured data from text analysis")
415
- parsed = {
416
- "face_analysis": face_text[:500],
417
- "eye_analysis": eye_text[:500],
418
- "combined_analysis": combined_text[:1000]
419
- }
420
-
421
- return parsed, combined_text
422
 
423
  except Exception as e:
424
- logger.warning("VLM space %s failed: %s", config["space"], str(e))
425
- last_error = e
426
  continue
427
 
428
- # All configs failed
429
- logger.error("All VLM spaces failed. Last error: %s", str(last_error))
430
- raise RuntimeError(f"All VLM spaces failed. Last error: {last_error}")
431
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
432
  def get_fallback_risk_assessment(vlm_output: Any, reason: str = "LLM unavailable") -> Dict[str, Any]:
433
  """Generate basic risk assessment from VLM output when LLM is unavailable"""
434
  logger.warning("Using fallback risk assessment: %s", reason)
@@ -540,7 +644,7 @@ def call_llm(vlm_output: Any, use_fallback_on_error: bool = True) -> Dict[str, A
540
  )
541
 
542
  text_out = json.dumps(result) if isinstance(result, (dict, list)) else str(result)
543
- logger.info("LLM raw output:\n%s", text_out)
544
 
545
  parsed = extract_json_from_llm_output(text_out)
546
  logger.info("LLM parsed JSON:\n%s", json.dumps(parsed, indent=2))
@@ -564,12 +668,11 @@ def call_llm(vlm_output: Any, use_fallback_on_error: bool = True) -> Dict[str, A
564
 
565
  error_msg = str(e).lower()
566
  if "quota" in error_msg or "gpu" in error_msg:
567
- logger.warning("GPU quota exceeded, using fallback assessment")
568
  if use_fallback_on_error:
569
  return get_fallback_risk_assessment(vlm_output, reason="gpu_quota_exceeded")
570
 
571
  if use_fallback_on_error:
572
- logger.warning("LLM error, using fallback assessment")
573
  return get_fallback_risk_assessment(vlm_output, reason=f"llm_error: {str(e)[:100]}")
574
 
575
  raise RuntimeError(f"LLM call failed: {e}")
@@ -591,6 +694,7 @@ async def process_screening(screening_id: str):
591
  face_path = entry["face_image_path"]
592
  eye_path = entry["eye_image_path"]
593
 
 
594
  face_img = Image.open(face_path).convert("RGB")
595
  detection_result = detect_face_and_eyes(face_img)
596
 
@@ -607,13 +711,14 @@ async def process_screening(screening_id: str):
607
  }
608
  screenings_db[screening_id]["quality_metrics"] = quality_metrics
609
 
610
- # Call VLM
611
  vlm_features, vlm_raw = await asyncio.to_thread(call_vlm, face_path, eye_path)
612
 
613
  # Call LLM with fallback enabled
614
  llm_input = vlm_raw if vlm_raw else (vlm_features if vlm_features else "{}")
615
  structured_risk = await asyncio.to_thread(call_llm, llm_input, use_fallback_on_error=True)
616
 
 
617
  screenings_db[screening_id]["ai_results"] = {
618
  "vlm_features": vlm_features,
619
  "vlm_raw": vlm_raw,
@@ -621,6 +726,7 @@ async def process_screening(screening_id: str):
621
  "processing_time_ms": 1200
622
  }
623
 
 
624
  disease_predictions = [
625
  {
626
  "condition": "Anemia-like-signs",
@@ -657,7 +763,7 @@ async def process_screening(screening_id: str):
657
  screenings_db[screening_id]["error"] = str(e)
658
 
659
  # ============================================================================
660
- # FastAPI App & Routes (REST OF THE CODE REMAINS THE SAME)
661
  # ============================================================================
662
  app = FastAPI(title="Elderly HealthWatch AI Backend")
663
  app.add_middleware(
@@ -671,14 +777,14 @@ app.add_middleware(
671
  @app.get("/")
672
  async def read_root():
673
  return {
674
- "message": "Elderly HealthWatch AI Backend - Using Qwen2.5-VL",
675
- "active_vlm": active_vlm_config["space"] if active_vlm_config else "Not yet determined",
676
- "available_vlm_spaces": [c["space"] for c in VLM_SPACES]
677
  }
678
 
679
  @app.get("/health")
680
  async def health_check():
681
- """Health check with LLM availability status"""
682
  llm_status = "available"
683
  llm_message = None
684
 
@@ -690,23 +796,24 @@ async def health_check():
690
  error_msg = str(e).lower()
691
  if "quota" in error_msg or "gpu" in error_msg:
692
  llm_status = "quota_exceeded"
693
- llm_message = "GPU quota exceeded. Using fallback assessments."
694
  else:
695
  llm_status = "error"
696
  llm_message = "LLM temporarily unavailable"
697
  else:
698
  llm_status = "not_installed"
699
- llm_message = "Gradio client not available"
700
 
701
  return {
702
  "status": "healthy",
703
  "detector": detector_type or "none",
704
  "vlm_available": GRADIO_AVAILABLE,
705
- "active_vlm_space": active_vlm_config["space"] if active_vlm_config else "Not yet determined",
706
- "available_vlm_spaces": [c["space"] for c in VLM_SPACES],
707
  "llm_space": LLM_GRADIO_SPACE,
708
  "llm_status": llm_status,
709
  "llm_message": llm_message,
 
 
710
  "fallback_enabled": True
711
  }
712
 
@@ -766,10 +873,13 @@ async def upload_images(
766
  face_path = os.path.join(TMP_DIR, f"{screening_id}_face.jpg")
767
  eye_path = os.path.join(TMP_DIR, f"{screening_id}_eye.jpg")
768
 
 
 
 
769
  with open(face_path, "wb") as f:
770
- f.write(await face_image.read())
771
  with open(eye_path, "wb") as f:
772
- f.write(await eye_image.read())
773
 
774
  screenings_db[screening_id] = {
775
  "id": screening_id,
@@ -830,52 +940,16 @@ async def get_history(user_id: str):
830
  history = [s for s in screenings_db.values() if s.get("user_id") == user_id]
831
  return {"screenings": history}
832
 
833
- @app.get("/api/v1/debug/spaces")
834
- async def debug_spaces():
835
- """Debug endpoint to test VLM and LLM spaces"""
836
- results = {
837
- "vlm_spaces": [],
838
- "llm": {"available": False, "error": None}
839
- }
840
-
841
- # Test each VLM space
842
- if GRADIO_AVAILABLE:
843
- for config in VLM_SPACES:
844
- space_result = {"space": config["space"], "available": False, "error": None}
845
- try:
846
- client = get_gradio_client(config["space"])
847
- space_result["available"] = True
848
- space_result["config"] = config
849
- except Exception as e:
850
- space_result["error"] = str(e)
851
- results["vlm_spaces"].append(space_result)
852
- else:
853
- results["vlm_error"] = "Gradio not installed"
854
-
855
- # Test LLM
856
- if GRADIO_AVAILABLE:
857
- try:
858
- client = get_gradio_client(LLM_GRADIO_SPACE)
859
- results["llm"]["available"] = True
860
- results["llm"]["space"] = LLM_GRADIO_SPACE
861
- except Exception as e:
862
- results["llm"]["error"] = str(e)
863
- else:
864
- results["llm"]["error"] = "Gradio not installed"
865
-
866
- results["active_vlm"] = active_vlm_config
867
- return results
868
-
869
  @app.post("/api/v1/get-vitals")
870
  async def get_vitals_from_upload(
871
  face_image: UploadFile = File(...),
872
  eye_image: UploadFile = File(...)
873
  ):
874
- """Synchronous VLM + LLM pipeline with graceful fallback"""
875
  if not GRADIO_AVAILABLE:
876
  raise HTTPException(
877
  status_code=503,
878
- detail="AI services temporarily unavailable. Please try again later."
879
  )
880
 
881
  try:
@@ -883,15 +957,18 @@ async def get_vitals_from_upload(
883
  face_path = os.path.join(TMP_DIR, f"{uid}_face.jpg")
884
  eye_path = os.path.join(TMP_DIR, f"{uid}_eye.jpg")
885
 
 
 
 
886
  with open(face_path, "wb") as f:
887
- f.write(await face_image.read())
888
  with open(eye_path, "wb") as f:
889
- f.write(await eye_image.read())
890
 
891
- # Call VLM
892
  vlm_features, vlm_raw = await asyncio.to_thread(call_vlm, face_path, eye_path)
893
 
894
- # Call LLM with fallback enabled
895
  llm_input = vlm_raw if vlm_raw else (vlm_features if vlm_features else "{}")
896
  structured_risk = await asyncio.to_thread(call_llm, llm_input, use_fallback_on_error=True)
897
 
@@ -900,7 +977,7 @@ async def get_vitals_from_upload(
900
  "vlm_raw": vlm_raw,
901
  "structured_risk": structured_risk,
902
  "using_fallback": structured_risk.get("fallback_mode", False),
903
- "vlm_space_used": active_vlm_config["space"] if active_vlm_config else "unknown"
904
  }
905
 
906
  except Exception as e:
@@ -910,17 +987,17 @@ async def get_vitals_from_upload(
910
  if "quota" in error_msg or "gpu" in error_msg:
911
  raise HTTPException(
912
  status_code=503,
913
- detail="AI service is currently at capacity. Please try again in a few minutes."
914
  )
915
 
916
  raise HTTPException(
917
  status_code=500,
918
- detail="Unable to process images. Please ensure images are clear and try again."
919
  )
920
 
921
  @app.post("/api/v1/get-vitals/{screening_id}")
922
  async def get_vitals_for_screening(screening_id: str):
923
- """Re-run VLM + LLM on existing screening with fallback support"""
924
  if screening_id not in screenings_db:
925
  raise HTTPException(status_code=404, detail="Screening not found")
926
 
@@ -929,7 +1006,7 @@ async def get_vitals_for_screening(screening_id: str):
929
  eye_path = entry.get("eye_image_path")
930
 
931
  if not (face_path and os.path.exists(face_path) and eye_path and os.path.exists(eye_path)):
932
- raise HTTPException(status_code=400, detail="Images missing for this screening")
933
 
934
  try:
935
  vlm_features, vlm_raw = await asyncio.to_thread(call_vlm, face_path, eye_path)
@@ -941,8 +1018,7 @@ async def get_vitals_for_screening(screening_id: str):
941
  "vlm_raw": vlm_raw,
942
  "structured_risk": structured_risk,
943
  "last_vitals_run": datetime.utcnow().isoformat() + "Z",
944
- "using_fallback": structured_risk.get("fallback_mode", False),
945
- "vlm_space_used": active_vlm_config["space"] if active_vlm_config else "unknown"
946
  })
947
 
948
  return {
@@ -950,24 +1026,12 @@ async def get_vitals_for_screening(screening_id: str):
950
  "vlm_features": vlm_features,
951
  "vlm_raw": vlm_raw,
952
  "structured_risk": structured_risk,
953
- "using_fallback": structured_risk.get("fallback_mode", False),
954
- "vlm_space_used": active_vlm_config["space"] if active_vlm_config else "unknown"
955
  }
956
 
957
  except Exception as e:
958
  logger.exception("Get vitals for screening failed")
959
- error_msg = str(e).lower()
960
-
961
- if "quota" in error_msg or "gpu" in error_msg:
962
- raise HTTPException(
963
- status_code=503,
964
- detail="AI service is currently at capacity. Please try again in a few minutes."
965
- )
966
-
967
- raise HTTPException(
968
- status_code=500,
969
- detail="Unable to re-process screening. Please try again."
970
- )
971
 
972
  if __name__ == "__main__":
973
  import uvicorn
 
1
  """
2
+ Elderly HealthWatch AI Backend (FastAPI) - With GCS Support
3
+ Simplified: Just upload gcs-credentials.json to repository root
4
  """
5
 
6
  import io
 
12
  import traceback
13
  import re
14
  from typing import Dict, Any, Optional, Tuple
15
+ from datetime import datetime, timedelta
16
 
17
  from fastapi import FastAPI, UploadFile, File, BackgroundTasks, HTTPException
18
  from fastapi.middleware.cors import CORSMiddleware
 
20
  import numpy as np
21
  import cv2
22
 
23
+ # Google Cloud Storage
24
+ try:
25
+ from google.cloud import storage
26
+ from google.oauth2 import service_account
27
+ GCS_AVAILABLE = True
28
+ except Exception:
29
+ GCS_AVAILABLE = False
30
+ logging.warning("Google Cloud Storage not available")
31
+
32
  try:
33
  from gradio_client import Client, handle_file
34
  GRADIO_AVAILABLE = True
 
41
  logging.basicConfig(level=logging.INFO)
42
  logger = logging.getLogger("elderly_healthwatch")
43
 
44
+ GRADIO_VLM_SPACE = os.getenv("GRADIO_SPACE", "developer0hye/Qwen3-VL-8B-Instruct")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  LLM_GRADIO_SPACE = os.getenv("LLM_GRADIO_SPACE", "Tonic/med-gpt-oss-20b-demo")
46
  HF_TOKEN = os.getenv("HF_TOKEN", None)
47
 
48
+ # GCS Configuration - Simple!
49
+ GCS_BUCKET_NAME = "elderly-healthwatch-images"
50
+ GCS_CREDENTIALS_FILE = "gcs-credentials.json"
51
+
52
  DEFAULT_VLM_PROMPT = (
53
+ "From the provided face/eye images, compute the required screening features "
54
+ "(pallor, sclera yellowness, redness, mobility metrics, quality checks) "
55
+ "and output a clean JSON feature vector only."
 
 
 
 
 
56
  )
57
 
58
  LLM_SYSTEM_PROMPT = (
 
73
  # In-memory database
74
  screenings_db: Dict[str, Dict[str, Any]] = {}
75
 
76
+ # ============================================================================
77
+ # Google Cloud Storage Setup
78
+ # ============================================================================
79
+ def setup_gcs_client():
80
+ """Initialize GCS client from credentials file"""
81
+ if not GCS_AVAILABLE:
82
+ logger.warning("GCS libraries not installed")
83
+ return None, None
84
+
85
+ try:
86
+ if os.path.exists(GCS_CREDENTIALS_FILE):
87
+ logger.info("Found GCS credentials file: %s", GCS_CREDENTIALS_FILE)
88
+ credentials = service_account.Credentials.from_service_account_file(GCS_CREDENTIALS_FILE)
89
+ client = storage.Client(credentials=credentials)
90
+ bucket = client.bucket(GCS_BUCKET_NAME)
91
+ logger.info("✅ GCS initialized successfully for bucket: %s", GCS_BUCKET_NAME)
92
+ return client, bucket
93
+ else:
94
+ logger.warning("⚠️ GCS credentials file not found at: %s", GCS_CREDENTIALS_FILE)
95
+ logger.warning("VLM will use file handles instead of URLs")
96
+ return None, None
97
+
98
+ except Exception as e:
99
+ logger.exception("Failed to initialize GCS: %s", str(e))
100
+ return None, None
101
+
102
+ gcs_client, gcs_bucket = setup_gcs_client()
103
+
104
+ def upload_to_gcs(local_path: str, blob_name: str) -> Optional[str]:
105
+ """Upload file to GCS and return public URL"""
106
+ if gcs_bucket is None:
107
+ return None
108
+
109
+ try:
110
+ blob = gcs_bucket.blob(blob_name)
111
+ blob.upload_from_filename(local_path, content_type='image/jpeg')
112
+ blob.make_public()
113
+ public_url = blob.public_url
114
+ logger.info("✅ Uploaded to GCS: %s -> %s", blob_name, public_url[:60])
115
+ return public_url
116
+ except Exception as e:
117
+ logger.exception("Failed to upload to GCS: %s", str(e))
118
+ return None
119
 
120
  # ============================================================================
121
  # Face Detection Setup
122
  # ============================================================================
123
  def setup_face_detector():
124
  """Initialize face detector (MTCNN or OpenCV fallback)"""
125
+ # Try facenet-pytorch MTCNN
126
  try:
127
  from facenet_pytorch import MTCNN
128
  return MTCNN(keep_all=False, device="cpu"), "facenet_pytorch"
129
  except Exception:
130
  pass
131
 
132
+ # Try classic MTCNN
133
  try:
134
  from mtcnn import MTCNN
135
  return MTCNN(), "mtcnn"
136
  except Exception:
137
  pass
138
 
139
+ # OpenCV Haar cascade fallback
140
  try:
141
  face_path = os.path.join(cv2.data.haarcascades, "haarcascade_frontalface_default.xml")
142
  eye_path = os.path.join(cv2.data.haarcascades, "haarcascade_eye.xml")
 
193
 
194
  img_arr = np.asarray(pil_img)
195
 
196
+ # Facenet-pytorch MTCNN
197
  if detector_type == "facenet_pytorch":
198
  try:
199
  boxes, probs, landmarks = face_detector.detect(pil_img, landmarks=True)
 
221
  return {"face_detected": False, "face_confidence": 0.0, "eye_openness_score": 0.0,
222
  "left_eye": None, "right_eye": None}
223
 
224
+ # Classic MTCNN
225
  elif detector_type == "mtcnn":
226
  try:
227
  detections = face_detector.detect_faces(img_arr)
 
245
  return {"face_detected": False, "face_confidence": 0.0, "eye_openness_score": 0.0,
246
  "left_eye": None, "right_eye": None}
247
 
248
+ # OpenCV fallback
249
  elif detector_type == "opencv":
250
  try:
251
  gray = cv2.cvtColor(img_arr, cv2.COLOR_RGB2GRAY)
 
337
  }
338
 
339
  # ============================================================================
340
+ # VLM Integration - WITH GCS URL SUPPORT
341
  # ============================================================================
342
  def get_gradio_client(space: str) -> Client:
343
  """Get Gradio client with optional auth"""
 
345
  raise RuntimeError("gradio_client not installed")
346
  return Client(space, hf_token=HF_TOKEN) if HF_TOKEN else Client(space)
347
 
348
+ def call_vlm_with_urls(face_url: str, eye_url: str, prompt: Optional[str] = None) -> Tuple[Optional[Dict], str]:
349
+ """Call VLM using image URLs instead of file handles"""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
350
  prompt = prompt or DEFAULT_VLM_PROMPT
351
 
352
+ logger.info("🔗 VLM Input - Face URL: %s", face_url[:80])
353
+ logger.info("🔗 VLM Input - Eye URL: %s", eye_url[:80])
354
 
355
+ client = get_gradio_client(GRADIO_VLM_SPACE)
 
 
 
 
356
 
357
+ # Try different message formats that VLM spaces commonly accept
358
+ message_formats = [
359
+ # Format 1: URLs in files array
360
+ {"text": prompt, "files": [face_url, eye_url]},
361
+ # Format 2: Separate image fields
362
+ {"prompt": prompt, "image1": face_url, "image2": eye_url},
363
+ # Format 3: Single message with URLs
364
+ {"message": f"{prompt}\n\nFace image: {face_url}\nEye image: {eye_url}"},
365
+ # Format 4: Images array
366
+ {"text": prompt, "images": [face_url, eye_url]},
367
+ ]
368
 
369
  last_error = None
370
+
371
+ for idx, message in enumerate(message_formats, 1):
372
  try:
373
+ logger.info("Trying VLM message format %d/%d: %s", idx, len(message_formats), list(message.keys()))
374
+ result = client.predict(message=message, history=[], api_name="/chat_fn")
375
 
376
+ logger.info("✅ VLM call succeeded with format %d", idx)
377
+ logger.info("VLM raw result type: %s", type(result))
 
 
 
 
378
 
379
+ # Process the result
380
+ if isinstance(result, (list, tuple)):
381
+ logger.info("VLM returned list/tuple with %d elements", len(result))
382
+ out = result[0] if len(result) > 0 else {}
383
+ elif isinstance(result, dict):
384
+ logger.info("VLM returned dict with keys: %s", list(result.keys()))
385
+ out = result
386
+ else:
387
+ logger.info("VLM returned type: %s, converting to string", type(result))
388
+ out = {"text": str(result)}
389
+
390
+ # Extract text from various possible formats
391
+ text_out = None
392
+ if isinstance(out, dict):
393
+ text_out = out.get("text") or out.get("output") or out.get("content") or out.get("response")
394
+
395
+ if not text_out:
396
+ if isinstance(result, str):
397
+ text_out = result
398
+ else:
399
+ text_out = json.dumps(out)
400
 
401
+ logger.info("VLM extracted text (first 300 chars): %s", text_out[:300] if text_out else "EMPTY")
 
402
 
403
+ if not text_out or len(text_out.strip()) == 0:
404
+ logger.warning("VLM returned empty text, trying next format...")
405
+ last_error = "Empty response"
406
+ continue
407
 
408
  # Try to parse JSON
409
  parsed = None
410
  try:
411
+ parsed = json.loads(text_out)
412
  if not isinstance(parsed, dict):
413
+ logger.warning("VLM JSON parsed but not a dict: %s", type(parsed))
414
  parsed = None
415
  else:
416
+ logger.info("VLM successfully parsed JSON with keys: %s", list(parsed.keys()))
417
  except Exception:
418
+ # Try to extract JSON from text
419
  try:
420
+ first = text_out.find("{")
421
+ last = text_out.rfind("}")
422
  if first != -1 and last != -1 and last > first:
423
+ json_str = text_out[first:last+1]
424
  parsed = json.loads(json_str)
425
+ if isinstance(parsed, dict):
426
+ logger.info("✅ Successfully extracted JSON from text with keys: %s", list(parsed.keys()))
427
  else:
428
+ parsed = None
429
  except Exception as extract_err:
430
+ logger.info("Could not extract JSON from VLM text: %s", str(extract_err))
431
  parsed = None
432
 
433
+ # Success! Return the result
434
+ return parsed, text_out
 
 
 
 
 
 
 
 
435
 
436
  except Exception as e:
437
+ logger.warning("VLM format %d failed: %s", idx, str(e))
438
+ last_error = str(e)
439
  continue
440
 
441
+ # All formats failed
442
+ raise RuntimeError(f"All VLM message formats failed. Last error: {last_error}")
 
443
 
444
+ def call_vlm(face_path: str, eye_path: str, prompt: Optional[str] = None) -> Tuple[Optional[Dict], str]:
445
+ """
446
+ Call VLM - wrapper that handles both local files and GCS URLs
447
+ Strategy: Try GCS first (if available), fallback to file handles
448
+ """
449
+
450
+ # Strategy 1: Try GCS URLs (if GCS is set up)
451
+ if gcs_bucket is not None:
452
+ logger.info("🌐 GCS is available, uploading images and using URLs for VLM")
453
+
454
+ try:
455
+ # Generate unique blob names
456
+ unique_id = str(uuid.uuid4())
457
+ face_blob_name = f"vlm_temp/{unique_id}_face.jpg"
458
+ eye_blob_name = f"vlm_temp/{unique_id}_eye.jpg"
459
+
460
+ # Upload to GCS
461
+ face_url = upload_to_gcs(face_path, face_blob_name)
462
+ eye_url = upload_to_gcs(eye_path, eye_blob_name)
463
+
464
+ if face_url and eye_url:
465
+ logger.info("✅ Successfully uploaded to GCS, calling VLM with URLs")
466
+ return call_vlm_with_urls(face_url, eye_url, prompt)
467
+ else:
468
+ logger.warning("⚠️ GCS upload failed, falling back to file handles")
469
+ except Exception as e:
470
+ logger.warning("⚠️ GCS error, falling back to file handles: %s", str(e))
471
+ else:
472
+ logger.info("ℹ️ GCS not available, using file handles for VLM")
473
+
474
+ # Strategy 2: Fallback to file handles (original method)
475
+ if not os.path.exists(face_path) or not os.path.exists(eye_path):
476
+ raise FileNotFoundError("Face or eye image path missing")
477
+
478
+ logger.info("📁 VLM Input - Face file: %s (size: %d bytes)", face_path, os.path.getsize(face_path))
479
+ logger.info("📁 VLM Input - Eye file: %s (size: %d bytes)", eye_path, os.path.getsize(eye_path))
480
+
481
+ prompt = prompt or DEFAULT_VLM_PROMPT
482
+ client = get_gradio_client(GRADIO_VLM_SPACE)
483
+ message = {"text": prompt, "files": [handle_file(face_path), handle_file(eye_path)]}
484
+
485
+ try:
486
+ logger.info("Calling VLM with file handles")
487
+ result = client.predict(message=message, history=[], api_name="/chat_fn")
488
+ logger.info("VLM raw result: %s", str(result)[:500])
489
+ except Exception as e:
490
+ logger.exception("VLM call with file handles failed")
491
+ raise RuntimeError(f"VLM call failed: {e}")
492
+
493
+ # Process result (same as in call_vlm_with_urls)
494
+ if isinstance(result, (list, tuple)):
495
+ out = result[0] if len(result) > 0 else {}
496
+ elif isinstance(result, dict):
497
+ out = result
498
+ else:
499
+ out = {"text": str(result)}
500
+
501
+ text_out = None
502
+ if isinstance(out, dict):
503
+ text_out = out.get("text") or out.get("output") or out.get("content")
504
+
505
+ if not text_out:
506
+ if isinstance(result, str):
507
+ text_out = result
508
+ else:
509
+ text_out = json.dumps(out)
510
+
511
+ if not text_out or len(text_out.strip()) == 0:
512
+ logger.warning("VLM returned empty text")
513
+ text_out = "{}"
514
+
515
+ parsed = None
516
+ try:
517
+ parsed = json.loads(text_out)
518
+ if not isinstance(parsed, dict):
519
+ parsed = None
520
+ except Exception:
521
+ try:
522
+ first = text_out.find("{")
523
+ last = text_out.rfind("}")
524
+ if first != -1 and last != -1:
525
+ parsed = json.loads(text_out[first:last+1])
526
+ if not isinstance(parsed, dict):
527
+ parsed = None
528
+ except Exception:
529
+ pass
530
+
531
+ return parsed, text_out
532
+
533
+ # ============================================================================
534
+ # LLM Integration
535
+ # ============================================================================
536
  def get_fallback_risk_assessment(vlm_output: Any, reason: str = "LLM unavailable") -> Dict[str, Any]:
537
  """Generate basic risk assessment from VLM output when LLM is unavailable"""
538
  logger.warning("Using fallback risk assessment: %s", reason)
 
644
  )
645
 
646
  text_out = json.dumps(result) if isinstance(result, (dict, list)) else str(result)
647
+ logger.info("LLM raw output:\n%s", text_out[:1000])
648
 
649
  parsed = extract_json_from_llm_output(text_out)
650
  logger.info("LLM parsed JSON:\n%s", json.dumps(parsed, indent=2))
 
668
 
669
  error_msg = str(e).lower()
670
  if "quota" in error_msg or "gpu" in error_msg:
671
+ logger.warning("GPU quota exceeded, using fallback")
672
  if use_fallback_on_error:
673
  return get_fallback_risk_assessment(vlm_output, reason="gpu_quota_exceeded")
674
 
675
  if use_fallback_on_error:
 
676
  return get_fallback_risk_assessment(vlm_output, reason=f"llm_error: {str(e)[:100]}")
677
 
678
  raise RuntimeError(f"LLM call failed: {e}")
 
694
  face_path = entry["face_image_path"]
695
  eye_path = entry["eye_image_path"]
696
 
697
+ # Load images and get quality metrics
698
  face_img = Image.open(face_path).convert("RGB")
699
  detection_result = detect_face_and_eyes(face_img)
700
 
 
711
  }
712
  screenings_db[screening_id]["quality_metrics"] = quality_metrics
713
 
714
+ # Call VLM (will use GCS URLs if available)
715
  vlm_features, vlm_raw = await asyncio.to_thread(call_vlm, face_path, eye_path)
716
 
717
  # Call LLM with fallback enabled
718
  llm_input = vlm_raw if vlm_raw else (vlm_features if vlm_features else "{}")
719
  structured_risk = await asyncio.to_thread(call_llm, llm_input, use_fallback_on_error=True)
720
 
721
+ # Store results
722
  screenings_db[screening_id]["ai_results"] = {
723
  "vlm_features": vlm_features,
724
  "vlm_raw": vlm_raw,
 
726
  "processing_time_ms": 1200
727
  }
728
 
729
+ # Build disease predictions
730
  disease_predictions = [
731
  {
732
  "condition": "Anemia-like-signs",
 
763
  screenings_db[screening_id]["error"] = str(e)
764
 
765
  # ============================================================================
766
+ # FastAPI App & Routes
767
  # ============================================================================
768
  app = FastAPI(title="Elderly HealthWatch AI Backend")
769
  app.add_middleware(
 
777
  @app.get("/")
778
  async def read_root():
779
  return {
780
+ "message": "Elderly HealthWatch AI Backend",
781
+ "gcs_enabled": gcs_bucket is not None,
782
+ "version": "1.0.0-gcs"
783
  }
784
 
785
  @app.get("/health")
786
  async def health_check():
787
+ """Health check with GCS and LLM status"""
788
  llm_status = "available"
789
  llm_message = None
790
 
 
796
  error_msg = str(e).lower()
797
  if "quota" in error_msg or "gpu" in error_msg:
798
  llm_status = "quota_exceeded"
799
+ llm_message = "GPU quota exceeded. Using fallback."
800
  else:
801
  llm_status = "error"
802
  llm_message = "LLM temporarily unavailable"
803
  else:
804
  llm_status = "not_installed"
805
+ llm_message = "Gradio not available"
806
 
807
  return {
808
  "status": "healthy",
809
  "detector": detector_type or "none",
810
  "vlm_available": GRADIO_AVAILABLE,
811
+ "vlm_space": GRADIO_VLM_SPACE,
 
812
  "llm_space": LLM_GRADIO_SPACE,
813
  "llm_status": llm_status,
814
  "llm_message": llm_message,
815
+ "gcs_available": gcs_bucket is not None,
816
+ "gcs_bucket": GCS_BUCKET_NAME if gcs_bucket else None,
817
  "fallback_enabled": True
818
  }
819
 
 
873
  face_path = os.path.join(TMP_DIR, f"{screening_id}_face.jpg")
874
  eye_path = os.path.join(TMP_DIR, f"{screening_id}_eye.jpg")
875
 
876
+ face_bytes = await face_image.read()
877
+ eye_bytes = await eye_image.read()
878
+
879
  with open(face_path, "wb") as f:
880
+ f.write(face_bytes)
881
  with open(eye_path, "wb") as f:
882
+ f.write(eye_bytes)
883
 
884
  screenings_db[screening_id] = {
885
  "id": screening_id,
 
940
  history = [s for s in screenings_db.values() if s.get("user_id") == user_id]
941
  return {"screenings": history}
942
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
943
  @app.post("/api/v1/get-vitals")
944
  async def get_vitals_from_upload(
945
  face_image: UploadFile = File(...),
946
  eye_image: UploadFile = File(...)
947
  ):
948
+ """Synchronous VLM + LLM pipeline with GCS support"""
949
  if not GRADIO_AVAILABLE:
950
  raise HTTPException(
951
  status_code=503,
952
+ detail="AI services temporarily unavailable."
953
  )
954
 
955
  try:
 
957
  face_path = os.path.join(TMP_DIR, f"{uid}_face.jpg")
958
  eye_path = os.path.join(TMP_DIR, f"{uid}_eye.jpg")
959
 
960
+ face_bytes = await face_image.read()
961
+ eye_bytes = await eye_image.read()
962
+
963
  with open(face_path, "wb") as f:
964
+ f.write(face_bytes)
965
  with open(eye_path, "wb") as f:
966
+ f.write(eye_bytes)
967
 
968
+ # Call VLM (will use GCS if available)
969
  vlm_features, vlm_raw = await asyncio.to_thread(call_vlm, face_path, eye_path)
970
 
971
+ # Call LLM
972
  llm_input = vlm_raw if vlm_raw else (vlm_features if vlm_features else "{}")
973
  structured_risk = await asyncio.to_thread(call_llm, llm_input, use_fallback_on_error=True)
974
 
 
977
  "vlm_raw": vlm_raw,
978
  "structured_risk": structured_risk,
979
  "using_fallback": structured_risk.get("fallback_mode", False),
980
+ "using_gcs": gcs_bucket is not None
981
  }
982
 
983
  except Exception as e:
 
987
  if "quota" in error_msg or "gpu" in error_msg:
988
  raise HTTPException(
989
  status_code=503,
990
+ detail="AI service at capacity. Please try again in a few minutes."
991
  )
992
 
993
  raise HTTPException(
994
  status_code=500,
995
+ detail="Unable to process images. Please try again."
996
  )
997
 
998
  @app.post("/api/v1/get-vitals/{screening_id}")
999
  async def get_vitals_for_screening(screening_id: str):
1000
+ """Re-run VLM + LLM on existing screening"""
1001
  if screening_id not in screenings_db:
1002
  raise HTTPException(status_code=404, detail="Screening not found")
1003
 
 
1006
  eye_path = entry.get("eye_image_path")
1007
 
1008
  if not (face_path and os.path.exists(face_path) and eye_path and os.path.exists(eye_path)):
1009
+ raise HTTPException(status_code=400, detail="Images missing")
1010
 
1011
  try:
1012
  vlm_features, vlm_raw = await asyncio.to_thread(call_vlm, face_path, eye_path)
 
1018
  "vlm_raw": vlm_raw,
1019
  "structured_risk": structured_risk,
1020
  "last_vitals_run": datetime.utcnow().isoformat() + "Z",
1021
+ "using_fallback": structured_risk.get("fallback_mode", False)
 
1022
  })
1023
 
1024
  return {
 
1026
  "vlm_features": vlm_features,
1027
  "vlm_raw": vlm_raw,
1028
  "structured_risk": structured_risk,
1029
+ "using_fallback": structured_risk.get("fallback_mode", False)
 
1030
  }
1031
 
1032
  except Exception as e:
1033
  logger.exception("Get vitals for screening failed")
1034
+ raise HTTPException(status_code=500, detail=str(e))
 
 
 
 
 
 
 
 
 
 
 
1035
 
1036
  if __name__ == "__main__":
1037
  import uvicorn