SmartHeal commited on
Commit
9eef931
·
verified ·
1 Parent(s): 35b28db

Update src/ai_processor.py

Browse files
Files changed (1) hide show
  1. src/ai_processor.py +205 -961
src/ai_processor.py CHANGED
@@ -1,992 +1,236 @@
1
- # smartheal_ai_processor.py
2
- # Verbose, instrumented version — preserves public class/function names
3
- # Turn on deep logging: export LOGLEVEL=DEBUG SMARTHEAL_DEBUG=1
4
-
5
  import os
6
  import logging
7
- from datetime import datetime
8
- from typing import Optional, Dict, List, Tuple
9
-
10
- # ---- Environment defaults (do NOT globally hint CUDA here) ----
11
- os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
12
- LOGLEVEL = os.getenv("LOGLEVEL", "INFO").upper()
13
- SMARTHEAL_DEBUG = os.getenv("SMARTHEAL_DEBUG", "0") == "1"
14
-
15
  import cv2
16
  import numpy as np
17
  from PIL import Image
18
- from PIL.ExifTags import TAGS
19
-
20
- # --- Logging config ---
21
- logging.basicConfig(
22
- level=getattr(logging, LOGLEVEL, logging.INFO),
23
- format="%(asctime)s - %(levelname)s - %(message)s",
24
- )
25
-
26
- def _log_kv(prefix: str, kv: Dict):
27
- logging.debug(prefix + " | " + " | ".join(f"{k}={v}" for k, v in kv.items()))
28
-
29
- # --- Spaces GPU decorator (REQUIRED) ---
30
- from spaces import GPU as _SPACES_GPU
31
-
32
- @_SPACES_GPU(enable_queue=True)
33
- def smartheal_gpu_stub(ping: int = 0) -> str:
34
- return "ready"
35
-
36
- # ---- Paths / constants ----
37
- UPLOADS_DIR = "uploads"
38
- os.makedirs(UPLOADS_DIR, exist_ok=True)
39
-
40
- HF_TOKEN = os.getenv("HF_TOKEN", None)
41
- YOLO_MODEL_PATH = "src/best.pt"
42
- SEG_MODEL_PATH = "src/segmentation_model.h5" # optional
43
- GUIDELINE_PDFS = ["src/eHealth in Wound Care.pdf", "src/IWGDF Guideline.pdf", "src/evaluation.pdf"]
44
- DATASET_ID = "SmartHeal/wound-image-uploads"
45
- DEFAULT_PX_PER_CM = 38.0
46
- PX_PER_CM_MIN, PX_PER_CM_MAX = 5.0, 1200.0
47
-
48
- # Segmentation preprocessing knobs
49
- SEG_EXPECTS_RGB = os.getenv("SEG_EXPECTS_RGB", "1") == "1" # most TF models trained on RGB
50
- SEG_NORM = os.getenv("SEG_NORM", "0to1") # "0to1" | "imagenet"
51
- SEG_THRESH = float(os.getenv("SEG_THRESH", "0.5"))
52
-
53
- models_cache: Dict[str, object] = {}
54
- knowledge_base_cache: Dict[str, object] = {}
55
-
56
- # ---------- Utilities to prevent CUDA in main process ----------
57
- from contextlib import contextmanager
58
-
59
- @contextmanager
60
- def _no_cuda_env():
61
- """
62
- Mask GPUs so any library imported/constructed in the main process
63
- cannot see CUDA (required for Spaces Stateless GPU).
64
- """
65
- prev = os.environ.get("CUDA_VISIBLE_DEVICES")
66
- os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
67
- try:
68
- yield
69
- finally:
70
- if prev is None:
71
- os.environ.pop("CUDA_VISIBLE_DEVICES", None)
72
- else:
73
- os.environ["CUDA_VISIBLE_DEVICES"] = prev
74
-
75
- # ---------- Lazy imports (wrapped where needed) ----------
76
- def _import_ultralytics():
77
- # Prevent Ultralytics from probing CUDA on import
78
- with _no_cuda_env():
79
- from ultralytics import YOLO
80
- return YOLO
81
-
82
- def _import_tf_loader():
83
- import tensorflow as tf
84
- tf.config.set_visible_devices([], "GPU")
85
- from tensorflow.keras.models import load_model
86
- return load_model
87
-
88
- def _import_hf_cls():
89
- from transformers import pipeline
90
- return pipeline
91
-
92
- def _import_embeddings():
93
- from langchain_community.embeddings import HuggingFaceEmbeddings
94
- return HuggingFaceEmbeddings
95
-
96
- def _import_langchain_pdf():
97
- from langchain_community.document_loaders import PyPDFLoader
98
- return PyPDFLoader
99
-
100
- def _import_langchain_faiss():
101
- from langchain_community.vectorstores import FAISS
102
- return FAISS
103
-
104
- def _import_hf_hub():
105
- from huggingface_hub import HfApi, HfFolder
106
- return HfApi, HfFolder
107
-
108
- # ---------- SmartHeal prompts (system + user prefix) ----------
109
- SMARTHEAL_SYSTEM_PROMPT = """\
110
- You are SmartHeal Clinical Assistant, a wound-care decision-support system.
111
- You analyze wound photographs and brief patient context to produce careful,
112
- specific, guideline-informed recommendations WITHOUT diagnosing. You always:
113
- - Use the measurements calculated by the vision pipeline as ground truth.
114
- - Prefer concise, actionable steps tailored to exudate level, infection risk, and pain.
115
- - Flag uncertainties and red flags that need escalation to a clinician.
116
- - Avoid contraindicated advice; do not infer unseen comorbidities.
117
- - Keep under 300 words and use the requested headings exactly.
118
- - Tone: professional, clear, and conservative; no definitive medical claims.
119
- - Safety: remind the user to seek clinician review for changes or red flags.
120
- """
121
-
122
- SMARTHEAL_USER_PREFIX = """\
123
- Patient: {patient_info}
124
- Visual findings: type={wound_type}, size={length_cm}x{breadth_cm} cm, area={area_cm2} cm^2,
125
- detection_conf={det_conf:.2f}, calibration={px_per_cm} px/cm.
126
-
127
- Guideline context (snippets you can draw principles from; do not quote at length):
128
- {guideline_context}
129
-
130
- Write a structured answer with these headings exactly:
131
- 1. Clinical Summary (max 4 bullet points)
132
- 2. Likely Stage/Type (if uncertain, say 'uncertain')
133
- 3. Treatment Plan (specific dressing choices and frequency based on exudate/infection risk)
134
- 4. Red Flags (what to escalate and when)
135
- 5. Follow-up Cadence (days)
136
- 6. Notes (assumptions/uncertainties)
137
-
138
- Keep to 220–300 words. Do NOT provide diagnosis. Avoid contraindicated advice.
139
- """
140
-
141
- # ---------- VLM (MedGemma replaced with Qwen2-VL) ----------
142
- @_SPACES_GPU(enable_queue=True)
143
- def _vlm_infer_gpu(messages, model_id: str, max_new_tokens: int, token: Optional[str]):
144
- """
145
- Runs entirely inside a Spaces GPU worker. It's the ONLY place we allow CUDA init.
146
- """
147
- from transformers import pipeline
148
- import torch # Ensure torch is imported here
149
- pipe = pipeline(
150
- task="image-text-to-text",
151
- model=model_id,
152
- torch_dtype=torch.bfloat16, # Use torch_dtype from the working example
153
- device_map="auto", # CUDA init happens here, safely in GPU worker
154
- token=token,
155
- trust_remote_code=True,
156
- model_kwargs={"low_cpu_mem_usage": True},
157
- )
158
- out = pipe(text=messages, max_new_tokens=max_new_tokens, do_sample=False, temperature=0.2)
159
- try:
160
- txt = out[0]["generated_text"][-1].get("content", "")
161
- except Exception:
162
- txt = out[0].get("generated_text", "")
163
- return (txt or "").strip() or "⚠️ Empty response"
164
-
165
- def generate_medgemma_report( # kept name so callers don't change
166
- patient_info: str,
167
- visual_results: Dict,
168
- guideline_context: str,
169
- image_pil: Image.Image,
170
- max_new_tokens: Optional[int] = None,
171
- ) -> str:
172
- """
173
- MedGemma replacement using Qwen/Qwen2-VL-2B-Instruct via image-text-to-text.
174
- Loads & runs ONLY inside a GPU worker to satisfy Stateless GPU constraints.
175
- """
176
- if os.getenv("SMARTHEAL_ENABLE_VLM", "1") != "1":
177
- return "⚠️ VLM disabled"
178
-
179
- model_id = os.getenv("SMARTHEAL_VLM_MODEL", "Qwen/Qwen2-VL-2B-Instruct")
180
- max_new_tokens = max_new_tokens or int(os.getenv("SMARTHEAL_VLM_MAX_TOKENS", "600"))
181
-
182
- uprompt = SMARTHEAL_USER_PREFIX.format(
183
- patient_info=patient_info,
184
- wound_type=visual_results.get("wound_type", "Unknown"),
185
- length_cm=visual_results.get("length_cm", 0),
186
- breadth_cm=visual_results.get("breadth_cm", 0),
187
- area_cm2=visual_results.get("surface_area_cm2", 0),
188
- det_conf=float(visual_results.get("detection_confidence", 0.0)),
189
- px_per_cm=visual_results.get("px_per_cm", "?"),
190
- guideline_context=(guideline_context or "")[:900],
191
- )
192
-
193
- messages = [
194
- {"role": "system", "content": [{"type": "text", "text": SMARTHEAL_SYSTEM_PROMPT}]},
195
- {"role": "user", "content": [
196
- {"type": "image", "image": image_pil},
197
- {"type": "text", "text": uprompt},
198
- ]},
199
- ]
200
-
201
- try:
202
- # IMPORTANT: do not import transformers or touch CUDA here. Only call the GPU worker.
203
- return _vlm_infer_gpu(messages, model_id, max_new_tokens, HF_TOKEN)
204
- except Exception as e:
205
- logging.error(f"VLM call failed: {e}")
206
- return "⚠️ VLM error"
207
-
208
- # ---------- Initialize CPU models ----------
209
- def load_yolo_model():
210
- YOLO = _import_ultralytics()
211
- # Construct model with CUDA masked to avoid auto-selecting cuda:0
212
- with _no_cuda_env():
213
- model = YOLO(YOLO_MODEL_PATH)
214
- return model
215
- def load_segmentation_model():
216
- import tensorflow as tf
217
- load_model = _import_tf_loader()
218
- return load_model(SEG_MODEL_PATH, compile=False, custom_objects={'InputLayer': tf.keras.layers.InputLayer})
219
-
220
- def load_classification_pipeline():
221
- pipe = _import_hf_cls()
222
- return pipe("image-classification", model="Hemg/Wound-classification", token=HF_TOKEN, device="cpu")
223
-
224
- def load_embedding_model():
225
- Emb = _import_embeddings()
226
- return Emb(model_name="sentence-transformers/all-MiniLM-L6-v2", model_kwargs={"device": "cpu"})
227
-
228
- def initialize_cpu_models() -> None:
229
- if HF_TOKEN:
230
- try:
231
- HfApi, HfFolder = _import_hf_hub()
232
- HfFolder.save_token(HF_TOKEN)
233
- logging.info("✅ HF token set")
234
- except Exception as e:
235
- logging.warning(f"HF token save failed: {e}")
236
-
237
- if "det" not in models_cache:
238
- try:
239
- models_cache["det"] = load_yolo_model()
240
- logging.info("✅ YOLO loaded (CPU; CUDA masked in main)")
241
- except Exception as e:
242
- logging.error(f"YOLO load failed: {e}")
243
-
244
- if "seg" not in models_cache:
245
- try:
246
- if os.path.exists(SEG_MODEL_PATH):
247
- models_cache["seg"] = load_segmentation_model()
248
- m = models_cache["seg"]
249
- ishape = getattr(m, "input_shape", None)
250
- oshape = getattr(m, "output_shape", None)
251
- logging.info(f"✅ Segmentation model loaded (CPU) | input_shape={ishape} output_shape={oshape}")
252
- else:
253
- models_cache["seg"] = None
254
- logging.warning("Segmentation model file missing; skipping.")
255
- except Exception as e:
256
- models_cache["seg"] = None
257
- logging.warning(f"Segmentation unavailable: {e}")
258
-
259
- if "cls" not in models_cache:
260
- try:
261
- models_cache["cls"] = load_classification_pipeline()
262
- logging.info("✅ Classifier loaded (CPU)")
263
- except Exception as e:
264
- models_cache["cls"] = None
265
- logging.warning(f"Classifier unavailable: {e}")
266
-
267
- if "embedding_model" not in models_cache:
268
- try:
269
- models_cache["embedding_model"] = load_embedding_model()
270
- logging.info("✅ Embeddings loaded (CPU)")
271
- except Exception as e:
272
- models_cache["embedding_model"] = None
273
- logging.warning(f"Embeddings unavailable: {e}")
274
-
275
- def setup_knowledge_base() -> None:
276
- if "vector_store" in knowledge_base_cache:
277
- return
278
- docs: List = []
279
- try:
280
- PyPDFLoader = _import_langchain_pdf()
281
- for pdf in GUIDELINE_PDFS:
282
- if os.path.exists(pdf):
283
- try:
284
- docs.extend(PyPDFLoader(pdf).load())
285
- logging.info(f"Loaded PDF: {pdf}")
286
- except Exception as e:
287
- logging.warning(f"PDF load failed ({pdf}): {e}")
288
- except Exception as e:
289
- logging.warning(f"LangChain PDF loader unavailable: {e}")
290
-
291
- if docs and models_cache.get("embedding_model"):
292
- try:
293
- from langchain.text_splitter import RecursiveCharacterTextSplitter
294
- FAISS = _import_langchain_faiss()
295
- chunks = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100).split_documents(docs)
296
- knowledge_base_cache["vector_store"] = FAISS.from_documents(chunks, models_cache["embedding_model"])
297
- logging.info(f"✅ Knowledge base ready ({len(chunks)} chunks)")
298
- except Exception as e:
299
- knowledge_base_cache["vector_store"] = None
300
- logging.warning(f"KB build failed: {e}")
301
- else:
302
- knowledge_base_cache["vector_store"] = None
303
- logging.warning("KB disabled (no docs or embeddings).")
304
-
305
- initialize_cpu_models()
306
- setup_knowledge_base()
307
-
308
- # ---------- Calibration helpers ----------
309
- def _exif_to_dict(pil_img: Image.Image) -> Dict[str, object]:
310
- out = {}
311
- try:
312
- exif = pil_img.getexif()
313
- if not exif:
314
- return out
315
- for k, v in exif.items():
316
- tag = TAGS.get(k, k)
317
- out[tag] = v
318
- except Exception:
319
- pass
320
- return out
321
-
322
- def _to_float(val) -> Optional[float]:
323
- try:
324
- if val is None:
325
- return None
326
- if isinstance(val, tuple) and len(val) == 2:
327
- num, den = float(val[0]), float(val[1]) if float(val[1]) != 0 else 1.0
328
- return num / den
329
- return float(val)
330
- except Exception:
331
- return None
332
-
333
- def _estimate_sensor_width_mm(f_mm: Optional[float], f35: Optional[float]) -> Optional[float]:
334
- if f_mm and f35 and f35 > 0:
335
- return 36.0 * f_mm / f35
336
- return None
337
-
338
- def estimate_px_per_cm_from_exif(pil_img: Image.Image, default_px_per_cm: float = DEFAULT_PX_PER_CM) -> Tuple[float, Dict]:
339
- meta = {"used": "default", "f_mm": None, "f35": None, "sensor_w_mm": None, "distance_m": None}
340
- try:
341
- exif = _exif_to_dict(pil_img)
342
- f_mm = _to_float(exif.get("FocalLength"))
343
- f35 = _to_float(exif.get("FocalLengthIn35mmFilm") or exif.get("FocalLengthIn35mm"))
344
- subj_dist_m = _to_float(exif.get("SubjectDistance"))
345
- sensor_w_mm = _estimate_sensor_width_mm(f_mm, f35)
346
- meta.update({"f_mm": f_mm, "f35": f35, "sensor_w_mm": sensor_w_mm, "distance_m": subj_dist_m})
347
-
348
- if f_mm and sensor_w_mm and subj_dist_m and subj_dist_m > 0:
349
- w_px = pil_img.width
350
- field_w_mm = sensor_w_mm * (subj_dist_m * 1000.0) / f_mm
351
- field_w_cm = field_w_mm / 10.0
352
- px_per_cm = w_px / max(field_w_cm, 1e-6)
353
- px_per_cm = float(np.clip(px_per_cm, PX_PER_CM_MIN, PX_PER_CM_MAX))
354
- meta["used"] = "exif"
355
- return px_per_cm, meta
356
- return float(default_px_per_cm), meta
357
- except Exception:
358
- return float(default_px_per_cm), meta
359
-
360
- # ---------- Segmentation helpers ----------
361
- def _imagenet_norm(arr: np.ndarray) -> np.ndarray:
362
- mean = np.array([123.675, 116.28, 103.53], dtype=np.float32)
363
- std = np.array([58.395, 57.12, 57.375], dtype=np.float32)
364
- return (arr.astype(np.float32) - mean) / std
365
-
366
- def _preprocess_for_seg(bgr_roi: np.ndarray, target_hw: Tuple[int, int]) -> np.ndarray:
367
- H, W = target_hw
368
- resized = cv2.resize(bgr_roi, (W, H), interpolation=cv2.INTER_LINEAR)
369
- if SEG_EXPECTS_RGB:
370
- resized = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB)
371
- if SEG_NORM.lower() == "imagenet":
372
- x = _imagenet_norm(resized)
373
- else:
374
- x = resized.astype(np.float32) / 255.0
375
- x = np.expand_dims(x, axis=0) # (1,H,W,3)
376
- return x
377
-
378
- def _to_prob(pred: np.ndarray) -> np.ndarray:
379
- p = np.squeeze(pred)
380
- pmin, pmax = float(p.min()), float(p.max())
381
- if pmax > 1.0 or pmin < 0.0:
382
- p = 1.0 / (1.0 + np.exp(-p))
383
- return p.astype(np.float32)
384
-
385
- # ---- Adaptive threshold + GrabCut grow ----
386
- def _adaptive_prob_threshold(p: np.ndarray) -> float:
387
- """
388
- Choose a threshold that avoids tiny blobs while not swallowing skin.
389
- Try Otsu and the 90th percentile, clamp to [0.25, 0.65], pick by area heuristic.
390
- """
391
- p01 = np.clip(p.astype(np.float32), 0, 1)
392
- p255 = (p01 * 255).astype(np.uint8)
393
-
394
- ret_otsu, _ = cv2.threshold(p255, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
395
- thr_otsu = float(np.clip(ret_otsu / 255.0, 0.25, 0.65))
396
- thr_pctl = float(np.clip(np.percentile(p01, 90), 0.25, 0.65))
397
-
398
- def area_frac(thr: float) -> float:
399
- return float((p01 >= thr).sum()) / float(p01.size)
400
-
401
- af_otsu = area_frac(thr_otsu)
402
- af_pctl = area_frac(thr_pctl)
403
-
404
- def score(af: float) -> float:
405
- target_low, target_high = 0.03, 0.10
406
- if af < target_low: return abs(af - target_low) * 3.0
407
- if af > target_high: return abs(af - target_high) * 1.5
408
- return 0.0
409
-
410
- return thr_otsu if score(af_otsu) <= score(af_pctl) else thr_pctl
411
-
412
- def _grabcut_refine(bgr: np.ndarray, seed01: np.ndarray, iters: int = 3) -> np.ndarray:
413
- """Grow from a confident core into low-contrast margins."""
414
- h, w = bgr.shape[:2]
415
- gc = np.full((h, w), cv2.GC_PR_BGD, np.uint8)
416
- k = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
417
- seed_dil = cv2.dilate(seed01, k, iterations=1)
418
- gc[seed01.astype(bool)] = cv2.GC_PR_FGD
419
- gc[seed_dil.astype(bool)] = cv2.GC_FGD
420
- gc[0, :], gc[-1, :], gc[:, 0], gc[:, 1] = cv2.GC_BGD, cv2.GC_BGD, cv2.GC_BGD, cv2.GC_BGD
421
- bgdModel = np.zeros((1, 65), np.float64)
422
- fgdModel = np.zeros((1, 65), np.float64)
423
- cv2.grabCut(bgr, gc, None, bgdModel, fgdModel, iters, cv2.GC_INIT_WITH_MASK)
424
- return np.where((gc == cv2.GC_FGD) | (gc == cv2.GC_PR_FGD), 1, 0).astype(np.uint8)
425
-
426
- def _fill_holes(mask01: np.ndarray) -> np.ndarray:
427
- h, w = mask01.shape[:2]
428
- ff = np.zeros((h + 2, w + 2), np.uint8)
429
- m = (mask01 * 255).astype(np.uint8).copy()
430
- cv2.floodFill(m, ff, (0, 0), 255)
431
- m_inv = cv2.bitwise_not(m)
432
- out = ((mask01 * 255) | m_inv) // 255
433
- return out.astype(np.uint8)
434
-
435
- def _clean_mask(mask01: np.ndarray) -> np.ndarray:
436
- """Open → Close → Fill holes → Largest component (no dilation)."""
437
- mask01 = (mask01 > 0).astype(np.uint8)
438
- k3 = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
439
- k5 = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
440
- mask01 = cv2.morphologyEx(mask01, cv2.MORPH_OPEN, k3, iterations=1)
441
- mask01 = cv2.morphologyEx(mask01, cv2.MORPH_CLOSE, k5, iterations=1)
442
- mask01 = _fill_holes(mask01)
443
- # Keep largest component only
444
- num, labels, stats, _ = cv2.connectedComponentsWithStats(mask01, 8)
445
- if num > 1:
446
- areas = stats[1:, cv2.CC_STAT_AREA]
447
- if areas.size:
448
- largest_idx = 1 + int(np.argmax(areas))
449
- mask01 = (labels == largest_idx).astype(np.uint8)
450
- return (mask01 > 0).astype(np.uint8)
451
-
452
- # Global last debug dict (per-process)
453
- _last_seg_debug: Dict[str, object] = {}
454
-
455
- def segment_wound(image_bgr: np.ndarray, ts: str, out_dir: str) -> Tuple[np.ndarray, Dict[str, object]]:
456
- """
457
- TF model → adaptive threshold on prob → GrabCut grow → cleanup.
458
- Fallback: KMeans-Lab.
459
- Returns (mask_uint8_0_255, debug_dict)
460
- """
461
- debug = {"used": None, "reason": None, "positive_fraction": 0.0,
462
- "thr": None, "heatmap_path": None, "roi_seen_by_model": None}
463
-
464
- seg_model = models_cache.get("seg", None)
465
-
466
- # --- Model path ---
467
- if seg_model is not None:
468
- try:
469
- ishape = getattr(seg_model, "input_shape", None)
470
- if not ishape or len(ishape) < 4:
471
- raise ValueError(f"Bad seg input_shape: {ishape}")
472
- th, tw = int(ishape[1]), int(ishape[2])
473
-
474
- x = _preprocess_for_seg(image_bgr, (th, tw))
475
- roi_seen_path = None
476
- if SMARTHEAL_DEBUG:
477
- roi_seen_path = os.path.join(out_dir, f"roi_for_seg_{ts}.png")
478
- cv2.imwrite(roi_seen_path, image_bgr)
479
-
480
- pred = seg_model.predict(x, verbose=0)
481
- if isinstance(pred, (list, tuple)): pred = pred[0]
482
- p = _to_prob(pred)
483
- p = cv2.resize(p, (image_bgr.shape[1], image_bgr.shape[0]), interpolation=cv2.INTER_LINEAR)
484
-
485
- heatmap_path = None
486
- if SMARTHEAL_DEBUG:
487
- hm = (np.clip(p, 0, 1) * 255).astype(np.uint8)
488
- heat = cv2.applyColorMap(hm, cv2.COLORMAP_JET)
489
- heatmap_path = os.path.join(out_dir, f"seg_pred_heatmap_{ts}.png")
490
- cv2.imwrite(heatmap_path, heat)
491
-
492
- thr = _adaptive_prob_threshold(p)
493
- core01 = (p >= thr).astype(np.uint8)
494
- core_frac = float(core01.sum()) / float(core01.size)
495
-
496
- if core_frac < 0.005:
497
- thr2 = max(thr - 0.10, 0.15)
498
- core01 = (p >= thr2).astype(np.uint8)
499
- thr = thr2
500
- core_frac = float(core01.sum()) / float(core01.size)
501
-
502
- if core01.any():
503
- gc01 = _grabcut_refine(image_bgr, core01, iters=3)
504
- mask01 = _clean_mask(gc01)
505
- else:
506
- mask01 = np.zeros(core01.shape, np.uint8)
507
-
508
- pos_frac = float(mask01.sum()) / float(mask01.size)
509
- logging.info(f"SegModel USED | thr={float(thr):.2f} core_frac={core_frac:.4f} final_frac={pos_frac:.4f}")
510
-
511
- debug.update({
512
- "used": "tf_model",
513
- "reason": "ok",
514
- "positive_fraction": pos_frac,
515
- "thr": float(thr),
516
- "heatmap_path": heatmap_path,
517
- "roi_seen_by_model": roi_seen_path
518
- })
519
- return (mask01 * 255).astype(np.uint8), debug
520
-
521
- except Exception as e:
522
- logging.warning(f"⚠️ Segmentation model failed → fallback. Reason: {e}")
523
- debug.update({"used": "fallback_kmeans", "reason": f"model_failed: {e}"})
524
-
525
- # --- Fallback: KMeans in Lab (reddest cluster as wound) ---
526
- Z = image_bgr.reshape((-1, 3)).astype(np.float32)
527
- criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)
528
- _, labels, centers = cv2.kmeans(Z, 2, None, criteria, 5, cv2.KMEANS_PP_CENTERS)
529
- centers_u8 = centers.astype(np.uint8).reshape(1, 2, 3)
530
- centers_lab = cv2.cvtColor(centers_u8, cv2.COLOR_BGR2LAB)[0]
531
- wound_idx = int(np.argmax(centers_lab[:, 1])) # maximize a* (red)
532
- mask01 = (labels.reshape(image_bgr.shape[:2]) == wound_idx).astype(np.uint8)
533
- mask01 = _clean_mask(mask01)
534
-
535
- pos_frac = float(mask01.sum()) / float(mask01.size)
536
- logging.info(f"KMeans USED | final_frac={pos_frac:.4f}")
537
-
538
- debug.update({
539
- "used": "fallback_kmeans",
540
- "reason": debug.get("reason") or "no_model",
541
- "positive_fraction": pos_frac,
542
- "thr": None
543
- })
544
- return (mask01 * 255).astype(np.uint8), debug
545
-
546
- # ---------- Measurement + overlay helpers ----------
547
- def largest_component_mask(binary01: np.ndarray, min_area_px: int = 50) -> np.ndarray:
548
- num, labels, stats, _ = cv2.connectedComponentsWithStats(binary01.astype(np.uint8), connectivity=8)
549
- if num <= 1:
550
- return binary01.astype(np.uint8)
551
- areas = stats[1:, cv2.CC_STAT_AREA]
552
- if areas.size == 0 or areas.max() < min_area_px:
553
- return binary01.astype(np.uint8)
554
- largest_idx = 1 + int(np.argmax(areas))
555
- return (labels == largest_idx).astype(np.uint8)
556
-
557
- def measure_min_area_rect(mask01: np.ndarray, px_per_cm: float) -> Tuple[float, float, Tuple]:
558
- contours, _ = cv2.findContours(mask01.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
559
- if not contours:
560
- return 0.0, 0.0, (None, None)
561
- cnt = max(contours, key=cv2.contourArea)
562
- rect = cv2.minAreaRect(cnt)
563
- (w_px, h_px) = rect[1]
564
- length_px, breadth_px = (max(w_px, h_px), min(w_px, h_px))
565
- length_cm = round(length_px / max(px_per_cm, 1e-6), 2)
566
- breadth_cm = round(breadth_px / max(px_per_cm, 1e-6), 2)
567
- box = cv2.boxPoints(rect).astype(int)
568
- return length_cm, breadth_cm, (box, rect[0])
569
-
570
- def area_cm2_from_contour(mask01: np.ndarray, px_per_cm: float) -> Tuple[float, Optional[np.ndarray]]:
571
- """Area from largest polygon (sub-pixel); returns (area_cm2, contour)."""
572
- m = (mask01 > 0).astype(np.uint8)
573
- contours, _ = cv2.findContours(m, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
574
- if not contours:
575
- return 0.0, None
576
- cnt = max(contours, key=cv2.contourArea)
577
- poly_area_px2 = float(cv2.contourArea(cnt))
578
- area_cm2 = round(poly_area_px2 / (max(px_per_cm, 1e-6) ** 2), 2)
579
- return area_cm2, cnt
580
-
581
- def clamp_area_with_minrect(cnt: np.ndarray, px_per_cm: float, area_cm2_poly: float) -> float:
582
- rect = cv2.minAreaRect(cnt)
583
- (w_px, h_px) = rect[1]
584
- rect_area_px2 = float(max(w_px, 0.0) * max(h_px, 0.0))
585
- rect_area_cm2 = rect_area_px2 / (max(px_per_cm, 1e-6) ** 2)
586
- return round(min(area_cm2_poly, rect_area_cm2 * 1.05), 2)
587
-
588
- def draw_measurement_overlay(
589
- base_bgr: np.ndarray,
590
- mask01: np.ndarray,
591
- rect_box: np.ndarray,
592
- length_cm: float,
593
- breadth_cm: float,
594
- thickness: int = 2
595
- ) -> np.ndarray:
596
- """
597
- 1) Strong red mask overlay + white contour
598
- 2) Min-area rectangle
599
- 3) Double-headed arrows labeled Length/Width
600
- """
601
- overlay = base_bgr.copy()
602
-
603
- # Mask tint
604
- mask255 = (mask01 * 255).astype(np.uint8)
605
- mask3 = cv2.merge([mask255, mask255, mask255])
606
- red = np.zeros_like(overlay); red[:] = (0, 0, 255)
607
- alpha = 0.55
608
- tinted = cv2.addWeighted(overlay, 1 - alpha, red, alpha, 0)
609
- overlay = np.where(mask3 > 0, tinted, overlay)
610
-
611
- # Contour
612
- cnts, _ = cv2.findContours(mask255, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
613
- if cnts:
614
- cv2.drawContours(overlay, cnts, -1, (255, 255, 255), 2)
615
-
616
- if rect_box is not None:
617
- cv2.polylines(overlay, [rect_box], True, (255, 255, 255), thickness)
618
- pts = rect_box.reshape(-1, 2)
619
-
620
- def midpoint(a, b): return (int((a[0] + b[0]) / 2), int((a[1] + b[1]) / 2))
621
- e = [np.linalg.norm(pts[i] - pts[(i + 1) % 4]) for i in range(4)]
622
- long_edge_idx = int(np.argmax(e))
623
- mids = [midpoint(pts[i], pts[(i + 1) % 4]) for i in range(4)]
624
- long_pair = (long_edge_idx, (long_edge_idx + 2) % 4)
625
- short_pair = ((long_edge_idx + 1) % 4, (long_edge_idx + 3) % 4)
626
-
627
- def draw_double_arrow(img, p1, p2):
628
- cv2.arrowedLine(img, p1, p2, (0, 0, 0), thickness + 2, tipLength=0.05)
629
- cv2.arrowedLine(img, p2, p1, (0, 0, 0), thickness + 2, tipLength=0.05)
630
- cv2.arrowedLine(img, p1, p2, (255, 255, 255), thickness, tipLength=0.05)
631
- cv2.arrowedLine(img, p2, p1, (255, 255, 255), thickness, tipLength=0.05)
632
-
633
- def put_label(text, anchor):
634
- org = (anchor[0] + 6, anchor[1] - 6)
635
- cv2.putText(overlay, text, org, cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 4, cv2.LINE_AA)
636
- cv2.putText(overlay, text, org, cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2, cv2.LINE_AA)
637
-
638
- draw_double_arrow(overlay, mids[long_pair[0]], mids[long_pair[1]])
639
- draw_double_arrow(overlay, mids[short_pair[0]], mids[short_pair[1]])
640
- put_label(f"Length: {length_cm:.2f} cm", mids[long_pair[0]])
641
- put_label(f"Width: {breadth_cm:.2f} cm", mids[short_pair[0]])
642
-
643
- return overlay
644
 
645
- # ---------- AI PROCESSOR ----------
646
  class AIProcessor:
647
  def __init__(self):
648
- self.models_cache = models_cache
649
- self.knowledge_base_cache = knowledge_base_cache
650
- self.uploads_dir = UPLOADS_DIR
651
- self.dataset_id = DATASET_ID
652
- self.hf_token = HF_TOKEN
653
-
654
- def _ensure_analysis_dir(self) -> str:
655
- out_dir = os.path.join(self.uploads_dir, "analysis")
656
- os.makedirs(out_dir, exist_ok=True)
657
- return out_dir
658
 
659
- def perform_visual_analysis(self, image_pil: Image.Image) -> Dict:
660
- """
661
- YOLO detect → crop ROI → segment_wound(ROI) → clean mask →
662
- minAreaRect measurement (cm) using EXIF px/cm → save outputs.
663
- """
664
  try:
665
- px_per_cm, exif_meta = estimate_px_per_cm_from_exif(image_pil, DEFAULT_PX_PER_CM)
666
- # Guardrails for calibration to avoid huge area blow-ups
667
- px_per_cm = float(np.clip(px_per_cm, 20.0, 350.0))
668
- if (exif_meta or {}).get("used") != "exif":
669
- logging.warning(f"Calibration fallback used: px_per_cm={px_per_cm:.2f} (default). Prefer ruler/Aruco for accuracy.")
670
-
671
- image_cv = cv2.cvtColor(np.array(image_pil.convert("RGB")), cv2.COLOR_RGB2BGR)
672
-
673
- # --- Detection ---
674
- det_model = self.models_cache.get("det")
675
- if det_model is None:
676
- raise RuntimeError("YOLO model not loaded")
677
- # Force CPU inference and avoid CUDA touch
678
- results = det_model.predict(image_cv, verbose=False, device="cpu")
679
- if (not results) or (not getattr(results[0], "boxes", None)) or (len(results[0].boxes) == 0):
680
- try:
681
- import gradio as gr
682
- raise gr.Error("No wound could be detected.")
683
- except Exception:
684
- raise RuntimeError("No wound could be detected.")
685
-
686
- box = results[0].boxes[0].xyxy[0].cpu().numpy().astype(int)
687
- x1, y1, x2, y2 = [int(v) for v in box]
688
- x1, y1 = max(0, x1), max(0, y1)
689
- x2, y2 = min(image_cv.shape[1], x2), min(image_cv.shape[0], y2)
690
- roi = image_cv[y1:y2, x1:x2].copy()
691
- if roi.size == 0:
692
- try:
693
- import gradio as gr
694
- raise gr.Error("Detected ROI is empty.")
695
- except Exception:
696
- raise RuntimeError("Detected ROI is empty.")
697
-
698
- out_dir = self._ensure_analysis_dir()
699
- ts = datetime.now().strftime("%Y%m%d_%H%M%S")
700
-
701
- # --- Segmentation (model-first + KMeans fallback) ---
702
- mask_u8_255, seg_debug = segment_wound(roi, ts, out_dir)
703
- mask01 = (mask_u8_255 > 127).astype(np.uint8)
704
-
705
- if mask01.any():
706
- mask01 = _clean_mask(mask01)
707
- logging.debug(f"Mask postproc: px_after={int(mask01.sum())}")
708
-
709
- # --- Measurement (accurate & conservative) ---
710
- if mask01.any():
711
- length_cm, breadth_cm, (box_pts, _) = measure_min_area_rect(mask01, px_per_cm)
712
- area_poly_cm2, largest_cnt = area_cm2_from_contour(mask01, px_per_cm)
713
- if largest_cnt is not None:
714
- surface_area_cm2 = clamp_area_with_minrect(largest_cnt, px_per_cm, area_poly_cm2)
715
- else:
716
- surface_area_cm2 = area_poly_cm2
717
-
718
- anno_roi = draw_measurement_overlay(roi, mask01, box_pts, length_cm, breadth_cm)
719
- segmentation_empty = False
720
- else:
721
- # Fallback if seg failed: use ROI dimensions
722
- h_px = max(0, y2 - y1); w_px = max(0, x2 - x1)
723
- length_cm = round(max(h_px, w_px) / px_per_cm, 2)
724
- breadth_cm = round(min(h_px, w_px) / px_per_cm, 2)
725
- surface_area_cm2 = round((h_px * w_px) / (px_per_cm ** 2), 2)
726
- anno_roi = roi.copy()
727
- cv2.rectangle(anno_roi, (2, 2), (anno_roi.shape[1]-3, anno_roi.shape[0]-3), (0, 0, 255), 3)
728
- cv2.line(anno_roi, (0, 0), (anno_roi.shape[1]-1, anno_roi.shape[0]-1), (0, 0, 255), 2)
729
- cv2.line(anno_roi, (anno_roi.shape[1]-1, 0), (0, anno_roi.shape[0]-1), (0, 0, 255), 2)
730
- box_pts = None
731
- segmentation_empty = True
732
-
733
- # --- Save visualizations ---
734
- original_path = os.path.join(out_dir, f"original_{ts}.png")
735
- cv2.imwrite(original_path, image_cv)
736
 
737
- det_vis = image_cv.copy()
738
- cv2.rectangle(det_vis, (x1, y1), (x2, y2), (0, 255, 0), 2)
739
- detection_path = os.path.join(out_dir, f"detection_{ts}.png")
740
- cv2.imwrite(detection_path, det_vis)
741
 
742
- roi_mask_path = os.path.join(out_dir, f"roi_mask_{ts}.png")
743
- cv2.imwrite(roi_mask_path, (mask01 * 255).astype(np.uint8))
744
-
745
- # ROI overlay (mask tint + contour, without arrows)
746
- mask255 = (mask01 * 255).astype(np.uint8)
747
- mask3 = cv2.merge([mask255, mask255, mask255])
748
- red = np.zeros_like(roi); red[:] = (0, 0, 255)
749
- alpha = 0.55
750
- tinted = cv2.addWeighted(roi, 1 - alpha, red, alpha, 0)
751
- if mask255.any():
752
- roi_overlay = np.where(mask3 > 0, tinted, roi)
753
- cnts, _ = cv2.findContours(mask255, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
754
- cv2.drawContours(roi_overlay, cnts, -1, (255, 255, 255), 2)
755
- else:
756
- roi_overlay = anno_roi
757
-
758
- seg_full = image_cv.copy()
759
- seg_full[y1:y2, x1:x2] = roi_overlay
760
- segmentation_path = os.path.join(out_dir, f"segmentation_{ts}.png")
761
- cv2.imwrite(segmentation_path, seg_full)
762
-
763
- segmentation_roi_path = os.path.join(out_dir, f"segmentation_roi_{ts}.png")
764
- cv2.imwrite(segmentation_roi_path, roi_overlay)
765
 
766
- # Annotated (mask + arrows + labels) in full-frame
767
- anno_full = image_cv.copy()
768
- anno_full[y1:y2, x1:x2] = anno_roi
769
- annotated_seg_path = os.path.join(out_dir, f"segmentation_annotated_{ts}.png")
770
- cv2.imwrite(annotated_seg_path, anno_full)
771
 
772
- # --- Optional classification ---
773
- wound_type = "Unknown"
774
- cls_pipe = self.models_cache.get("cls")
775
- if cls_pipe is not None:
776
- try:
777
- preds = cls_pipe(Image.fromarray(cv2.cvtColor(roi, cv2.COLOR_BGR2RGB)))
778
- if preds:
779
- wound_type = max(preds, key=lambda x: x.get("score", 0)).get("label", "Unknown")
780
- except Exception as e:
781
- logging.warning(f"Classification failed: {e}")
782
 
783
- # Log end-of-seg summary
784
- seg_summary = {
785
- "seg_used": seg_debug.get("used"),
786
- "seg_reason": seg_debug.get("reason"),
787
- "positive_fraction": round(float(seg_debug.get("positive_fraction", 0.0)), 6),
788
- "threshold": seg_debug.get("thr"),
789
- "segmentation_empty": segmentation_empty,
790
- "exif_px_per_cm": round(px_per_cm, 3),
791
- }
792
- _log_kv("SEG_SUMMARY", seg_summary)
793
 
794
- return {
795
- "wound_type": wound_type,
796
- "length_cm": length_cm,
797
- "breadth_cm": breadth_cm,
798
- "surface_area_cm2": surface_area_cm2,
799
- "px_per_cm": round(px_per_cm, 2),
800
- "calibration_meta": exif_meta,
801
- "detection_confidence": float(results[0].boxes.conf[0].cpu().item())
802
- if getattr(results[0].boxes, "conf", None) is not None else 0.0,
803
- "detection_image_path": detection_path,
804
- "segmentation_image_path": annotated_seg_path,
805
- "segmentation_annotated_path": annotated_seg_path,
806
- "segmentation_roi_path": segmentation_roi_path,
807
- "roi_mask_path": roi_mask_path,
808
- "segmentation_empty": segmentation_empty,
809
- "segmentation_debug": seg_debug,
810
- "original_image_path": original_path,
811
- }
812
  except Exception as e:
813
- logging.error(f"Visual analysis failed: {e}", exc_info=True)
814
- raise
815
 
816
- # ---------- Knowledge base + reporting ----------
817
- def query_guidelines(self, query: str) -> str:
818
  try:
819
- vs = self.knowledge_base_cache.get("vector_store")
820
- if not vs:
821
- return "Knowledge base is not available."
822
- retriever = vs.as_retriever(search_kwargs={"k": 5})
823
- # Modern API (avoid get_relevant_documents deprecation)
824
- docs = retriever.invoke(query)
825
- lines: List[str] = []
826
- for d in docs:
827
- src = (d.metadata or {}).get("source", "N/A")
828
- txt = (d.page_content or "")[:300]
829
- lines.append(f"Source: {src}\nContent: {txt}...")
830
- return "\n\n".join(lines) if lines else "No relevant guideline snippets found."
 
831
  except Exception as e:
832
- logging.warning(f"Guidelines query failed: {e}")
833
- return f"Guidelines query failed: {str(e)}"
834
-
835
- def _generate_fallback_report(self, patient_info: str, visual_results: Dict, guideline_context: str) -> str:
836
- return f"""# 🩺 SmartHeal AI - Comprehensive Wound Analysis Report
837
-
838
- ## 📋 Patient Information
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
839
  {patient_info}
840
-
841
- ## 🔍 Visual Analysis Results
842
- - **Wound Type**: {visual_results.get('wound_type', 'Unknown')}
843
- - **Dimensions**: {visual_results.get('length_cm', 0)} cm × {visual_results.get('breadth_cm', 0)} cm
844
- - **Surface Area**: {visual_results.get('surface_area_cm2', 0)} cm²
845
- - **Detection Confidence**: {visual_results.get('detection_confidence', 0):.1%}
846
- - **Calibration**: {visual_results.get('px_per_cm','?')} px/cm ({(visual_results.get('calibration_meta') or {}).get('used','default')})
847
-
848
- ## 📊 Analysis Images
849
- - **Original**: {visual_results.get('original_image_path', 'N/A')}
850
- - **Detection**: {visual_results.get('detection_image_path', 'N/A')}
851
- - **Segmentation**: {visual_results.get('segmentation_image_path', 'N/A')}
852
- - **Annotated**: {visual_results.get('segmentation_annotated_path', 'N/A')}
853
-
854
- ## 🎯 Clinical Summary
855
- Automated analysis provides quantitative measurements; verify via clinical examination.
856
-
857
- ## 💊 Recommendations
858
- - Cleanse wound gently; select dressing per exudate/infection risk
859
- - Debride necrotic tissue if indicated (clinical decision)
860
- - Document with serial photos and measurements
861
-
862
- ## 📅 Monitoring
863
- - Daily in week 1, then every 2–3 days (or as indicated)
864
- - Weekly progress review
865
-
866
- ## 📚 Guideline Context
867
- {(guideline_context or '')[:800]}{"..." if guideline_context and len(guideline_context) > 800 else ''}
868
-
869
- **Disclaimer:** Automated, for decision support only. Verify clinically.
870
  """
871
 
872
- def generate_final_report(
873
- self,
874
- patient_info: str,
875
- visual_results: Dict,
876
- guideline_context: str,
877
- image_pil: Image.Image,
878
- max_new_tokens: Optional[int] = None,
879
- ) -> str:
880
- try:
881
- report = generate_medgemma_report(
882
- patient_info, visual_results, guideline_context, image_pil, max_new_tokens
883
- )
884
- if report and report.strip() and not report.startswith(("⚠️", "❌")):
885
- return report
886
- logging.warning("VLM unavailable/invalid; using fallback.")
887
- return self._generate_fallback_report(patient_info, visual_results, guideline_context)
888
- except Exception as e:
889
- logging.error(f"Report generation failed: {e}")
890
- return self._generate_fallback_report(patient_info, visual_results, guideline_context)
891
 
892
- def save_and_commit_image(self, image_pil: Image.Image) -> str:
893
  try:
894
- os.makedirs(self.uploads_dir, exist_ok=True)
895
- ts = datetime.now().strftime("%Y%m%d_%H%M%S")
896
- filename = f"{ts}.png"
897
- path = os.path.join(self.uploads_dir, filename)
898
- image_pil.convert("RGB").save(path)
899
- logging.info(f"✅ Image saved locally: {path}")
900
-
901
- if HF_TOKEN and DATASET_ID:
902
- try:
903
- HfApi, HfFolder = _import_hf_hub()
904
- HfFolder.save_token(HF_TOKEN)
905
- api = HfApi()
906
- api.upload_file(
907
- path_or_fileobj=path,
908
- path_in_repo=f"images/{filename}",
909
- repo_id=DATASET_ID,
910
- repo_type="dataset",
911
- token=HF_TOKEN,
912
- commit_message=f"Upload wound image: {filename}",
913
- )
914
- logging.info("✅ Image committed to HF dataset")
915
- except Exception as e:
916
- logging.warning(f"HF upload failed: {e}")
917
-
918
- return path
919
  except Exception as e:
920
- logging.error(f"Failed to save/commit image: {e}")
921
- return ""
922
-
923
- def full_analysis_pipeline(self, image_pil: Image.Image, questionnaire_data: Dict) -> Dict:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
924
  try:
925
- saved_path = self.save_and_commit_image(image_pil)
926
- visual_results = self.perform_visual_analysis(image_pil)
927
-
928
- pi = questionnaire_data or {}
929
- patient_info = (
930
- f"Age: {pi.get('age','N/A')}, "
931
- f"Diabetic: {pi.get('diabetic','N/A')}, "
932
- f"Allergies: {pi.get('allergies','N/A')}, "
933
- f"Date of Wound: {pi.get('date_of_injury','N/A')}, "
934
- f"Professional Care: {pi.get('professional_care','N/A')}, "
935
- f"Oozing/Bleeding: {pi.get('oozing_bleeding','N/A')}, "
936
- f"Infection: {pi.get('infection','N/A')}, "
937
- f"Moisture: {pi.get('moisture','N/A')}"
938
- )
939
-
940
- query = (
941
- f"best practices for managing a {visual_results.get('wound_type','Unknown')} "
942
- f"with moisture '{pi.get('moisture','unknown')}' and infection '{pi.get('infection','unknown')}' "
943
- f"in a diabetic status '{pi.get('diabetic','unknown')}'"
944
- )
945
  guideline_context = self.query_guidelines(query)
946
 
947
- report = self.generate_final_report(patient_info, visual_results, guideline_context, image_pil)
948
-
949
- return {
950
- "success": True,
951
- "visual_analysis": visual_results,
952
- "report": report,
953
- "saved_image_path": saved_path,
954
- "guideline_context": (guideline_context or "")[:500] + (
955
- "..." if guideline_context and len(guideline_context) > 500 else ""
956
- ),
957
- }
958
- except Exception as e:
959
- logging.error(f"Pipeline error: {e}")
960
- return {
961
- "success": False,
962
- "error": str(e),
963
- "visual_analysis": {},
964
- "report": f"Analysis failed: {str(e)}",
965
- "saved_image_path": None,
966
- "guideline_context": "",
967
- }
968
-
969
- def analyze_wound(self, image, questionnaire_data: Dict) -> Dict:
970
- try:
971
- if isinstance(image, str):
972
- if not os.path.exists(image):
973
- raise ValueError(f"Image file not found: {image}")
974
- image_pil = Image.open(image)
975
- elif isinstance(image, Image.Image):
976
- image_pil = image
977
- elif isinstance(image, np.ndarray):
978
- image_pil = Image.fromarray(image)
979
- else:
980
- raise ValueError(f"Unsupported image type: {type(image)}")
981
 
982
- return self.full_analysis_pipeline(image_pil, questionnaire_data or {})
983
  except Exception as e:
984
- logging.error(f"Wound analysis error: {e}")
985
- return {
986
- "success": False,
987
- "error": str(e),
988
- "visual_analysis": {},
989
- "report": f"Analysis initialization failed: {str(e)}",
990
- "saved_image_path": None,
991
- "guideline_context": "",
992
- }
 
 
 
 
 
1
  import os
2
  import logging
 
 
 
 
 
 
 
 
3
  import cv2
4
  import numpy as np
5
  from PIL import Image
6
+ import torch
7
+ import json
8
+ from datetime import datetime
9
+ import tensorflow as tf
10
+ from transformers import pipeline
11
+ from ultralytics import YOLO
12
+ from tensorflow.keras.models import load_model
13
+ from langchain_community.document_loaders import PyPDFLoader
14
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
15
+ from langchain_community.embeddings import HuggingFaceEmbeddings
16
+ from langchain_community.vectorstores import FAISS
17
+ from huggingface_hub import HfApi, HfFolder
18
+ import spaces
19
+
20
+ from src.config import Config
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
 
 
22
  class AIProcessor:
23
  def __init__(self):
24
+ self.models_cache = {}
25
+ self.knowledge_base_cache = {}
26
+ self.config = Config()
27
+ self.px_per_cm = 38
28
+ self._initialize_models()
 
 
 
 
 
29
 
30
+ def _initialize_models(self):
 
 
 
 
31
  try:
32
+ HfFolder.save_token(self.config.HF_TOKEN)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
 
34
+ self.models_cache['yolo'] = YOLO(self.config.YOLO_MODEL_PATH)
35
+ self.models_cache['segmentation'] = load_model(self.config.SEG_MODEL_PATH, compile=False)
 
 
36
 
37
+ self.models_cache['medgemma_pipe'] = pipeline(
38
+ "image-text-to-text",
39
+ model="google/medgemma-4b-it",
40
+ torch_dtype=torch.bfloat16,
41
+ device_map="auto",
42
+ token=self.config.HF_TOKEN
43
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
 
45
+ self.models_cache['embedding_model'] = HuggingFaceEmbeddings(
46
+ model_name="sentence-transformers/all-MiniLM-L6-v2",
47
+ model_kwargs={'device': 'cpu'}
48
+ )
 
49
 
50
+ self.models_cache['cls'] = pipeline(
51
+ "image-classification",
52
+ model="Hemg/Wound-classification",
53
+ token=self.config.HF_TOKEN,
54
+ device="cpu"
55
+ )
 
 
 
 
56
 
57
+ logging.info("✅ All models loaded.")
58
+ self._load_knowledge_base()
 
 
 
 
 
 
 
 
59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  except Exception as e:
61
+ logging.error(f"Error initializing AI models: {e}")
 
62
 
63
+ def _load_knowledge_base(self):
 
64
  try:
65
+ docs = []
66
+ for pdf in self.config.GUIDELINE_PDFS:
67
+ if os.path.exists(pdf):
68
+ loader = PyPDFLoader(pdf)
69
+ docs.extend(loader.load())
70
+ if docs:
71
+ splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
72
+ chunks = splitter.split_documents(docs)
73
+ vectorstore = FAISS.from_documents(chunks, self.models_cache['embedding_model'])
74
+ self.knowledge_base_cache['vectorstore'] = vectorstore
75
+ logging.info(" Knowledge base loaded.")
76
+ else:
77
+ self.knowledge_base_cache['vectorstore'] = None
78
  except Exception as e:
79
+ logging.warning(f"Knowledge base error: {e}")
80
+
81
+ def perform_visual_analysis(self, image_pil):
82
+ image_cv = cv2.cvtColor(np.array(image_pil), cv2.COLOR_RGB2BGR)
83
+ results = self.models_cache['yolo'].predict(image_cv, verbose=False, device="cpu")
84
+ if not results or not results[0].boxes:
85
+ raise ValueError("No wound detected.")
86
+
87
+ box = results[0].boxes[0].xyxy[0].cpu().numpy().astype(int)
88
+ region_cv = image_cv[box[1]:box[3], box[0]:box[2]]
89
+
90
+ input_size = self.models_cache['segmentation'].input_shape[1:3]
91
+ resized = cv2.resize(region_cv, (input_size[1], input_size[0]))
92
+ mask = self.models_cache['segmentation'].predict(np.expand_dims(resized / 255.0, 0), verbose=0)[0]
93
+ mask_np = (mask[:, :, 0] > 0.5).astype(np.uint8)
94
+
95
+ contours, _ = cv2.findContours(mask_np, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
96
+ length = breadth = area = 0
97
+ if contours:
98
+ cnt = max(contours, key=cv2.contourArea)
99
+ x, y, w, h = cv2.boundingRect(cnt)
100
+ length = round(h / self.px_per_cm, 2)
101
+ breadth = round(w / self.px_per_cm, 2)
102
+ area = round(cv2.contourArea(cnt) / (self.px_per_cm ** 2), 2)
103
+
104
+ wound_type = max(self.models_cache['cls'](Image.fromarray(cv2.cvtColor(region_cv, cv2.COLOR_BGR2RGB))), key=lambda x: x['score'])['label']
105
+
106
+ return {
107
+ 'wound_type': wound_type,
108
+ 'length_cm': length,
109
+ 'breadth_cm': breadth,
110
+ 'surface_area_cm2': area
111
+ }
112
+
113
+ def query_guidelines(self, query: str):
114
+ vector_store = self.knowledge_base_cache.get("vectorstore")
115
+ if not vector_store:
116
+ return "Knowledge base unavailable."
117
+
118
+ retriever = vector_store.as_retriever(search_kwargs={"k": 10})
119
+ docs = retriever.invoke(query)
120
+ return "\n\n".join([
121
+ f"Source: {doc.metadata.get('source', 'N/A')}, Page: {doc.metadata.get('page', 'N/A')}\nContent: {doc.page_content}"
122
+ for doc in docs
123
+ ])
124
+
125
+ def generate_final_report(self, patient_info, visual_results, guideline_context, image_pil, max_new_tokens=2048):
126
+ prompt = f"""
127
+ 🩺 You are SmartHeal-AI, a world-class wound care AI specialist trained in clinical wound assessment and guideline-based treatment planning.
128
+ Your task is to process the following structured inputs (patient data, wound measurements, clinical guidelines, and image) and perform **clinical reasoning and decision-making** to generate a complete wound care report.
129
+ ---
130
+ 🔍 **YOUR PROCESS — FOLLOW STRICTLY:**
131
+ ### Step 1: Clinical Reasoning (Chain-of-Thought)
132
+ Use the provided information to think step-by-step about:
133
+ - Patient’s risk factors (e.g. diabetes, age, healing limitations)
134
+ - Wound characteristics (size, tissue appearance, moisture, infection signs)
135
+ - Visual clues from the image (location, granulation, maceration, inflammation, surrounding skin)
136
+ - Clinical guidelines provided — selectively choose the ones most relevant to this case
137
+ Do NOT list all guidelines verbatim. Use judgment: apply them where relevant. Explain why or why not.
138
+ Also assess whether this wound appears:
139
+ - Acute vs chronic
140
+ - Surgical vs traumatic
141
+ - Inflammatory vs proliferative healing phase
142
+ ---
143
+ ### Step 2: Structured Clinical Report
144
+ Generate the following report sections using markdown and medical terminology:
145
+ #### **1. Clinical Summary**
146
+ - Describe wound appearance and tissue types (e.g., slough, necrotic, granulating, epithelializing)
147
+ - Include size, wound bed condition, peri-wound skin, and signs of infection or biofilm
148
+ - Mention inferred location (e.g., heel, forefoot) if image allows
149
+ - Summarize patient's systemic risk profile
150
+ #### **2. Medicinal & Dressing Recommendations**
151
+ Based on your analysis:
152
+ - Recommend specific **wound care dressings** (e.g., hydrocolloid, alginate, foam, antimicrobial silver, etc.) suitable to wound moisture level and infection risk
153
+ - Propose **topical or systemic agents** ONLY if relevant — include name classes (e.g., antiseptic: povidone iodine, antibiotic ointments, enzymatic debriders)
154
+ - Mention **techniques** (e.g., sharp debridement, NPWT, moisture balance, pressure offloading, dressing frequency)
155
+ - Avoid repeating guidelines — **apply them**
156
+ #### **3. Key Risk Factors**
157
+ Explain how the patient’s condition (e.g., diabetic, poor circulation, advanced age, poor hygiene) may affect wound healing
158
+ #### **4. Prognosis & Monitoring Advice**
159
+ - Mention how often wound should be reassessed
160
+ - Indicate signs to monitor for deterioration or improvement
161
+ - Include when escalation to specialist is necessary
162
+ #### **5. Disclaimer**
163
+ This is an AI-generated summary based on available data. It is not a substitute for clinical evaluation by a wound care professional.
164
+ **Note:** Every dressing change is a chance for wound reassessment. Always perform a thorough wound evaluation at each dressing change.
165
+ ---
166
+ 🧾 **INPUT DATA**
167
+ **Patient Info:**
168
  {patient_info}
169
+ **Wound Details:**
170
+ - Type: {visual_results['wound_type']}
171
+ - Size: {visual_results['length_cm']} × {visual_results['breadth_cm']} cm
172
+ - Area: {visual_results['surface_area_cm2']} cm²
173
+ **Clinical Guideline Evidence:**
174
+ {guideline_context}
175
+ You may now begin your analysis and generate the two-part report.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
176
  """
177
 
178
+ messages = [
179
+ {
180
+ "role": "system",
181
+ "content": [{"type": "text", "text": "You are a world-class medical AI assistant..."}],
182
+ },
183
+ {
184
+ "role": "user",
185
+ "content": [
186
+ {"type": "image", "image": image_pil},
187
+ {"type": "text", "text": prompt},
188
+ ]
189
+ }
190
+ ]
 
 
 
 
 
 
191
 
 
192
  try:
193
+ output = self.models_cache['medgemma_pipe'](
194
+ text=messages,
195
+ max_new_tokens=max_new_tokens,
196
+ do_sample=False,
197
+ )
198
+ return output[0]['generated_text'][-1].get('content', '').strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
199
  except Exception as e:
200
+ logging.error(f"MedGemma error: {e}", exc_info=True)
201
+ return f"❌ Failed to generate report: {e}"
202
+
203
+ def save_and_commit_image(self, image_pil):
204
+ filename = f"{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.png"
205
+ local_path = os.path.join(self.config.UPLOADS_DIR, filename)
206
+ image_pil.convert("RGB").save(local_path)
207
+ logging.info(f"Image saved locally: {local_path}")
208
+
209
+ if self.config.HF_TOKEN and self.config.DATASET_ID:
210
+ try:
211
+ api = HfApi()
212
+ api.upload_file(
213
+ path_or_fileobj=local_path,
214
+ path_in_repo=f"images/{filename}",
215
+ repo_id=self.config.DATASET_ID,
216
+ repo_type="dataset",
217
+ commit_message=f"Upload wound image: {filename}"
218
+ )
219
+ logging.info("✅ Image uploaded to HF dataset.")
220
+ except Exception as e:
221
+ logging.warning(f"Upload failed: {e}")
222
+
223
+ @spaces.GPU(enable_queue=True, duration=120)
224
+ def full_analysis_pipeline(self, image, questionnaire_data):
225
  try:
226
+ self.save_and_commit_image(image)
227
+ visual = self.perform_visual_analysis(image)
228
+ patient_info = ", ".join([f"{k}: {v}" for k, v in questionnaire_data.items()])
229
+ query = f"best practices for managing a {visual['wound_type']} with moisture level '{questionnaire_data.get('moisture')}' and signs of infection '{questionnaire_data.get('infection')}' in a patient who is diabetic '{questionnaire_data.get('diabetic')}'"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
230
  guideline_context = self.query_guidelines(query)
231
 
232
+ return self.generate_final_report(patient_info, visual, guideline_context, image)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
233
 
 
234
  except Exception as e:
235
+ logging.error(f"Pipeline error: {e}", exc_info=True)
236
+ return f"❌ Error: {e}"