import base64 import json from datetime import datetime from io import BytesIO from typing import Any, Dict, List, Optional import requests from langchain_core.chat_history import InMemoryChatMessageHistory from langchain_core.messages import AIMessage, HumanMessage from langchain_core.prompts import PromptTemplate from langchain_core.runnables import RunnableLambda from PIL import Image # ============================================================ # CONFIG # ============================================================ ENDPOINT_URL = "https://hariharansuthan05--medgemma-1-5-deployment-medgemmaservi-1b5255.modal.run" def query_medgemma(prompt: str, image_data: str = None): """Sends a payload to the hosted MedGemma API endpoint.""" payload = {"prompt": prompt} if image_data: payload["image_data"] = image_data try: response = requests.post(ENDPOINT_URL, json=payload, timeout=300) response.raise_for_status() return response.json() except requests.exceptions.RequestException as e: return {"error": f"API request failed: {e}"} # ============================================================ # SYSTEM PROMPT # ============================================================ MEDICAL_SYSTEM_PROMPT = """ You are MedGemma AI, an expert medical imaging assistant. ROLE: - Radiology support - Differential diagnosis - Clinical reasoning OUTPUT FORMAT: 1. FINDINGS 2. IMPRESSION 3. DIFFERENTIAL DIAGNOSES 4. RECOMMENDATIONS Use medical terminology. Recommend clinical correlation. """ # ============================================================ # PROMPTS # ============================================================ image_analysis_template = PromptTemplate.from_template( """ Please analyze this {image_type} image. Clinical Question: {clinical_question} Provide: 1. Findings 2. Impression 3. Differential Diagnoses 4. Recommendations """ ) followup_template = PromptTemplate.from_template( """ Previous Context: {context} Question: {question} Provide a focused clinical answer. """ ) ddx_template = PromptTemplate.from_template( """ Imaging Findings: {findings} Provide: 1. Differential Diagnoses 2. Ranking by likelihood 3. Distinguishing features 4. Additional tests """ ) # ============================================================ # IMAGE UTIL # ============================================================ def image_to_data_url(image: Image.Image) -> str: buffer = BytesIO() image.convert("RGB").save(buffer, format="JPEG") encoded = base64.b64encode(buffer.getvalue()).decode() return f"data:image/jpeg;base64,{encoded}" def _prompt_to_text(prompt_value: Any) -> str: if hasattr(prompt_value, "to_string"): return prompt_value.to_string() return str(prompt_value) # ============================================================ # MEDGEMMA RUNNABLE # ============================================================ class MedGemmaRunnable: def __init__(self): self.system_prompt = MEDICAL_SYSTEM_PROMPT self.current_image: Optional[Image.Image] = None self.conversation_history: List[Dict[str, Any]] = [] def set_image(self, image: Optional[Image.Image]): self.current_image = image def reset(self): self.current_image = None self.conversation_history = [] def _prepare_prompt(self, prompt: str) -> str: if not self.conversation_history: return f"{self.system_prompt.strip()}\n\n{prompt}" return prompt def invoke(self, prompt: str, max_tokens: int = 1500) -> str: prompt = self._prepare_prompt(prompt) image = self.current_image image_url = None if image is not None: image_url = image_to_data_url(image) self.conversation_history.append({"role": "user", "content": prompt}) result = query_medgemma(prompt=prompt, image_data=image_url) if "error" in result: raise RuntimeError(result["error"]) response = result.get("response", "") if not response: raise RuntimeError("API returned an empty response.") self.conversation_history.append({"role": "assistant", "content": response}) self.current_image = None return response # Backward-compatible alias for app imports MedGemmaLLM = MedGemmaRunnable # ============================================================ # CHATBOT # ============================================================ class LangChainMedicalChatbot: def __init__(self, llm: MedGemmaRunnable): self.llm = llm self.chat_history = InMemoryChatMessageHistory() self.stats = { "images_analyzed": 0, "queries_processed": 0, "findings_detected": [], "session_start": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), } self._create_chains() def _create_chains(self): self.image_analysis_chain = image_analysis_template | RunnableLambda( lambda prompt: self.llm.invoke(_prompt_to_text(prompt)) ) self.followup_chain = followup_template | RunnableLambda( lambda prompt: self.llm.invoke(_prompt_to_text(prompt)) ) self.ddx_chain = ddx_template | RunnableLambda( lambda prompt: self.llm.invoke(_prompt_to_text(prompt)) ) def analyze_image( self, image: Image.Image, image_type: str, clinical_question: str, ) -> Dict: self.llm.set_image(image) response = self.image_analysis_chain.invoke( { "image_type": image_type, "clinical_question": clinical_question, } ) self.chat_history.add_message(HumanMessage(content=clinical_question)) self.chat_history.add_message(AIMessage(content=response)) self.stats["images_analyzed"] += 1 self.stats["queries_processed"] += 1 findings = self._detect_findings(response) if findings: self.stats["findings_detected"].extend(findings) return { "response": response, "findings": findings, "has_image": True, "chain_used": "image_analysis_chain", "timestamp": datetime.now().strftime("%H:%M:%S"), } def ask_followup(self, question: str) -> Dict: context = "\n".join([msg.content for msg in self.chat_history.messages][-10:]) response = self.followup_chain.invoke( { "question": question, "context": context, } ) self.chat_history.add_message(HumanMessage(content=question)) self.chat_history.add_message(AIMessage(content=response)) self.stats["queries_processed"] += 1 return { "response": response, "has_image": False, "chain_used": "followup_chain", "timestamp": datetime.now().strftime("%H:%M:%S"), } def get_differential_diagnosis(self, findings: str) -> Dict: response = self.ddx_chain.invoke({"findings": findings}) self.stats["queries_processed"] += 1 return { "response": response, "has_image": False, "chain_used": "ddx_chain", "timestamp": datetime.now().strftime("%H:%M:%S"), } def chat(self, message: str, image: Optional[Image.Image] = None) -> Dict: if image: self.llm.set_image(image) response = self.llm.invoke(message) self.chat_history.add_message(HumanMessage(content=message)) self.chat_history.add_message(AIMessage(content=response)) self.stats["queries_processed"] += 1 if image is not None: self.stats["images_analyzed"] += 1 findings = self._detect_findings(response) if findings: self.stats["findings_detected"].extend(findings) return { "response": response, "findings": findings, "has_image": image is not None, "chain_used": "direct_llm", "timestamp": datetime.now().strftime("%H:%M:%S"), } def _detect_findings(self, text: str) -> List[str]: keywords = [ "fracture", "lesion", "mass", "opacity", "consolidation", "pneumonia", "atelectasis", "effusion", "pneumothorax", "cardiomegaly", "edema", "nodule", "ischemia", ] text_lower = text.lower() return list(set(k.capitalize() for k in keywords if k in text_lower)) def get_memory(self) -> str: memory = [ {"type": type(msg).__name__, "content": msg.content} for msg in self.chat_history.messages ] return json.dumps(memory, indent=2) def get_stats(self) -> Dict: return { "Session Start": self.stats["session_start"], "Images Analyzed": self.stats["images_analyzed"], "Queries Processed": self.stats["queries_processed"], "Unique Findings": list(set(self.stats["findings_detected"])), "Total Findings": len(self.stats["findings_detected"]), } def reset(self): self.chat_history.clear() self.llm.reset() self.stats = { "images_analyzed": 0, "queries_processed": 0, "findings_detected": [], "session_start": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), }