import re import numpy as np import base64 import gradio as gr from pathlib import Path import time import shutil from typing import AsyncGenerator, List, Optional, Tuple from gradio import ChatMessage class ChatInterface: """ A chat interface for interacting with a medical AI agent through Gradio. Handles file uploads, message processing, and chat history management. Supports both regular image files and DICOM medical imaging files. """ def __init__(self, agent, tools_dict): """ Initialize the chat interface. Args: agent: The medical AI agent to handle requests tools_dict (dict): Dictionary of available tools for image processing """ self.agent = agent self.tools_dict = tools_dict self.upload_dir = Path("temp") self.upload_dir.mkdir(exist_ok=True) self.current_thread_id = None # Separate storage for original and display paths self.original_file_path = None # For LLM (.dcm or other) self.display_file_path = None # For UI (always viewable format) def handle_upload(self, file_path: str) -> str: """ Handle new file upload and set appropriate paths. Args: file_path (str): Path to the uploaded file Returns: str: Display path for UI, or None if no file uploaded """ if not file_path: return None source = Path(file_path) timestamp = int(time.time()) # Save original file with proper suffix suffix = source.suffix.lower() saved_path = self.upload_dir / f"upload_{timestamp}{suffix}" shutil.copy2(file_path, saved_path) # Use file_path directly instead of source self.original_file_path = str(saved_path) # Handle DICOM conversion for display only if suffix == ".dcm": output, _ = self.tools_dict["DicomProcessorTool"]._run(str(saved_path)) self.display_file_path = output["image_path"] else: self.display_file_path = str(saved_path) return self.display_file_path def add_message( self, message: str, display_image: str, history: List[dict] ) -> Tuple[List[dict], gr.Textbox]: """ Add a new message to the chat history. Args: message (str): Text message to add display_image (str): Path to image being displayed history (List[dict]): Current chat history Returns: Tuple[List[dict], gr.Textbox]: Updated history and textbox component """ image_path = self.original_file_path or display_image if image_path is not None: history.append({"role": "user", "content": {"path": image_path}}) if message is not None: history.append({"role": "user", "content": message}) return history, gr.Textbox(value=message, interactive=False) async def process_message( self, message: str, display_image: Optional[str], chat_history: List[ChatMessage] ) -> AsyncGenerator[Tuple[List[ChatMessage], Optional[str], str], None]: """ Process a message and generate responses. Args: message (str): User message to process display_image (Optional[str]): Path to currently displayed image chat_history (List[ChatMessage]): Current chat history Yields: Tuple[List[ChatMessage], Optional[str], str]: Updated chat history, display path, and empty string """ chat_history = chat_history or [] # Initialize thread if needed if not self.current_thread_id: self.current_thread_id = str(time.time()) messages = [] image_path = self.original_file_path or display_image if image_path is not None: # Send path for tools messages.append({"role": "user", "content": f"image_path: {image_path}"}) # Load and encode image for multimodal with open(image_path, "rb") as img_file: img_base64 = base64.b64encode(img_file.read()).decode("utf-8") messages.append( { "role": "user", "content": [ { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_base64}"}, } ], } ) if message is not None: messages.append({"role": "user", "content": [{"type": "text", "text": message}]}) try: for event in self.agent.workflow.stream( {"messages": messages}, {"configurable": {"thread_id": self.current_thread_id}} ): if isinstance(event, dict): if "process" in event: content = event["process"]["messages"][-1].content if content: if isinstance(content, list): text_parts = [] for part in content: if isinstance(part, dict) and "text" in part: text_parts.append(part["text"]) elif isinstance(part, str): text_parts.append(part) content = " ".join(text_parts) elif not isinstance(content, str): content = str(content) content = re.sub(r"temp/[^\s]*", "", content) chat_history.append(ChatMessage(role="assistant", content=content)) yield chat_history, self.display_file_path, "" elif "execute" in event: for message in event["execute"]["messages"]: tool_name = message.name # Robustly parse the tool result tool_result = None try: if isinstance(message.content, str): parsed = eval(message.content) tool_result = parsed[0] if isinstance(parsed, (list, tuple)) and len(parsed) > 0 else parsed else: tool_result = message.content[0] if isinstance(message.content, (list, tuple)) and len(message.content) > 0 else message.content except Exception: tool_result = str(message.content) if tool_result: metadata = {"title": f"🖼️ Imagen de herramienta: {tool_name}"} formatted_result = " ".join( line.strip() for line in str(tool_result).splitlines() ).strip() metadata["description"] = formatted_result chat_history.append( ChatMessage( role="assistant", content=formatted_result, metadata=metadata, ) ) # For image_visualizer, use display path if available if tool_name == "image_visualizer": if isinstance(tool_result, dict) and "image_path" in tool_result: self.display_file_path = tool_result["image_path"] chat_history.append( ChatMessage( role="assistant", # content=gr.Image(value=self.display_file_path), content={"path": self.display_file_path}, ) ) yield chat_history, self.display_file_path, "" except Exception as e: chat_history.append( ChatMessage( role="assistant", content=f"❌ Error: {str(e)}", metadata={"title": "Error"} ) ) yield chat_history, self.display_file_path, "" def create_demo(agent, tools_dict): """ Create a Gradio demo interface for the medical AI agent. Args: agent: The medical AI agent to handle requests tools_dict (dict): Dictionary of available tools for image processing Returns: gr.Blocks: Gradio Blocks interface """ interface = ChatInterface(agent, tools_dict) with gr.Blocks(theme=gr.themes.Soft()) as demo: with gr.Column(): gr.Markdown( """ # 🏥 [RAI-X] Agente de razonamiento médico para radiografías de tórax """ ) with gr.Row(): with gr.Column(scale=3): chatbot = gr.Chatbot( [], height=800, container=True, show_label=True, elem_classes="chat-box", label="Agente", avatar_images=( None, "assets/medrax_logo.jpg", ), ) with gr.Row(): with gr.Column(scale=3): txt = gr.Textbox( show_label=False, placeholder="Pregunte por la radiografía...", container=False, ) with gr.Column(scale=3): image_display = gr.Image( label="Imagen", type="filepath", height=700, container=True ) with gr.Row(): upload_button = gr.UploadButton( "📎 Subir Radiografía", ) dicom_upload = gr.UploadButton( "📄 Subir DICOM", file_types=["file"], ) with gr.Row(): clear_btn = gr.Button("Chat claro") new_thread_btn = gr.Button("Nuevo hilo") # Event handlers def clear_chat(): interface.original_file_path = None interface.display_file_path = None return [], None def new_thread(): interface.current_thread_id = str(time.time()) return [], interface.display_file_path def handle_file_upload(file): if isinstance(file, str): file_path = file elif isinstance(file, dict) and "path" in file: file_path = file["path"] elif hasattr(file, "name"): file_path = file.name else: file_path = str(file) return interface.handle_upload(file_path) chat_msg = txt.submit( interface.add_message, inputs=[txt, image_display, chatbot], outputs=[chatbot, txt] ) bot_msg = chat_msg.then( interface.process_message, inputs=[txt, image_display, chatbot], outputs=[chatbot, image_display, txt], api_name="chat" ) bot_msg.then(lambda: gr.Textbox(interactive=True), None, [txt]) upload_button.upload(handle_file_upload, inputs=upload_button, outputs=image_display, api_name="upload") dicom_upload.upload(handle_file_upload, inputs=dicom_upload, outputs=image_display) clear_btn.click(clear_chat, outputs=[chatbot, image_display]) new_thread_btn.click(new_thread, outputs=[chatbot, image_display]) return demo