add code for mediquery-assist

.gitignore

@@ -0,0 +1,6 @@
+.venv
+.gradio
+.env
+data/
+__pycache__
+.DS_Store

audio_handler.py

@@ -0,0 +1,20 @@
+from transformers import pipeline
+
+
+class AudioHandler:
+    def __init__(self):
+        self.transcriber = pipeline("automatic-speech-recognition", model="openai/whisper-small")
+    
+    def transcribe_audio(self, audio, current_text, file_input, message_history, chat_func):
+        if audio is None:
+            return message_history, current_text, None, file_input
+        
+        transcript = self.transcriber(audio)["text"].strip()
+        
+        updated_history, cleared_text, cleared_file = chat_func(
+            transcript, 
+            file_input, 
+            message_history
+        )
+        
+        return updated_history, current_text, None, cleared_file

chat_handler.py

@@ -0,0 +1,67 @@
+import json
+import uuid
+from langgraph.errors import GraphRecursionError
+from prompts import REACT_SYSTEM_PROMPT
+
+
+class ChatHandler:
+    def __init__(self, graph, rag_setup):
+        self.graph = graph
+        self.rag = rag_setup
+        self.session_id = str(uuid.uuid4())
+        print(self.session_id)
+    
+    def chat(self, user_message, uploaded_file, message_history):
+        user_query_parts = []
+        try: 
+            if user_message and user_message.strip():
+                user_query_parts.append(user_message)
+            
+            if uploaded_file is not None:
+                result = self.rag.store_data(uploaded_file)
+                result_str = json.dumps(result, indent=2)
+                user_query_parts.append(f"""A medical document was uploaded. Here are the upload details: {result_str} Please inform the user about the upload status in a friendly, professional way.""")
+
+            if not user_query_parts:
+                return message_history, "", None, None
+            
+            user_query = (' ').join(user_query_parts)
+            
+            config = {"configurable": {"thread_id": self.session_id}, "recursion_limit" : 25}
+            current_state = self.graph.get_state(config)
+            
+            if not current_state.values.get("messages"):
+                messages = {
+                    "messages": [
+                        {"role": "system", "content": REACT_SYSTEM_PROMPT},
+                        {"role": "user", "content": user_query}
+                    ]
+                }
+            else:
+                messages = {"messages": [{"role": "user", "content": user_query}]}
+
+            result = self.graph.invoke(
+                messages,
+                config=config
+            )
+            
+            last_message = result["messages"][-1].content
+            
+            updated_history = message_history + [
+                {"role": "user", "content": user_message},
+                {"role": "assistant", "content": last_message}
+            ]
+            
+            return updated_history, "", None
+            
+        except GraphRecursionError:
+            error_message = "This query is too complex and exceeded the reasoning limit. Please simplify or break it into smaller questions."
+            return message_history + [
+                {"role": "assistant", "content": error_message}
+            ], "", None
+           
+        except Exception as e:
+            error_message = f"Error: {str(e)}"
+            return message_history + [
+                {"role": "assistant", "content": error_message}
+            ], "", None

experimentation.ipynb

@@ -0,0 +1,601 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "f9c151c2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langgraph.graph import START, END, StateGraph\n",
+    "import sqlite3\n",
+    "from langgraph.checkpoint.sqlite import SqliteSaver\n",
+    "from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint\n",
+    "from langchain_huggingface.embeddings import HuggingFaceEmbeddings\n",
+    "from langchain.tools import tool\n",
+    "from langchain_community.utilities import GoogleSerperAPIWrapper\n",
+    "from langchain_community.document_loaders import PyPDFLoader\n",
+    "from typing_extensions import TypedDict, Annotated\n",
+    "from langgraph.graph.message import add_messages\n",
+    "from langgraph.prebuilt import ToolNode, tools_condition\n",
+    "from dotenv import load_dotenv\n",
+    "from IPython.display import display, Image\n",
+    "import gradio as gr\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
+    "from langchain_chroma import Chroma\n",
+    "import uuid\n",
+    "from langgraph.errors import GraphRecursionError"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "919b6be4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "load_dotenv(override=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "21a63f2c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class RAG_Setup:\n",
+    "    def __init__(self):\n",
+    "        self.embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-mpnet-base-v2\")\n",
+    "        self.vector_store = Chroma(\n",
+    "            collection_name=\"medical_history_collection\",\n",
+    "            embedding_function=self.embeddings,\n",
+    "            persist_directory=\"data/patient_record_db\", \n",
+    "        )\n",
+    "\n",
+    "    def _calculate_file_hash(self, file_path):\n",
+    "        import hashlib\n",
+    "        sha256 = hashlib.sha256()\n",
+    "        with open(file_path, 'rb') as f:\n",
+    "            while chunk := f.read(8192):\n",
+    "                sha256.update(chunk)\n",
+    "        return sha256.hexdigest()\n",
+    "\n",
+    "    def _is_file_uploaded(self, file_hash):\n",
+    "        results = self.vector_store.get(\n",
+    "            where={\"file_hash\": file_hash},\n",
+    "            limit=1\n",
+    "        )\n",
+    "        return len(results['ids']) > 0\n",
+    "    \n",
+    "    def _extract_content(self, file_path):\n",
+    "        pdf_loader = PyPDFLoader(file_path)\n",
+    "        content = pdf_loader.load()\n",
+    "        return content\n",
+    "\n",
+    "    def _split_content(self, content):\n",
+    "        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, add_start_index=True)\n",
+    "        chunks = text_splitter.split_documents(content)\n",
+    "        return chunks\n",
+    "\n",
+    "    def _embed_content(self, chunks):\n",
+    "        self.vector_store.add_documents(chunks)\n",
+    "\n",
+    "    def store_data(self, file_path):\n",
+    "\n",
+    "        file_hash = self._calculate_file_hash(file_path)\n",
+    "        \n",
+    "        if self._is_file_uploaded(file_hash):\n",
+    "            return {\n",
+    "                \"status\": \"skipped\",\n",
+    "                \"message\": f\"File already exists in database\"\n",
+    "            }\n",
+    "        \n",
+    "        try:\n",
+    "            content = self._extract_content(file_path)\n",
+    "            chunks = self._split_content(content)\n",
+    "            \n",
+    "            for chunk in chunks:\n",
+    "                chunk.metadata.update({\n",
+    "                    'file_hash': file_hash\n",
+    "                })\n",
+    "            \n",
+    "            self._embed_content(chunks)\n",
+    "            \n",
+    "            return {\n",
+    "                \"status\": \"success\",\n",
+    "                \"message\": f\"File successfully uploaded\",\n",
+    "                \"chunks\": len(chunks)\n",
+    "            }\n",
+    "        except Exception as e:\n",
+    "            return {\n",
+    "                \"status\": \"error\",\n",
+    "                \"message\": f\"Failed to upload file: {str(e)}\"\n",
+    "            }\n",
+    "\n",
+    "    def retrieve_info(self, query: str):\n",
+    "        try:\n",
+    "            results = self.vector_store.similarity_search(query, k=5)\n",
+    "            print(\"printing tool results\", results)\n",
+    "            \n",
+    "            if not results:\n",
+    "                return \"No medical history found for this query.\"\n",
+    "            \n",
+    "            content = \"\\n\\n---DOCUMENT---\\n\\n\".join([doc.page_content for doc in results])\n",
+    "            \n",
+    "            return content\n",
+    "        \n",
+    "        except Exception as e:\n",
+    "            return \"Failed to retrieve medical record\"\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "796b25c1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rag = RAG_Setup()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "795f7bce",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "@tool\n",
+    "def check_medical_history(query: str):\n",
+    "    '''Retrieves relevent medical history of the user\n",
+    "\n",
+    "    Args:\n",
+    "        query: medical history to be searched for\n",
+    "    '''\n",
+    "    return rag.retrieve_info(query)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "a9efe8fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "serper = GoogleSerperAPIWrapper()\n",
+    "@tool\n",
+    "def web_search(query: str):\n",
+    "    ''' Search web for answering queries with latest information\n",
+    "    Args:\n",
+    "        query: query to be searched on the web\n",
+    "    '''\n",
+    "    print(\"Websearch tool calling\")\n",
+    "    return serper.run(query)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "9bc930eb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tools = [web_search, check_medical_history]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "96d52fc2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = HuggingFaceEndpoint(\n",
+    "    repo_id=\"deepseek-ai/DeepSeek-V3\",\n",
+    "    task=\"text-generation\",\n",
+    "    max_new_tokens=1024,\n",
+    "    do_sample=False,\n",
+    "    repetition_penalty=1.03,\n",
+    "    provider=\"auto\", \n",
+    ")\n",
+    "llm = ChatHuggingFace(llm=llm)\n",
+    "llm_with_tools = llm.bind_tools(tools)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "4bff6b8c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class State(TypedDict):\n",
+    "    messages: Annotated[list, add_messages]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "c4ea1e72",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "REACT_SYSTEM_PROMPT = '''You are a helpful medical assistant with access to patient records and web search.\n",
+    "\n",
+    "You solve problems using the ReAct (Reasoning and Acting) framework:\n",
+    "1. Thought: Reason about what information you need\n",
+    "2. Action: Call the appropriate tool\n",
+    "3. Observation: Receive the tool result\n",
+    "4. Repeat until you can answer confidently\n",
+    "\n",
+    "AVAILABLE TOOLS:\n",
+    "- check_medical_history: Search patient's personal medical records (medications, appointments, conditions, lab results)\n",
+    "- web_search: Search the web for general medical information, drug interactions, side effects, treatment guidelines\n",
+    "\n",
+    "MULTI-STEP REASONING EXAMPLES:\n",
+    "\n",
+    "Example 1: Drug Interaction Query\n",
+    "User: \"Can I take ibuprofen with my medicines?\"\n",
+    "Thought: I need to first check what medications the patient is currently taking.\n",
+    "Action: check_medical_history(query=\"current medications\")\n",
+    "Observation: Patient takes Metformin, Lisinopril, Atorvastatin, Levothyroxine, Omeprazole, Aspirin, Vitamin D3\n",
+    "Thought: Now I need to check if ibuprofen interacts with these specific medications, especially Aspirin (both are NSAIDs).\n",
+    "Action: web_search(query=\"ibuprofen interactions with aspirin metformin lisinopril atorvastatin\")\n",
+    "Observation: Ibuprofen + Aspirin can reduce aspirin's cardioprotective effect. Risk of bleeding increases. Should avoid concurrent use.\n",
+    "Answer: Based on your current medications, taking ibuprofen with aspirin is not recommended...\n",
+    "\n",
+    "Example 2: Simple Patient Query\n",
+    "User: \"What medications am I taking?\"\n",
+    "Thought: This is a straightforward question about the patient's records.\n",
+    "Action: check_medical_history(query=\"current medications\")\n",
+    "Observation: [Patient medication list]\n",
+    "Answer: You are currently taking...\n",
+    "\n",
+    "Example 3: General Medical Question\n",
+    "User: \"What are the side effects of Metformin?\"\n",
+    "Thought: This is a general medical question, not specific to the patient's records.\n",
+    "Action: web_search(query=\"Metformin side effects\")\n",
+    "Observation: [Web search results]\n",
+    "Answer: Common side effects of Metformin include...\n",
+    "\n",
+    "CRITICAL RULES:\n",
+    "- Use multiple tools when needed - don't stop after one tool if more information is required\n",
+    "- Think step-by-step and be thorough\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "57c7bd31",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def personal_assistant(state: State):\n",
+    "    print(\"assistant responses:\")\n",
+    "    print(state[\"messages\"])\n",
+    "    messages = state[\"messages\"]\n",
+    "    return {\n",
+    "        \"messages\":  llm_with_tools.invoke(messages)\n",
+    "    }"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "c27a0d7c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db_path = 'data/long_term_memory.db'\n",
+    "conn = sqlite3.connect(db_path, check_same_thread=False)\n",
+    "memory = SqliteSaver(conn)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "6a6029e6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANgAAAD5CAIAAADKsmwpAAAQAElEQVR4nOydCVhUVRvHz72zMOz7vgiIguJChpp+Za5pn3tZ7jtpWu58aqlJaOaulVumZlZKprhmmrmVGq6hoIIiIIssguzbDHPv985cGAaYGUC5M3dmzk8fnnvPOXf/zznnfc/Gp2kaYTC6ho8wGA6AhYjhBFiIGE6AhYjhBFiIGE6AhYjhBFiIdclOEd+LKsh/Ji4voaRSSiquiaIRTZAEohBBIpqSBxE0QRBV20geTsvS0NLaJyVlhyK61qkgOVH/8qT8hLUPZy5H8JAivOYG5PBFhFBIiix5rj6mwX1skB5CYD8iQ3p8xcXI7LycCpqiBUKeyIzkC0mShyoraj44AXoCpVC0QgcEj5BJr0aIBCLkKqNqv1VIJq0dUqXDui+f5BEQRNdOLFM2RTN/q0NqCVEg4lEULSmjKsooSSUtFJFuPqJBIa5If8BCRNmplcd3ponLpNb2wg6vW7V/wxrpNVJ04XBOYmxxeYnUpYXo3dnuSB8wdiH+uin9WXqZZ4DF4BAXZFjkZkhO7k4vK5L2es/VP9gMcRujFuJ3S5KEAnJiWAtkuNyPKvkrMsszwHzgFGfEYYxXiLuXJbn5mr09mdOfp7nYvSw5uJ9txx7crXUYqRB3LHrcsoNVv7GOyGjYtSTJ0dN06IccrYGQyPjYszzZK8DcqFQIhHzhk51aevlIDuIkRifE499mgNvkv5MNzTRpDB+E+0b/nY84iZEJUYpSH5ZMDvNGxgkPebU23xOWjLiHcQlx3+oURw9TZMQM+dC1vFj68FYx4hjGJcTCXPH7c/TDwcse7n5mV0/mIo5hREI8viPD1Iyv5SdevHjxsWPHUNPp169feno6YoGBU12LCySIYxiRELNSylsEaruB4f79+6jpZGRk5OXlIXbgC5BQxDt34BniEkYkRLGYerWPPWKHK1euTJ8+/fXXXx82bNjy5ctzcmRekuDg4KdPn65YsaJnz56wW1xcvGPHjokTJzLJNm3aVF5ezhzep0+fAwcOfPDBB3DIpUuXBg8eDIFDhw5dsGABYgFbJ+HTpFLEJYxFiI/vlpIEfAAeYoG4uLg5c+Z07tz50KFDCxcufPjwYVhYGJKrE/4uW7bs4sWLsBEREbF3797x48dv3rwZ0p89e3bnzp3MGQQCwZEjR/z9/bdu3fqf//wHEkAglOkbNmxALODsJaoooRCXMJb+iBlJZTwBgdghOjpaJBJNmTKFJEkXF5e2bdsmJCTUTzZu3DjI+Xx8fJjdO3fuXL16dfbs2UjWI4ywtrYODQ1FWsHFy+T+NSxEXVBWLCVJtoQYFBQEhezcuXO7du3ao0cPT09PKGHrJ4Ns759//oGCG7LMyspKCLGzs1PEgnyRtrB1FFKV3GraNZaimaJoGrH16gMCAr7++mtHR8dvvvlm+PDhM2fOhNyufjKIhbIYEhw9evTmzZuTJ09WjhUKhUhr8Hkqu4frEGMRopkFX9ZZnzW6d+8OdcETJ05A7bCgoAByRybPU0DT9OHDh0eOHAlChOIbQoqKipCOyM8uQxzDWITo5CmSStnKEW/dugW1PdiATHHQoEFg6oLIwAWjnEYikZSVlTk5OTG7YrH4r7/+QjoiO1XME3Dr0xuLEP2DzSvFlLiUFS1CQQzGcmRkJDj/YmNjwToGRbq6upqYmIDyoqKioCAGO8bb2/v48eNpaWn5+fnh4eFQsywsLCwpKal/QkgJf8GshrMhFkhPLOULcdGsIwRC8p8zzxELgDkMBe769euhOWTatGnm5uZQF+TzZYYgmNI3btyAPBKyw1WrVoFxPWLECHAidunS5eOPP4bdvn37gq+xzgk9PDzAlQhOR6hWIhbIzxK7eXGrzd2IOsZGrEstKaqcGu6DjJ5v5j0KWdHS1IJD2ZAR5Yj9J7iUFkmR0fP73kwTUx6nVIiMaoC9rbNAKCKPbE0f/pHqDjhSqRQcziqjwLYALyChyuXh6+u7Z88exA575aiMsrCwgDZDlVGBgYHQQoPUkHy/pFMvO8QxjGvMSnpCeeTWtFmb/NQlqF9dY4BPDh9eZRTUBRW2cLNTJEdlFLjQoYqpMgp+M2AtqYw682N2UmzRh2taIo5hdIOnDqxNBT/OuE+8kFGydUHC8Blebn5adJ43DqMbszJ6oWdxviTqFCvmM8fZ+3myl785B1WIjHMUHxRMty/kFWUZV1Gwf20aOLEHT+PohDjGO8B+W+jjfiNdW3Xm+lwczcIPK1Lt3fiDpnJ3WiajnnJk28JENy/RsI/dkEGze1mSmaVg9EIPxGGMfRKmPWHJknKq6wD7oJ56PgmYKiK3Ps1ILG0VZPXWeLbs+uYCT0uHrhx/fvdyHiKQV2uzAeNdeVysyjeNxLul18/kPs8Sm1nxJy1pgVjplt7MYCFWcfHQs4e3iirKpSSPMLXgWVgLLK0FBI+SiGveT830mCQiSZKqpOpEySbqlPc2o6pjSAhHRNXcnnTVxJyKQNn0s3TNGeRzd8r2mcOZZLVOKJ/ak8cjpVKKScCE8wWEtJIoK6osKZKWFcu6n1nZCd58x9Gjtd4M4sZCrMvlY7lpj0rLi6XSSiSlaKlST2aZTJg9ECJoSKm9kImSyUje+qJ4qVV7svmIqwIpigKtyxSlFFiVkq51uFyVtHwOWqT8lXgkklK17ocvlKnTxJS0shO2fsXSv7M50jewELXNrFmzxowZ061bN4RRAk/mrm0qKyuZHmIYZfAb0TZYiCrBb0TbYCGqBL8RbSORSAQCAcLUBgtR2+AcUSX4jWgbLESV4DeibbAQVYLfiLYBIeI6Yn2wELUNzhFVgt+ItsFCVAl+I9oGC1El+I1oGyxEleA3om3AoY2FWB/8RrSKbJFxiuLx9KGrqnbBQtQquFxWB34pWgULUR34pWgV3ONBHViIWgXniOrAL0WrYCGqA78UrYKFqA78UrQKFqI68EvRKthYUQcWolbBOaI68EvRNurmcjVysBC1CjTuZWZmIkw9sBC1CpTLdZZGwzBgIWoVLER1YCFqFSxEdWAhahUsRHVgIWoVLER1YCFqFSxEdWAhahUsRHVgIWoVLER1YCFqFRCiVIpXSFWBMa48pVugcQVrsT5YiNoGl84qwULUNliIKsF1RG2DhagSLERtg4WoEixEbYOFqBIsRG2DhagSvPKUlggKCiLJKtMQ3jlsw99BgwaFh4cjDLaatUaHDh2QbBlHGeBKJAjC1dV13LhxCCMHC1FLTJgwwdy81lqNHTt2bN26NcLIwULUEn379lWWnb29/ejRoxGmGixE7TFp0iQrKytmOyAgoH379ghTDRai9njjjTf8/f1hw9raeuzYsQijBFes5tvnCnKellWUU2pTEFXLv6tDsZy7mliCouoeL1u4G14AhZQX8H6Bq2s+XDk2vyA/JuaOpYU1GNENnrahJ1IbWycKHlPxlRt/VGOiCBLRag4xtxQEBFu5+ZmgxqF7If57Pv/aH89JguDxkbhcw/dsQIgaXor6WJpZJr7BkzeQoNGxNEHLVUkw69I3oG+NT6Th2LoHyi5KNHjOF4lSfw9CEU9cUSky400O80aNQMdCvH+t6K/InB7vunn6N/ang9EjLh189jSpeNoqnwZT6lKIyXfLzvycOebThu8So7/cOJ2XGFsQssJbczJdGiuXjuU4e5shjEHTeYAtLaWvn83XnEyXQiwvFrfsYIkwho6pJf/JvRLNaXTZ6QGa/vlCAmEMHfBXVJQ0MDpCl0KkKZqi8OgNw0daSdENFb24GxiGE2AhYlgHvKaooSoYbuLDsI7MQ9iQkxDniBjWIUiiQZtUlzkiIWvrxVmy4QNWKd1QlqhTq1nWqkMhjKEDTdUEtpoxOoemGqwiYiFiuAEWIoZ1qju9aQILEcM6YDU3qERdC5HAbc2GDyWlyYaMFV17Twx3eP/hyIi+b3VF2iIxMaFXn+C7d/9F+gl24xkINja2E8aHODm5aEiTlPR41JhB6OUY/m6/pxnpqLnBdUQDwc7OfvKkDzWniX94H70cmZkZ+fl56EXgsEO7qRz89af9B/aGzl+6cfMqeB1ubh4TxoW89dZAJvbevbs/7NsZF3fP2sa222tvTJwwjZlZAYrI/Qe+nzf3k+VhC4cNe3/WR6FR16788su+uPh7dnYO7dp1nBYyy97eAVKWlpbCmaOjbxYVFXq38H377aHDhr6H5BnJlJCR27b+sH//95evXHR0dOrV861pH8zi8XgQG3nkl6iovx88iBWamHTs0Gnq1I/c3Twa/1D//PP3+Qtn7sb8W1hY0Cag3fjxIa8EBTNR6u5TZTgUzVM/GPXVpu86dHilqLjo+707rkVdzst/7t+6bd++bw/87zAI2ffjLjgcSvCZM+a9N2Ksukure15IOX+BTOtjxw2dO2fx0CEjGvmMsiY+jtcRG2HX18Dj8UtKis+dP/3zj8eOHjnXp3f/1WvDUlOfQFRaemrowpnlFeVbvvl+xefrExMfzZs/jZl0SygUlpaWHD9+6JPF4cOHvv/wUdwnn8555ZXOe/ccmj1r4ePHD9esDWPOv/jT2U+fpq0I33Aw4lSPHn2++nrNg7h7EM4s9b1h48o+fQb8cfqfJZ+shJ/EhYtnITAmJvqbLesCAzuGh69fvOjzvLznX6xa2ugHQuXl5V98ubSiogKOXfXFZi8v7yVL5z1/ngtR6u5Tw/0rWLv28/v37s6d+wmkadOm3abNX8KvFPLLUSMnODu7XDh3E1So4dLqnhdk+uUXmyHq55+ONV6Fcmiud3qgm2g0g7beGT7KFECmkyZOj4yMOHf+zKSJ0/7883cBXwAStLa2gWShC5aNHjsYfs093+wLDdrw0keNmtjplc4QBYeIRKJxY6eQJAlfJcC/bWJSApJnM6CqPbt+8fFpCbtjx0y+dv0KZLGrV33FXPrNHn3hbEg2Z00nN1f3hw8f9O0zoG3b9t/vPujh4cUsB14pkXy6dF5BYYG1lXVjHgfuZNfOCHga5rYhWzp2/FBMbPSbPfrExkSrvE914crcuXsbNNc5+DXYhpzszTf7WlvZNP7SGp4XvRCylhWuN/E13Wpu3boNswEKg9I5JSUJycrlOwEBgcw7BVxcXCEKihLmVQIB/oHMRrv2QaDLT5bMDX61a7duPTzcPavLowT4NowKqy7Uqg3kvvWvC1hYWBYXFyH5EgGQiW7dtuFBXGxJSdWwjPy8540UIpLVB0p27d4SfedWbm5O1eHySpi6+1QXrkz79kGQhxUU5ENVoXPnbv5Kd96YS2t4XvbQP6vZxKRmBLSJSASFNWzAa7pxMwpqP4r/oI88eUHDAAU0s9G6VcDqL792sHfc+d034ycMD/3fzNjYOxAOH0MkMlW+kJmZWVlZqWKXVOUKu3Ll0pJl8/39227e+N35P2+sXbMFNYWsrMw580IkEsmyJaugEDx7JkoRpe4+1YUrs2hh2Ih3x9y4+Q/c2zvv9tvz/fb6U4Nqd961/gAAEABJREFUuLSG531xDK8/ImQ8ivndKsrLbW3sYMPO3gGygTpmY/3yiKFrl+7wHxLfunXtcOSBT5fMjTx8Fs5ZXl5W60KlJfC9kUZOnjoC1w2Z+hGz29Rs4+Kls2KxGGppsrpG7QxJ3X1CHUBluPKBVpZWUHZD7QI0+vflCz/+tBuytPffG9f4SzcvsiY+rju0m/6z+zf6BrMBFe2U1GSmMG3p2yo7OxNKIiinmP8gUKiA1z88OvrWtetXYcPBwbF//0EfzVwANmZmVgZYl1DkPUqIV6QEQ9hbqaRWCdibjg5Oit2//z6PmgIcbmlpxUgBuPTXuQbvU1244kCooYIhD88CVRf4kYCBDG8DTJzGX7rZ0YehAlTT+iNCeQHWRkpKslQqhRIHtNint6wGPWLEWIqitmzbAB8A7Ohvd34NDoj6tXgg9t6dsM8XnjgZCXnA/QexkUci4Iu6OLt26dIdqpUbN34RF38fjMfde7aBEEe+N17z/fi1bA1Vgn+jb0LZ9+uhn5lAZVloxte3FVQJjp84DIeDvG7fvg7VXPhFabhPdeGKc/J5fLCxwsIXQXYID/LHH789Sohr30425xMYVXC5y5cvwivScGkNeMp/2xcvns3IfIoaDXxkuqHvrGdFM/zKoYiZH/ohvET4NS9eGObp2QLJC6Pdu36JiPhh+oxxIFMwXP4XugyqU/XPAIfDJ9yydf3GTaug4ti7V/9NG3cyNu/K8A07vt0886OJEA7faUX4eshRNN/PlCkzocq/dNn8srIyMOehpMvISF/8yewln65EjQA8UE+eJO778TvwsICRC3W7iF/2ga8UHJkffxSq8j413D8D1DHCw9Z9s3XdrDlTYRdKjA+nz317wBDYfq3r66DIZctDwckKrgZ1l65TiCsDLtIB/QeDSxIuDdVQ1Hzocu6bLfMSeo1y8QqwaGR6cE1v277x3NnrCKNXHNqczCPRhGXeGtLovPcNwhg8ZCNaVnQsRII2cCWCkxysWnWxP/14VOH7NGy4PlSg4aYfJd59ZxT8R3oF1DJ37tyvLtZIVEg1POMI7n3DPq4ubgjTEFiIGPYhGu6Jr+s6IjZWjAG64T4FOnZo44UAjQGwmkmu977BGAH1FxapDxYihhPoVIi4goipRqdCxBVETDW4aMZwAixEDCfQpRB5fILkCxHG0BGJ+IjXQD1Ml35EvoCXm16GMIZORYXUyqaBHEeXQnRwFybGFCCMoVNWJB0wyllzGl0KcfhHbuIS6aWIZwhjuESsS/JoacZrqPez7tdr3rfiCfhxPPwt7F1NJfVGPRLyxZTr3yJBqO5DprQ0cXWI/C+zwII6x6V8ae8X9Goqzl83UOlOlJc1rgqrjlU+nEC11o5W3q2zMDJdfQlU+xJMQK3EdNV8GrT6+6fr7da8kOpLqLi36h3FXdR6TJpMe1ycmVzW9S27oJ4Nj/LmxAr2J7/LzEwpqxTTEnH9MTa0fPH5uqGyHr+UaoHSjXjl9XQMFyAJVPUFlI9AtdfMVic7uvYNMJ05mI9ZX0ZIzX3WVp7sLMypFClrdCD7bkStKzJHybZp5vdIyg+vf9uKh6r781D6OdJVJ6s6Svl11bk9pLSsuPKtCk1IEzPy1Z727Xs0at1PTghRt2zatAn+zps3D2mFOXPmjBw5snv37ogFDh48CI8jEAjMzc0dHR29vb2DgoLayEHcxqiFGBMT0759+3v37gUGBiJtsWLFiiFDhnTs2BGxA6j80aNHJElS8nKEIAhra2tLS8tjx44hDmOkE3XCz2/mzJmZmbJhvNpUIbBs2TL2VAgMHDhQJBIh+RhwUjZsiSgsLExNTUXcxhhzxNzcXPg8CQkJXbp0QVoH1G9ra6s8g0/zUlZWNn78+OTkZEWImZnZX3/9hbiNceWIFRUV06dPh09lZ2enExUCixYtgt8AYg1TU9N+/foR1X3foYBeubJRo/11i3EJ8bfffps2bZqHRxNmdG12nJ2dIYtCbPLOO++4uMgm0wYV3r59++jRo9u3b0fcxiiEWFBQEBoaiuRf6NVXX0U6Ze3atT4+PohNwF7u2bMnbLi5yQYQbty4USgUzpo1C3EYoxBieHj41KlTETdIT0+vP1ths7NgwQKoiZ48eZLZhccfM2ZM796909LSECcxZGMFzIKLFy+OGsWtMfngu9mxYweTV2kZMJ8nTJgwY8aM/v37I45hsDliaWlpSEhIjx49EMeA2ptiVkItY2VlBfVFsKAZHz6nMMAcMSMjo6ioyN3dXTGxLKYO+/fvP3/+/K5duxBnMLQc8cGDB4xdzFkVpqSkUJSOl0uH+iLYLt26dXv48CHiBoYjxKdPZXOYgqfwxIkTbPtHXoZx48aVl5cjXQOtO1BGh4WFQWGNOICBCBHEt3z5ctiANn7EbcBMUSxxoFsEAgGU0bGxsV988QXSNXpfR8zPz7exsYmMjAQfIcK8EEeOHDl06NC+ffuYRd10gn4L8bvvvoN3N2XKFKQ/PHnypEWLFohjxMfHT5w48dtvv2W1Q4YG9LVohrpgbm4u1Pr1S4VQOxw7diziHv7+/lFRUV9//fWBAweQLtBLIe7cuRNsTyiRp0+fjvQKKH98fX0RV9m9ezfYfEuXNmFdy+ZC/4R46tQp+NuqVSsdVmheGHBlQ1UMcRhoG3z99dehwg2+WKRF9KmOCJ8QWqgKCgqsrRu75CLXkEql4G/XbfefxgAFDlQZV69e3bVrV6QV9CZHXLRoEdPxWH9VCDx79uzDDz9EnMfLy+vChQvwy9+zZw/SCnogxCtXrsDf+fPnv//++0jPIQiCgyazOrZu3QpGIRTWiH04LcTKysohQ4YwveqdnZ2R/gNPAV8X6Q8zZsyATzBgwIDs7GzEJtytI2ZmZkILBPg7dNJjiiXEYnFOTo7ePRHcM9TO16xZ0759e8QOHM0RoekpJibGzs7OkFSI5COboClS7xoRHBwcwFkBXsasrCzEDhwVImSHYB0jgwMsrW3btkHLuM474LwA0dHR7FWQ8EwPuiE1NZUkSXd3d6QnPHr06LPPPmOv3YWjOaJUDjJcPD09Z86cWVJSgvQEECI0IiDW4KgQofz6+eefkUFz7Nix+Pj44uJipA88fvzYz88PsQZHhcjeRAicolOnTunp6VevXkWcB3JEVoXI0cncp02bhowDf3//2bNnd+jQwcKiobksdUpCQoIx5ogGX0dUBtwihYWFnB1xjOQzFEATi5OTE2INjgoRWjl37NiBjAZwl+bl5emqL2CDsJ0dIi7XEQkjW0IXGi2ePn0KHm/EPbQgROxH5BalpaVxcXFgxCAusXLlynbt2g0bNgyxBq4jcgszMzORSLRq1SrEJSBHZNWJiDgrxCNHjqxbtw4ZJW3btg0ICEBcwnjriEKh0NjqiMowQ2OPHz+OOAC0Rjo6OrLt2eWoEIcMGbJo0SJk3ID5wkzrqFvYbtxj4KgQKYrSwiSCHMfHx2fSpElI12ihXEacFeLZs2eZKUSMHLBVUfVKMLrCqIUoEAhI0kiX3qgP5Is6HHKlnaIZ+xH1g6KiIktLS6iu8Pmy7gEDBgyA3+qJEycQy0DLXu/evZnxa6yC64j6AagQyUe/l5SUDBo0KCcnB5oEz5w5g1hGCx5EBo4KMSoqSjujGPWLr7766u2332YWzILGwHPnziGWYbv3lwLu1hGN2Y+ojpEjR0IbILMN7yc+Pp4RJXtox1JBnBVi586dN2/ejDBKjBkz5vHjx8ohWVlZly5dQmyiHUsFcVaIYEJJJBKEUQLqzR4eHspTT4nFYvBzITZhe4SAAo720I6JiYEcUWsTr+gFERERt2/fvnHjxrVr14qLizMyMpzNO9GFdn8ceeju6lKzxDhRvaG0on3V+vZ01RLgddcMZ1KSSgury9cCB1Pd2+HN1PtEKiqUnZOJq7OQPaq9KLpsyfKaOhVJEk4eJg7uDU/VzC33TUhICLxiuCX4C1ahk5MTZANQK/rzzz8RRonvP08sLZSCXKQy10KVuhgRENUyogn5P2a1eUTUXoieVoiIYBair0ks1yQJGXDVkvaKkxNyhdFKK9XL4xTL3jMhsF+zyxfAPiEQEh3+Y9v1vzYanohbOWLbtm1/+uknhSub6T0PLe4Io8TOxYmOLUxHzHRFnJgTvmHuXS2IuZrn6m3i1VbtSkfcqiOOGzeu/tyBulrPlpvs/DSxTWf7vmP0RoVAYHfrkaHep37IuPmH2tk7uCVEKIsHDhyoHGJvb8/NSad1wu8/ZPMFvKC+ejlDZJuuNtGXctXFcs5qHj16tHKmGBQU1Lp1a4SRk5VS7uAqQvpJpz52EgktVjOfAOeEaGVlNXjwYKZF1c7Obvz48QhTjaSiki/S474gYADlZKkeHcbFp1Jkiu3kIEw1lWK6UqzH7lVKSlNqehC8lNUsKUNXfnv2LKWiqKBSIqbAeIcrKWIZJ4IyjNmvHE6QBE0p+Y9krixZop4tvpR6SPk8/vaFiXLHWO1kqMaVpXAlVAWi2m4t5iH5cCGSz0PmdgIPP9NuA+0QhmO8oBBP78tKiSuRlFPweaH6TAp4JhYCme9KSQVyB1UtUSg8rgp3qNwPpZyAQDRdx9Wq5P1SgbIQZVdUlZTP54GMpRWVzzMl2SllN/98LjLnBQRbvTHMHmG4QZOFeHpv1uPYYh6ftHSwdA/Uy6yFElMpsTl3L+fHXMnv1Mvmtf/qlRwNtPto04S489MkyFpatHe1cNLj2bpIIendSTaNS3Zi4a1zufeuFU/9vAXSF/S5TxKB1N5/Y42V1PjyLfMTLBwsAnp46bUKlXHytQrs60Py+NtCHyO9gGDa8PQVGqnN0RslxIJnkmPfprXt7ePWxgCr+T6dXVwCHLfqgxZBhKSBDu1oWIgJd0p/Xpvarh/kHMhQsXM39+3iuTU0AXEbusp2M0AaFuKZfRmtunB97biXx9SS59DCdvtCbueLNNLrwW6EelurASHuWppk5WQhsDDczFAJZz8bvgn/wNpUhGEHWr2tpUmIFw7lVFRQnh0ckNHQqrtHbmZFRrIYcROC1muz+QWt5vgbhc6+RtcIYWFn9tvudMRRajpK6yMvYjVfOZ4rraQcvK0QJ4mO+TN0WdfikjzU3Hi/6lxWLC3I4eLsjDrJDIe903ffj7sQy6gV4r2oAlNrfe1x9JIITHh//JyBuAdNNzk//Dx88anfjyHOo1aIEjHtGmCkffQtHM2fpVUggyA+/j7SB1Q38cVfly3NZWrJ1oiW5JS7f1zYlZp238Lcto3/62/1ChGJzCH8StSvZy/tmTFl+76IT7KyE12d/Xp0H9250yDmqJOnv7l555SJ0OyVDv2dHLwQa7j62eSlFSIOQhBN8iP26hMMf9etX7F9x6YTxy4i2Srsl37Yt/NJSpK1tY2fn/+cWYucnV2YxBqiGMBzdDjywJkzJ1PTnrTw8gkOfm3K5BnKw1sbvn3URGMl6X4xT8CWyw/GlSoAAAfRSURBVCYnN/XbvbMkkoqPp+2aOGZNRtaj7XtmSOXD0Xh8QVlZ0dHf1r8/7NN14VEd2vU+eHRlXr5sMoOr1w9fvX7onYH/mzP9e3tbt7MXdiPW4Al5BInirhchjkHIO8k1Pv3pU7LJk/4XuoxR4c1b1z4L+99bbw08GHFq+bLVWVkZm79ezaTUEKUgMjLip5/3jHh3TMT+k4MHv/vbqaMRv+xDTaHJxkrhcwl7k8LdvnOazxNMGr3G2dHbxcn3vaFL0jPiYx9UzVgglUr69Qpp4dmeIIjgoIHwK0zPeAjhl/852CGwD0jTzMwK8kg/32DEJvBDz8ngnBNHVkd8CaN5z/fbe7zRG5QEeV5gYIeZM+ZHRV2Ok5fdGqIU3Ll729+/bf/+g2xsbAcNHL51y96uXf6DmgnVcquspAiCLSVCuezp0dbcvGqUq52tq72dR9KTaEUCL/dAZsPMVGazl5UXgRxznqc6O/ko0ni4sTvdOWQ8xUWc6wtNEC9lOCcmPgoICFTs+rduC3/j4u5pjlLQrl3HW7eurV0XfvrMiYLCAnc3Dz+/pg0nIpDy6Pta8NUdQbHmryorL05Nvw/OF+XAwqKa8V31p18qryihKKmJiZkiRCg0RaxCEHwe58ZRvIDVrKC4uLiiosLEpMYTYmYme5+lpSUaopTPAPmlmZn5lauX1qz9nM/n9+zZb/oHsx0cmmbRqnunqoUohEoSYsuRZmlp79MiqH/vWss+mptrGiIpMjEnSZ5EUq4IqRCXIjaBPFhkyr2GTeLFOz2IRDKdlZfXjF0qkevM3s5BQ5TyGUiShBIZ/icnJ96+fX3vvp0lJcWrVjZtWmVKTbhqIVrZC9irIbk5t7p155Sv9yuKGR0ysxMd7TVZwZBH2tq4JqfEvFldJ3kQz+4cphRFu/iwnOm+ADR64Uoi5GH+rdvcu3dXEcJs+7ZspSFK+QxgL7du3cbHp6W3ty/8Lyou+u3UEdQUqqY+UYXqnLJVRwtoVkHsAB4ZiqKO/75JLC7Pfvbk5JktG7aMychqoAtWx3Z9Y+5fgAYV2D7/974nabGINcTFUkTRfh3NkJ5jYmLi6Oh082bUv9E3Kysrhw8befnKxcOHDxQWFULItu0bO73SuZWfP6TUEKXg3PnTYFlfvfoXVBDBlPn78vl2gR1RM6E6R/Rpbwa/vKJn5ZaOzd+4AmZv6Mf7L/z94+YdE7OfJXt5BL43bEmDxkffNyeXlOQdPbXhp4NLoGQf8vbc/b9+xlKfqKykPIGIk/OkNd1YGTtmyvd7d1y/cfXA/pPgnXmWk/3Lrz9u2bYBfITBr772QcjHTDINUQoWzF+6Zev6JcvmI9mQc3soo98bMQ41E2pnA9sb/kRK8Vp2dUXGR/zFFOcWomEzOffs2xc+dvcz7TXSDekne8MShn/o7uGvos6j1jAM6mFbXmwgzVxNRSKRDpvBxV8gM5Ib6S0aWlbUFkBBPa2un8nNjMt3CVA9rV1+Qdb6LWNURpmaWJRVqJ7jxMXR9+Np36HmY+kXfdRFQWsNj6fiAb29OoSMV2vrPb6WYWUr5Obnlt+UYXYD01QT6tTX7vrpHHVCtLSwnz/zR5VRYIUIhaorlyTZzHUvdfcguw1JhVCgYsAhn6dpRreywvLJq7UxWe8LQELTI2mYY1Y0ySK4j3Xs5fykm5k+wS71YyGzsbPVfWWlee/h4d+pnq3MSK5OPSibO4bS8zErLzauedLyFuVF5fkZ7HqPOULa3Wfg2Rw6g8OmwEs4tLlAk/2IysxY3TLtXjYydDLuPy/KLQlZ6Y24zEs4tLnAy830QKIZa1rGnk3Ke2qw+WLq3dzCnOIZa1sibtPE7oic42VnegDT8+ONfk/vZyXe4GIH+pck/nJqaV7J9C99EOfR8wxRE03oYPLRBj9EVd6/kJz5sPmHLOmE5H+zIae3seVPX60HKpRhoCpETZ0NbEqY97UzeXcu5eWlF5paiRx8bS1s9Wdy+2ry0otzkgoqysRCEW/4dE93f/2ZU0pDJUsfeBGHtjq69reF/zf/zL93tSD5VjpUWsC5RZDy/zxCeYpYunqJmOq7gB2i1rSciEYEQaueCbZ2IK3C7K8/I608kEdTdTuwQRhNkdJKKVyxUkxBNcvCVtBvtLt3O+71r9GMhkqWPvCCDm0NBPe1gf+wkfBvScLdooIcSVmJVNaJT0kZJA8pz2RM8mlEyabzVkDwEUHRVJ3pjXmQrJ46eYiu1z1Sfv66gTwTibSirmb5AoIUEAKBwMHNpE1nS9eWRjpMlsu8bDuH3yvm8B9hMC8HRxeFxKhEIJTNWI70Fj6fQGpmN8RC1CcEIqKilK0Oy1oAbAIPX9XWrR6vHmOEeLexzM3U1755V4/nmJjykJoMHQtRn3jzXTswxM7v18sW1+TYwt7vOamL5dZ6zZjGsG/lE6hpderp0CJQD8z/4nz69p/PnsQVTVzqbW6ttoKLhaiX/Lo5/XmmWFpJSaVqP59KjyxN0USdHo1q5xNWMSmourQqHb2I6UBJIFML/ltjnd38NP1ssBD1GTEqK1NypTLrztfsMo3T1btVC9BVC0wRDpmUtHYaxPQ3qx6ErLRSWE1gncQ0KRdjrVXEZLs8nqkFagxYiBhOgN03GE6AhYjhBFiIGE6AhYjhBFiIGE6AhYjhBP8HAAD//12KgYsAAAAGSURBVAMAeTldEe5KWYwAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<IPython.core.display.Image object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "graph_builder = StateGraph(State)\n",
+    "graph_builder.add_node(\"personal_assistant\", personal_assistant)\n",
+    "graph_builder.add_node(\"tools\", ToolNode(tools))\n",
+    "graph_builder.add_conditional_edges(\"personal_assistant\", tools_condition, {\"tools\": \"tools\", \"__end__\": END})\n",
+    "graph_builder.add_edge(START, \"personal_assistant\")\n",
+    "graph_builder.add_edge(\"tools\", \"personal_assistant\")\n",
+    "\n",
+    "graph = graph_builder.compile(checkpointer=memory)\n",
+    "display(Image(graph.get_graph().draw_mermaid_png()))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "368db378",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Device set to use mps:0\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import pipeline\n",
+    "\n",
+    "transcriber = pipeline(\"automatic-speech-recognition\", model=\"openai/whisper-small\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "1733601d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "b0196b9a-8b29-407c-b736-e2d6d6abeeb7\n"
+     ]
+    }
+   ],
+   "source": [
+    "import json\n",
+    "session_id = str(uuid.uuid4())\n",
+    "print(session_id)\n",
+    "\n",
+    "def chat(user_message, uploaded_file, message_history):\n",
+    "    \"\"\"\n",
+    "    Handle chat with text, audio, or file upload.\n",
+    "    The LLM decides what to do with uploaded files.\n",
+    "    \"\"\"\n",
+    "    user_query_parts = []\n",
+    "    try: \n",
+    "        if user_message and user_message.strip():\n",
+    "            user_query_parts.append(user_message)\n",
+    "        \n",
+    "        if uploaded_file is not None:\n",
+    "            result = rag.store_data(uploaded_file)\n",
+    "            result_str = json.dumps(result, indent=2)\n",
+    "            user_query_parts.append(f\"\"\"A medical document was uploaded. Here are the upload details: {result_str} Please inform the user about the upload status in a friendly, professional way.\"\"\")\n",
+    "\n",
+    "        if not user_query_parts:\n",
+    "            return message_history, \"\", None, None\n",
+    "        \n",
+    "        user_query = (' ').join(user_query_parts)\n",
+    "        \n",
+    "        config = {\"configurable\": {\"thread_id\": session_id}, \"recursion_limit\" : 25}\n",
+    "        current_state = graph.get_state(config)\n",
+    "        \n",
+    "        if not current_state.values.get(\"messages\"):\n",
+    "            messages = {\n",
+    "                \"messages\": [\n",
+    "                    {\"role\": \"system\", \"content\": REACT_SYSTEM_PROMPT},\n",
+    "                    {\"role\": \"user\", \"content\": user_query}\n",
+    "                ]\n",
+    "            }\n",
+    "        else:\n",
+    "            messages = {\"messages\": [{\"role\": \"user\", \"content\": user_query}]}\n",
+    "\n",
+    "        result = graph.invoke(\n",
+    "            messages,\n",
+    "            config=config\n",
+    "        )\n",
+    "        \n",
+    "        last_message = result[\"messages\"][-1].content\n",
+    "        \n",
+    "        updated_history = message_history + [\n",
+    "            {\"role\": \"user\", \"content\": user_message},\n",
+    "            {\"role\": \"assistant\", \"content\": last_message}\n",
+    "        ]\n",
+    "        \n",
+    "        return updated_history, \"\", None\n",
+    "        \n",
+    "    except GraphRecursionError:\n",
+    "        error_message = \"This query is too complex and exceeded the reasoning limit. Please simplify or break it into smaller questions.\"\n",
+    "        return message_history + [\n",
+    "            {\"role\": \"assistant\", \"content\": error_message}\n",
+    "        ], \"\", None\n",
+    "       \n",
+    "    except Exception as e:\n",
+    "        error_message = f\"Error: {str(e)}\"\n",
+    "        return message_history + [\n",
+    "            {\"role\": \"assistant\", \"content\": error_message}\n",
+    "        ], \"\", None"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "2cd6085c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "* Running on local URL:  http://127.0.0.1:7860\n",
+      "* Running on public URL: https://a92b3a19656a2e6316.gradio.live\n",
+      "\n",
+      "This share link expires in 1 week. For free permanent hosting and GPU upgrades, run `gradio deploy` from the terminal in the working directory to deploy to Hugging Face Spaces (https://huggingface.co/spaces)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"https://a92b3a19656a2e6316.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def transcribe_audio(audio, current_text, file_input, message_history):\n",
+    "    if audio is None:\n",
+    "        return message_history, current_text, None, file_input\n",
+    "    \n",
+    "    transcript = transcriber(audio)[\"text\"].strip()\n",
+    "    \n",
+    "    updated_history, cleared_text, cleared_file = chat(\n",
+    "        transcript, \n",
+    "        file_input, \n",
+    "        message_history\n",
+    "    )\n",
+    "    \n",
+    "    return updated_history, current_text, None, cleared_file\n",
+    "\n",
+    "with gr.Blocks(title=\"Medical Assistant\") as demo:\n",
+    "    gr.Markdown(\"# 🏥 Medical Assistant\")\n",
+    "    gr.Markdown(\"Ask questions using text, voice, or upload medical documents\")\n",
+    "    \n",
+    "    chatbot = gr.Chatbot(label=\"Conversation\", height=400)\n",
+    "    \n",
+    "    with gr.Row():\n",
+    "        with gr.Column(scale=3):\n",
+    "            text_input = gr.Textbox(\n",
+    "                placeholder=\"Type your medical question here...\",\n",
+    "                label=\"Text Input\",\n",
+    "                lines=2\n",
+    "            )\n",
+    "        with gr.Column(scale=1):\n",
+    "            audio_input = gr.Audio(\n",
+    "                sources=[\"microphone\"],\n",
+    "                type=\"filepath\",\n",
+    "                label=\"🎤 Voice\"\n",
+    "            )\n",
+    "        with gr.Column(scale=1):\n",
+    "            file_input = gr.File(\n",
+    "                label=\"📄 Upload PDF\",\n",
+    "                file_types=[\".pdf\"],\n",
+    "                type=\"filepath\"\n",
+    "            )\n",
+    "    \n",
+    "    with gr.Row():\n",
+    "        submit_btn = gr.Button(\"Send\", variant=\"primary\")\n",
+    "        clear_btn = gr.ClearButton([chatbot, text_input, audio_input, file_input])\n",
+    "    \n",
+    "    gr.Markdown(\"### Tips:\\n- Upload medical records (PDFs) and I'll process them automatically\\n- Ask about medications, interactions, or symptoms\\n- I can store new medical information you share\")\n",
+    "    \n",
+    "    submit_btn.click(\n",
+    "        chat,\n",
+    "        inputs=[text_input, file_input, chatbot],\n",
+    "        outputs=[chatbot, text_input, file_input]\n",
+    "    )\n",
+    "    \n",
+    "    text_input.submit(\n",
+    "        chat,\n",
+    "        inputs=[text_input, file_input, chatbot],\n",
+    "        outputs=[chatbot, text_input, file_input]\n",
+    "    )\n",
+    "\n",
+    "    audio_input.change(\n",
+    "        transcribe_audio,\n",
+    "        inputs=[audio_input, text_input, file_input, chatbot],\n",
+    "        outputs=[chatbot, text_input, audio_input, file_input] \n",
+    "    )\n",
+    "\n",
+    "demo.launch(share=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "0d7db61f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<function __main__.<lambda>()>"
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import atexit\n",
+    "atexit.register(lambda: conn.close())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "599a937a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a6305c14",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv (3.11.9)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

graph_setup.py

@@ -0,0 +1,57 @@
+import sqlite3
+from typing_extensions import TypedDict, Annotated
+from langgraph.graph import START, END, StateGraph
+from langgraph.graph.message import add_messages
+from langgraph.prebuilt import ToolNode, tools_condition
+from langgraph.checkpoint.sqlite import SqliteSaver
+from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint
+
+
+class State(TypedDict):
+    messages: Annotated[list, add_messages]
+
+
+class GraphSetup:
+    def __init__(self, tools):
+        self.tools = tools
+        self.llm = self._setup_llm()
+        self.llm_with_tools = self.llm.bind_tools(self.tools)
+        self.memory = self._setup_memory()
+        self.graph = self._build_graph()
+        
+    def _setup_llm(self):
+        llm = HuggingFaceEndpoint(
+            repo_id="deepseek-ai/DeepSeek-V3",
+            task="text-generation",
+            max_new_tokens=1024,
+            do_sample=False,
+            repetition_penalty=1.03,
+            provider="auto", 
+        )
+        return ChatHuggingFace(llm=llm)
+    
+    def _setup_memory(self):
+        db_path = 'data/long_term_memory.db'
+        conn = sqlite3.connect(db_path, check_same_thread=False)
+        return SqliteSaver(conn)
+    
+    def _personal_assistant(self, state: State):
+        print("assistant responses:")
+        print(state["messages"])
+        messages = state["messages"]
+        return {
+            "messages": self.llm_with_tools.invoke(messages)
+        }
+    
+    def _build_graph(self):
+        graph_builder = StateGraph(State)
+        graph_builder.add_node("personal_assistant", self._personal_assistant)
+        graph_builder.add_node("tools", ToolNode(self.tools))
+        graph_builder.add_conditional_edges("personal_assistant", tools_condition, {"tools": "tools", "__end__": END})
+        graph_builder.add_edge(START, "personal_assistant")
+        graph_builder.add_edge("tools", "personal_assistant")
+        
+        return graph_builder.compile(checkpointer=self.memory)
+    
+    def get_graph(self):
+        return self.graph

main.py

@@ -0,0 +1,83 @@
+import gradio as gr
+from dotenv import load_dotenv
+from rag_setup import RAG_Setup
+from tools import MedicalTools
+from graph_setup import GraphSetup
+from chat_handler import ChatHandler
+from audio_handler import AudioHandler
+
+
+load_dotenv(override=True)
+
+rag = RAG_Setup()
+medical_tools = MedicalTools(rag)
+tools = medical_tools.get_tools()
+graph_setup = GraphSetup(tools)
+graph = graph_setup.get_graph()
+chat_handler = ChatHandler(graph, rag)
+audio_handler = AudioHandler()
+
+
+def transcribe_audio_wrapper(audio, current_text, file_input, message_history):
+    return audio_handler.transcribe_audio(
+        audio, 
+        current_text, 
+        file_input, 
+        message_history, 
+        chat_handler.chat
+    )
+
+
+with gr.Blocks(title="Medical Assistant") as demo:
+    gr.Markdown("# 🏥 Medical Assistant")
+    gr.Markdown("Ask questions using text, voice, or upload medical documents")
+    
+    chatbot = gr.Chatbot(label="Conversation", height=400)
+    
+    with gr.Row():
+        with gr.Column(scale=3):
+            text_input = gr.Textbox(
+                placeholder="Type your medical question here...",
+                label="Text Input",
+                lines=2
+            )
+        with gr.Column(scale=1):
+            audio_input = gr.Audio(
+                sources=["microphone"],
+                type="filepath",
+                label="🎤 Voice"
+            )
+        with gr.Column(scale=1):
+            file_input = gr.File(
+                label="📄 Upload PDF",
+                file_types=[".pdf"],
+                type="filepath"
+            )
+    
+    with gr.Row():
+        submit_btn = gr.Button("Send", variant="primary")
+        clear_btn = gr.ClearButton([chatbot, text_input, audio_input, file_input])
+    
+    gr.Markdown("### Tips:\n- Upload medical records (PDFs) and I'll process them automatically\n- Ask about medications, interactions, or symptoms\n- I can store new medical information you share")
+    
+    submit_btn.click(
+        chat_handler.chat,
+        inputs=[text_input, file_input, chatbot],
+        outputs=[chatbot, text_input, file_input]
+    )
+    
+    text_input.submit(
+        chat_handler.chat,
+        inputs=[text_input, file_input, chatbot],
+        outputs=[chatbot, text_input, file_input]
+    )
+
+    audio_input.change(
+        transcribe_audio_wrapper,
+        inputs=[audio_input, text_input, file_input, chatbot],
+        outputs=[chatbot, text_input, audio_input, file_input] 
+    )
+
+
+if __name__ == "__main__":
+    demo.launch(share=True)

prompts.py

@@ -0,0 +1,42 @@
+REACT_SYSTEM_PROMPT = '''You are a helpful medical assistant with access to patient records and web search.
+
+You solve problems using the ReAct (Reasoning and Acting) framework:
+1. Thought: Reason about what information you need
+2. Action: Call the appropriate tool
+3. Observation: Receive the tool result
+4. Repeat until you can answer confidently
+
+AVAILABLE TOOLS:
+- check_medical_history: Search patient's personal medical records (medications, appointments, conditions, lab results)
+- web_search: Search the web for general medical information, drug interactions, side effects, treatment guidelines
+
+MULTI-STEP REASONING EXAMPLES:
+
+Example 1: Drug Interaction Query
+User: "Can I take ibuprofen with my medicines?"
+Thought: I need to first check what medications the patient is currently taking.
+Action: check_medical_history(query="current medications")
+Observation: Patient takes Metformin, Lisinopril, Atorvastatin, Levothyroxine, Omeprazole, Aspirin, Vitamin D3
+Thought: Now I need to check if ibuprofen interacts with these specific medications, especially Aspirin (both are NSAIDs).
+Action: web_search(query="ibuprofen interactions with aspirin metformin lisinopril atorvastatin")
+Observation: Ibuprofen + Aspirin can reduce aspirin's cardioprotective effect. Risk of bleeding increases. Should avoid concurrent use.
+Answer: Based on your current medications, taking ibuprofen with aspirin is not recommended...
+
+Example 2: Simple Patient Query
+User: "What medications am I taking?"
+Thought: This is a straightforward question about the patient's records.
+Action: check_medical_history(query="current medications")
+Observation: [Patient medication list]
+Answer: You are currently taking...
+
+Example 3: General Medical Question
+User: "What are the side effects of Metformin?"
+Thought: This is a general medical question, not specific to the patient's records.
+Action: web_search(query="Metformin side effects")
+Observation: [Web search results]
+Answer: Common side effects of Metformin include...
+
+CRITICAL RULES:
+- Use multiple tools when needed - don't stop after one tool if more information is required
+- Think step-by-step and be thorough
+'''

rag_setup.py

@@ -0,0 +1,88 @@
+import hashlib
+from langchain_huggingface.embeddings import HuggingFaceEmbeddings
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_chroma import Chroma
+
+
+class RAG_Setup:
+    def __init__(self):
+        self.embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
+        self.vector_store = Chroma(
+            collection_name="medical_history_collection",
+            embedding_function=self.embeddings,
+            persist_directory="data/patient_record_db", 
+        )
+
+    def _calculate_file_hash(self, file_path):
+        sha256 = hashlib.sha256()
+        with open(file_path, 'rb') as f:
+            while chunk := f.read(8192):
+                sha256.update(chunk)
+        return sha256.hexdigest()
+
+    def _is_file_uploaded(self, file_hash):
+        results = self.vector_store.get(
+            where={"file_hash": file_hash},
+            limit=1
+        )
+        return len(results['ids']) > 0
+    
+    def _extract_content(self, file_path):
+        pdf_loader = PyPDFLoader(file_path)
+        content = pdf_loader.load()
+        return content
+
+    def _split_content(self, content):
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, add_start_index=True)
+        chunks = text_splitter.split_documents(content)
+        return chunks
+
+    def _embed_content(self, chunks):
+        self.vector_store.add_documents(chunks)
+
+    def store_data(self, file_path):
+        file_hash = self._calculate_file_hash(file_path)
+        
+        if self._is_file_uploaded(file_hash):
+            return {
+                "status": "skipped",
+                "message": f"File already exists in database"
+            }
+        
+        try:
+            content = self._extract_content(file_path)
+            chunks = self._split_content(content)
+            
+            for chunk in chunks:
+                chunk.metadata.update({
+                    'file_hash': file_hash
+                })
+            
+            self._embed_content(chunks)
+            
+            return {
+                "status": "success",
+                "message": f"File successfully uploaded",
+                "chunks": len(chunks)
+            }
+        except Exception as e:
+            return {
+                "status": "error",
+                "message": f"Failed to upload file: {str(e)}"
+            }
+
+    def retrieve_info(self, query: str):
+        try:
+            results = self.vector_store.similarity_search(query, k=5)
+            print("printing tool results", results)
+            
+            if not results:
+                return "No medical history found for this query."
+            
+            content = "\n\n---DOCUMENT---\n\n".join([doc.page_content for doc in results])
+            
+            return content
+        
+        except Exception as e:
+            return "Failed to retrieve medical record"

readme.md

@@ -0,0 +1,163 @@
+---
+title: MedQuery-Assist
+app_file: main.py
+sdk: gradio
+sdk_version: 6.4.0
+---
+# Medical Assistant Chatbot
+
+A conversational AI medical assistant that supports text, voice, and document-based interactions. Built with LangGraph, RAG, and Gradio.
+
+## Features
+
+- **Multi-modal Input**: Text, voice (Whisper), and PDF document upload
+- **RAG System**: Store and retrieve patient medical records from PDF documents
+- **Web Search**: Access latest medical information via Google Serper API
+- **Conversational Memory**: Maintains context across conversation using LangGraph checkpointing
+- **ReAct Framework**: Step-by-step reasoning with tool usage
+- **Auto-transcription**: Voice messages automatically transcribed and sent
+
+## Architecture
+
+```
+├── rag_setup.py          # Document processing and vector store
+├── tools.py              # Medical history search and web search tools
+├── graph_setup.py        # LangGraph workflow configuration
+├── prompts.py            # System prompts
+├── chat_handler.py       # Chat logic and session management
+├── audio_handler.py      # Audio transcription
+├── main.py               # Gradio interface
+└── data/
+    ├── patient_record_db/    # Chroma vector store
+    └── long_term_memory.db   # SQLite conversation checkpoints
+```
+
+## Installation
+
+```bash
+pip install langgraph langchain-huggingface langchain-community langchain-chroma langgraph-checkpoint-sqlite langchain
+pip install gradio transformers torch
+pip install python-dotenv pypdf sentence-transformers
+```
+
+## Environment Setup
+
+Create a `.env` file:
+
+```env
+HUGGINGFACEHUB_API_TOKEN=your_hf_token
+SERPER_API_KEY=your_serper_key
+```
+
+## Usage
+
+```bash
+python main.py
+```
+
+Access the interface at `http://127.0.0.1:7860`
+
+## How It Works
+
+### 1. Document Upload
+- Upload PDF medical records
+- Documents are chunked, embedded, and stored in Chroma vector database
+- Duplicate detection via file hashing
+
+### 2. Query Processing
+- User queries are processed through LangGraph workflow
+- LLM decides which tools to use (medical history search or web search)
+- Multi-step reasoning follows ReAct pattern
+
+### 3. Voice Input
+- Record audio via microphone
+- Automatic transcription using Whisper-small
+- Auto-send to chat after transcription
+
+### 4. Response Generation
+- DeepSeek-V3 model generates responses
+- Can make multiple tool calls per query
+- Maintains conversation context via SQLite checkpointing
+
+## Components
+
+### RAG_Setup
+- Embeddings: `sentence-transformers/all-mpnet-base-v2`
+- Vector Store: Chroma with persistence
+- Chunk size: 1000 characters
+- Similarity search returns top 5 results
+
+### GraphSetup
+- LLM: DeepSeek-V3 via HuggingFace Inference
+- Max tokens: 1024
+- Recursion limit: 25
+- Memory: SQLite checkpointing
+
+### Tools
+- `check_medical_history`: Searches patient records
+- `web_search`: Google Serper API for medical information
+
+### AudioHandler
+- Model: `openai/whisper-small`
+- Auto-send after transcription
+- Clears audio input after processing
+
+## Session Management
+
+- Each application instance generates a unique session ID
+- All users in the same instance share conversation history
+- Restart application to create new session
+
+## File Structure
+
+```
+data/
+├── patient_record_db/           # Vector embeddings
+│   └── chroma.sqlite3
+└── long_term_memory.db          # Conversation checkpoints
+```
+
+## Limitations
+
+- Single global session (all users share history)
+- SQLite connection with `check_same_thread=False` (thread safety concern)
+- No user authentication
+- File uploads not validated beyond extension
+- No cleanup of uploaded temporary files
+
+## Example Queries
+
+**Simple Query:**
+```
+What medications am I taking?
+```
+
+**Complex Query:**
+```
+Can I take ibuprofen with my current medications?
+```
+
+**Upload Flow:**
+1. Upload PDF medical record
+2. System confirms upload success
+3. Ask questions about the uploaded document
+
+## Dependencies
+
+- langgraph
+- langchain-huggingface
+- langchain-community
+- langchain-chroma
+- gradio
+- transformers
+- sentence-transformers
+- pypdf
+- google-serper-api
+- python-dotenv
+
+## Notes
+
+- Requires active internet for HuggingFace Inference API
+- Requires Serper API key for web search
+- First run downloads embedding model (~400MB)
+- Whisper model downloads on first audio transcription (~500MB)

tools.py

@@ -0,0 +1,29 @@
+from langchain.tools import tool
+from langchain_community.utilities import GoogleSerperAPIWrapper
+
+
+class MedicalTools:
+    def __init__(self, rag_setup):
+        self.rag = rag_setup
+        self.serper = GoogleSerperAPIWrapper()
+        
+    def get_tools(self):
+        @tool
+        def check_medical_history(query: str):
+            '''Retrieves relevent medical history of the user
+
+            Args:
+                query: medical history to be searched for
+            '''
+            return self.rag.retrieve_info(query)
+        
+        @tool
+        def web_search(query: str):
+            ''' Search web for answering queries with latest information
+            Args:
+                query: query to be searched on the web
+            '''
+            print("Websearch tool calling")
+            return self.serper.run(query)
+        
+        return [web_search, check_medical_history]