Prescription feature done

agents/__init__.py

@@ -1,2 +1,3 @@
 from .rag_decision_agent import rag_decision
-from .rag_retrieval_agent import get_top_k
+from .rag_retrieval_agent import get_top_k
+from .get_prescription_text_agent import get_prescription_text

agents/get_prescription_text_agent.py

@@ -0,0 +1,31 @@
+from openai import OpenAI
+import os
+import re
+
+from prompts import prescription_text_user
+
+def get_prescription_text(messages):
+    """
+    Function to inference openai to get prescription text.
+    """
+    client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+    prompt = prescription_text_user
+    cleaned_messages = [
+        {
+            "role": "system",
+            "content": [
+                {
+                    "type": "text",
+                    "text": prompt,
+                }
+            ]
+        }
+    ]
+    cleaned_messages.extend(messages)
+    # Make the API call to OpenAI
+    response = client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=cleaned_messages
+    )   
+    prescription_text = response.choices[0].message.content.strip()
+    return prescription_text

agents/rag_decision_agent.py

@@ -9,7 +9,7 @@ def rag_decision(query):
     """
     client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
     prompt = f"""
-    You are an agent of a Diagnostics Lab agentic ai Chatbot system. 
+    You are an agent of a Diagnostics Lab agentic AI Chatbot system. 
     Your task is to determine whether the user's query requires Retrieval-Augmented 
     Generation (RAG) or not. Currently the system has access to information about 
     the following topics: Details and cost of Radiology Scans, Laboratory Tests, Health Packages, 

app.py

@@ -11,6 +11,7 @@ load_dotenv()
 
 from agents import rag_decision
 from agents import get_top_k
+from agents import get_prescription_text
 from prompts import bot_welcome_message, openai_opening_system_message, prescription_upload_message
 
 client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
@@ -36,6 +37,13 @@ with gr.Blocks() as demo:
             temperature=0.7
         )
         return response.choices[0].message.content.strip()
+    
+    def agent4_get_prescription_text(messages):
+        """
+        Function to inference openai to get prescription text.
+        """
+        prescription_text = get_prescription_text(messages)
+        return prescription_text
     ###########################################################################
     def encode_image(image_path):
         with open(image_path, "rb") as f:
@@ -51,18 +59,32 @@ with gr.Blocks() as demo:
         return [{"role": "assistant", "content": bot_welcome_message}], None
         
     def add_message(history, message):
+        # Send the image to the agent4_get_prescription_text
+        messages = []
+        if message["text"] is not None:
+            messages.append({
+                            "role": "user",
+                            "content":[{"type": "text", "text": message["text"]}]
+                            })
         for x in message["files"]:
+            encoded_content = encode_image(x)
+            messages[0]["content"].append({
+                "type": "image_url",
+                "image_url": {"url": f"data:image/jpeg;base64,{encoded_content}"}
+            })
             history.append({"role": "user", "content": {"path": x}})
+        # call agent4_get_prescription_text to get prescription text
+        prescription_text = "Extracted lab tests from prescription: \n" + agent4_get_prescription_text(messages)
+        history.append({"role": "system", "content": prescription_text})
+        
         if message["text"] is not None:
             history.append({"role": "user", "content": message["text"]})
+        
         return history, gr.MultimodalTextbox(value=None, interactive=False, file_count="multiple", placeholder="Enter message or upload file...")
     
     def respond(history):
-        # print("history", history)
         if len(history) == 2:
-            # print("Inside")
             history.insert(0,{"role": "system", "content": openai_opening_system_message})
-            # print("history after insert", history)
         messages = copy.deepcopy(history)
         for i, msg in enumerate(messages):
             if isinstance(msg["content"], str):
@@ -72,37 +94,36 @@ with gr.Blocks() as demo:
                     "text": msg["content"]
                 }]
             if isinstance(msg["content"],tuple):
-                # print("inside tuple", msg["content"])
                 # If the content is a file path, encode it
-                file_path = msg["content"][0]
-                # print("file_path", file_path)
-                encoded_content = encode_image(file_path)
+                # file_path = msg["content"][0]
+                # encoded_content = encode_image(file_path)
                 messages[i]["content"] = [{
-                            "type": "image_url",
-                            "image_url": {"url": f"data:image/jpeg;base64,{encoded_content}"}}]
-                
+                            "type": "text",
+                            "text": "User Image"}]
         clean_messages = [] # OpenAI doesnot accept metadata or options in messages
         for msg in messages:
-            try:
-                if msg["content"][0]["type"] == "image_url":
-                    clean_messages.append({
-                    "role": "user",
-                    "content": [{
-                        "type": "text",
-                        "text": prescription_upload_message
-                    }]})
-            except:
-                pass
             clean_msg = {
                 "role": msg["role"],
                 "content": msg["content"]
             }
-            clean_messages.append(clean_msg)            
-        # print("First messages", messages[0])
+            clean_messages.append(clean_msg)
 
         ########################### AGENTIC WORKFLOW ##########################
         # Call Agent1- the RAG Decision Agent
-        rag_decision = agent1_rag_decision(clean_messages[-1]["content"])
+        if clean_messages[-1]["role"] == "system" and "No prescription found" in clean_messages[-1]["content"]:
+            # If the last message is a system message with "No prescription found", skip RAG decision
+            rag_decision = False
+        elif clean_messages[-2]["role"] == "system" and "No prescription found" in clean_messages[-2]["content"]:
+            rag_decision = False
+        else:
+            rag_query = ""
+            # Get the last 10 messages in the format "role: <message>"
+            last_10 = clean_messages[-10:] if len(clean_messages) > 10 else clean_messages
+            rag_query = "\n".join(
+                f"{msg['role']}: {msg['content'][0]['text'] if isinstance(msg['content'], list) and msg['content'] and 'text' in msg['content'][0] else ''}"
+                for msg in last_10
+            )
+            rag_decision = agent1_rag_decision(rag_query)
 
         if rag_decision == True:
             #Call Agent2 - the RAG Retrieval Agent
@@ -120,12 +141,10 @@ with gr.Blocks() as demo:
             response = agent3_llm_agent(clean_messages)
         #######################################################################
 
-        # print("response:", response)
         # history.append({"role": "assistant", "content": response})
         history.append({"role": "assistant", "content": ""})
 
         # return history
-        # print("history", history)
         for character in response:
             history[-1]['content'] += character
             time.sleep(0.005)

prompts.py

@@ -11,21 +11,28 @@ and may vary from the information provided here.
 **Can I help you with anything?**
 """
 openai_opening_system_message = """"You are a helpful assistant of a diagnostic 
-services business.
+services business in an agentic AI framework.
 The system uses RAG to retrieve relevant information from a knowledge base.
 You can also answer questions based on the information provided by the user.
+If the user has uploaded an image and an agent has replied in the message that No prescription
+found, then just reply "The image does not seem to that of a prescription, please 
+upload good quality staright images of prescription" and nothing else.
 Do not provide any medical advice or diagnosis. 
 """
-prescription_upload_message = """The user has uploaded an image.
-Check if the image contains a prescription, if not, ask the user to upload a valid prescription
-image or provide details about the tests they are interested in. Do not provide 
-information about any other image if the image is not a prescription. Just reply that "I am 
-only trained to assist with prescription images and the uploaded image does not seem 
-to be a prescription." Do not answer any question if the uploaded
-image is not a prescription.
-If the image contains a prescription, extract the tests advised by the doctor 
-and provide details about those tests like rates and offers. For the tests 
+prescription_upload_message = """For the tests 
 which are not in your knowledge base, say, "Not in our lab". 
 At last, also ask the user if You missed any test.
 Do not provide any medical advice or diagnosis. 
+"""
+
+prescription_text_user = f"""You are an agent of a Diagnostics Lab agentic AI 
+Chatbot system using RAG on a knowledge base. Your job is to extract the
+lab tests advised by the doctor in the prescription image uploaded by the user.
+The user has uploaded an image. 
+Check if the image contains a medical prescription, if not, just reply 
+"No prescription found". DO NOT provide any other information about the image 
+if the image is not a prescription.
+If the image is a prescription, extract the tests advised by the doctor and 
+using your knowledge about those tests, enumerate those in the reply.
+Do not reply anything else other than the tests advised by the doctor.
 """