Update app.py

app.py

@@ -6,6 +6,7 @@ import requests
 from bs4 import BeautifulSoup
 import os
 import gradio as gr
+import asyncio  # Import asyncio for asynchronous processing
 
 # Step 1: Define PromptTemplate class using LangChain's format
 class PromptTemplate:
@@ -109,22 +110,20 @@ def extract_content(url):
 
 # Step 8: Use the LangChain text generation pipeline for generating answers
 generation_model_name = "microsoft/Phi-3-mini-4k-instruct"
-text_generator = pipeline("text-generation", model=generation_model_name, device=-1)  # Use CPU or change to device=0 for GPU
+# Use CPU or change to device=0 for GPU (depending on your setup)
+text_generator = pipeline("text-generation", model=generation_model_name, device=-1)
 
 # Step 9: Function to generate answer based on query and content
-def generate_answer(query, contents):
+async def generate_answer(query, contents):
     answers = []
     prompt_template = PromptTemplate("""
     
     ### Medical Assistant Context ###
 As a helpful medical assistant, I'm here to assist you with your query.
-
 ### Medical Query ###
 Query: {query}
-
 ### Explanation ###
 {generated_text}
-
 ### Revised Response ###
 Response: {generated_text}
 """)
@@ -138,8 +137,8 @@ Response: {generated_text}
     if not batch_prompts:
         return ["No content available to generate an answer."] * len(contents)
 
-    # Generate responses in batch
-    generated_texts = text_generator(batch_prompts, max_new_tokens=200, num_return_sequences=len(batch_prompts), truncation=True)
+    # Generate responses in batch asynchronously
+    generated_texts = await asyncio.gather(*[loop.run_in_executor(None, lambda: text_generator(prompt, max_new_tokens=200, num_return_sequences=1, truncation=True)) for prompt in batch_prompts])
 
     for i, generated_text in enumerate(generated_texts):
         if generated_text and isinstance(generated_text, list) and len(generated_text) > 0:
@@ -156,7 +155,7 @@ def process_query(query):
     top_results = search_similar(query, top_k=3)
     if top_results:
         content = extract_content(top_results[0])
-        answer = generate_answer(query, [content])[0]
+        answer = asyncio.run(generate_answer(query, [content]))[0]
 
         response = f"Rank 1: URL - {top_results[0]}\n"
         response += f"Generated Answer:\n{answer}\n"
@@ -176,4 +175,5 @@ demo = gr.Interface(
 )
 
 if __name__ == "__main__":
+    loop = asyncio.get_event_loop()
     demo.launch()