Update app.py

app.py

@@ -37,7 +37,7 @@ MAX_NEW_TOKENS = 4096
 MAX_CHUNK_TOKENS = 8192
 BATCH_SIZE = 2
 PROMPT_OVERHEAD = 300
-SAFE_SLEEP = 0.5  # seconds between batches
+SAFE_SLEEP = 0.5
 
 # === Utility Functions ===
 def estimate_tokens(text: str) -> int:
@@ -48,6 +48,17 @@ def clean_response(text: str) -> str:
     text = re.sub(r"\n{3,}", "\n\n", text)
     return text.strip()
 
+def remove_duplicate_paragraphs(text: str) -> str:
+    paragraphs = text.strip().split("\n\n")
+    seen = set()
+    unique_paragraphs = []
+    for p in paragraphs:
+        clean_p = p.strip()
+        if clean_p and clean_p not in seen:
+            unique_paragraphs.append(clean_p)
+            seen.add(clean_p)
+    return "\n\n".join(unique_paragraphs)
+
 def extract_text_from_excel(path: str) -> str:
     all_text = []
     xls = pd.ExcelFile(path)
@@ -79,6 +90,9 @@ def extract_text_from_csv(path: str) -> str:
     return "\n".join(all_text)
 
 def extract_text_from_pdf(path: str) -> str:
+    import logging
+    logging.getLogger("pdfminer").setLevel(logging.ERROR)
+
     all_text = []
     try:
         with pdfplumber.open(path) as pdf:
@@ -138,7 +152,6 @@ def init_agent() -> TxAgent:
     agent.init_model()
     return agent
 
-# === Main Processing ===
 def analyze_batches(agent, batches: List[List[str]]) -> List[str]:
     results = []
     for batch in batches:
@@ -172,7 +185,23 @@ def analyze_batches(agent, batches: List[List[str]]) -> List[str]:
     return results
 
 def generate_final_summary(agent, combined: str) -> str:
-    final_prompt = f"Provide a structured medical report based on the following summaries:\n\n{combined}\n\nRespond in detailed medical bullet points."
+    combined = remove_duplicate_paragraphs(combined)
+    final_prompt = f"""
+You are an expert clinical summarizer. Analyze the following summaries carefully and generate a **single final concise structured medical report**, avoiding any repetition or redundancy.
+
+Summaries:
+{combined}
+
+Respond with:
+- Diagnostic Patterns
+- Medication Issues
+- Missed Opportunities
+- Inconsistencies
+- Follow-up Recommendations
+
+Avoid repeating the same points multiple times.
+""".strip()
+
     final_response = ""
     for r in agent.run_gradio_chat(
         message=final_prompt,
@@ -191,7 +220,10 @@ def generate_final_summary(agent, combined: str) -> str:
                     final_response += m.content
         elif hasattr(r, "content"):
             final_response += r.content
-    return clean_response(final_response)
+
+    final_response = clean_response(final_response)
+    final_response = remove_duplicate_paragraphs(final_response)
+    return final_response
 
 def process_report(agent, file, messages: List[Dict[str, str]]) -> Tuple[List[Dict[str, str]], Union[str, None]]:
     if not file or not hasattr(file, "name"):
@@ -231,38 +263,18 @@ def process_report(agent, file, messages: List[Dict[str, str]]) -> Tuple[List[Di
 
 def create_ui(agent):
     with gr.Blocks(css="""
-        html, body, .gradio-container {
-            background: #0e1621;
-            color: #e0e0e0;
-            padding: 16px;
-        }
-        button.svelte-1ipelgc {
-            background: linear-gradient(to right, #1e88e5, #0d47a1) !important;
-            border: 1px solid #0d47a1 !important;
-            color: white !important;
-            font-weight: bold !important;
-            padding: 10px 20px !important;
-            border-radius: 8px !important;
-        }
-        button.svelte-1ipelgc:hover {
-            background: linear-gradient(to right, #2196f3, #1565c0) !important;
-            border: 1px solid #1565c0 !important;
-        }
-        .gr-column {
-            align-items: center !important;
-            gap: 12px;
-        }
-        .gr-file, .gr-button {
-            width: 100% !important;
-            max-width: 400px;
-        }
+        html, body, .gradio-container { background: #0e1621; color: #e0e0e0; padding: 16px; }
+        button.svelte-1ipelgc { background: linear-gradient(to right, #1e88e5, #0d47a1) !important; border: 1px solid #0d47a1 !important; color: white !important; font-weight: bold !important; padding: 10px 20px !important; border-radius: 8px !important; }
+        button.svelte-1ipelgc:hover { background: linear-gradient(to right, #2196f3, #1565c0) !important; border: 1px solid #1565c0 !important; }
+        .gr-column { align-items: center !important; gap: 12px; }
+        .gr-file, .gr-button { width: 100% !important; max-width: 400px; }
     """) as demo:
         gr.Markdown("""
         <h2 style="text-align:center;">📄 CPS: Clinical Patient Support System</h2>
         <p style="text-align:center;">Analyze and summarize unstructured medical files using AI (optimized for A100 GPU).</p>
         """)
         with gr.Column():
-            chatbot = gr.Chatbot(label="🧠 CPS Assistant", height=480, type="messages")  # Reduced height
+            chatbot = gr.Chatbot(label="🧠 CPS Assistant", height=480, type="messages")
             upload = gr.File(label="📂 Upload Medical File", file_types=[".xlsx", ".csv", ".pdf"])
             analyze = gr.Button("🧠 Analyze")
             download = gr.File(label="📥 Download Report", visible=False, interactive=False)
@@ -281,4 +293,4 @@ def create_ui(agent):
 if __name__ == "__main__":
     agent = init_agent()
     ui = create_ui(agent)
-    ui.launch(server_name="0.0.0.0", server_port=7860, allowed_paths=["/data/hf_cache/reports"], share=False)
+    ui.launch(server_name="0.0.0.0", server_port=7860, allowed_paths=["/data/hf_cache/reports"], share=False)