update hybrid_retriever_tool file

tools/hybrid_retriever_tool.py

@@ -122,10 +122,11 @@ class HybridRetrieverTool(RagTool):
         return "\n\n".join(top_passages)
     
     def summarize_passages(self, topic: str, passages):
-        """Summarize the retrieved content while retaining citations"""
+        """Summarize retrieved content into a coherent short digest, keeping citations."""
         if isinstance(passages, str):
             passages = [passages]
-        # 🧹 Clean each passage (remove links, HTML tags, redundant whitespace)
+
+        # Clean and compress passages 
         main_text = []
         urls = []
         for p in passages:
@@ -133,30 +134,53 @@ class HybridRetrieverTool(RagTool):
             if text:
                 main_text.append(text)
                 urls.extend(found_urls)
-        # Build condensed input (limit total tokens)
-        text_block = " ".join(main_text[:5])[:4000]
+
+        if not main_text:
+            return "No meaningful content found to summarize."
+
+        # --- Limit and re-rank by diversity ---
+        unique_texts = list(dict.fromkeys(main_text))[:5]  # prevent duplication
+        text_block = " ".join(unique_texts)
+        text_block = re.sub(r"\s{2,}", " ", text_block).strip()
+        text_block = text_block[:4000]  # safety limit for token size
+
         unique_urls = list(dict.fromkeys(urls))[:5]
 
+        # --- Structured summarization ---
+        prompt = f"""
+You are a research assistant creating a clean, readable summary.
+
+Topic: {topic}
+
+Condense the following information into **2–3 coherent paragraphs** that:
+1. Focus on factual insights and trends, not raw data or footnotes.
+2. Remove list items, footers, or numeric citations (like (1), (2)).
+3. Retain key facts, organizations, or findings.
+4. Avoid repeating words or phrases.
+5. Conclude with a single “Sources” section listing the most relevant URLs.
+
+Text to summarize:
+{text_block}
+
+Return output in Markdown format.
+"""
+
         try:
             response = self._client.chat.completions.create(
                 model="gpt-4o-mini",
                 messages=[
-                    {
-                        "role": "system",
-                        "content": (
-                            "You are a concise research summarizer. "
-                            "Produce a 1–2 paragraph overview that highlights key facts, "
-                            "themes, and findings relevant to the topic. "
-                            "Exclude URLs or boilerplate text, but clearly label 'Sources' at the end."
-                        ),
-                    },
-                    {"role": "user", "content": f"Summarize these passages about {topic}:\n\n{text_block}"}
+                    {"role": "system", "content": "You are a concise, professional summarizer."},
+                    {"role": "user", "content": prompt},
                 ],
                 temperature=0.3
             )
+
             summary = response.choices[0].message.content.strip()
+
             if unique_urls:
-                summary += "\n\n**Sources**\n" + "\n".join(unique_urls)
+                summary += "\n\n**Sources:**\n" + "\n".join(f"- {u}" for u in unique_urls)
+
             return summary
+
         except Exception as e:
-            return f"Summarization failed: {e}"
+            return f"Summarization failed: {e}"