Update app.py

app.py

@@ -17,8 +17,9 @@ import arxiv
 import scholarly
 import pymed
 import wikipedia
-#from newspaper import Article  # Removed direct import
-from newspaper3k import Article  # Import from newspaper3k
+#from newspaper3k import Article  # Removed newspaper3k
+import trafilatura  # Import trafilatura
+from trafilatura import extract, fetch_url
 import pickle
 import faiss
 import threading
@@ -282,10 +283,10 @@ def tool_search_scholar(query: str, max_results: int = 5) -> list:
 
 def extract_article_content(url: str) -> str:
     try:
-        article = Article(url)
-        article.download()
-        article.parse()
-        return article.text
+        downloaded = fetch_url(url)
+        if downloaded is None:  # Handle potential download failures
+            return ""
+        return extract(downloaded, favor_precision=True) #Added favor_precision
     except Exception as e:
         logger.error(f"Failed to extract article content from {url}: {e}")
         return ""
@@ -575,14 +576,9 @@ def tool_draft_research_plan(prompt: str, entities: list, focus_areas: list = []
     return "Could not generate a research plan due to an error."
 
 def tool_extract_article(url: str) -> str:
-    content = extract_article_content(url)
-    if not content:
-        return f"Could not extract content from {url}"
-
-    if len(content) > MAX_FULL_TEXT_LENGTH:
-        content = content[:MAX_FULL_TEXT_LENGTH] + "... [content truncated]"
-
-    return content
+    # Use trafilatura's extraction function
+    extracted_text = extract_article_content(url)
+    return extracted_text if extracted_text else f"Could not extract content from {url}"
 
 tools = {
     "search_web": {
@@ -679,7 +675,7 @@ tools = {
         "description": "Identifies contradictions across multiple insights.",
         "parameters": {
             "insights": {"type": "array", "description": "Collection of insights to analyze for contradictions."},
-        },
+                },
     },
     "identify_focus_areas": {
         "function": tool_identify_focus_areas,
@@ -761,7 +757,7 @@ def deep_research(prompt):
     context = research_data.get('context', [])
     all_insights = research_data.get('all_insights', [])
     entity_specific_insights = research_data.get('entity_specific_insights', {})
-    intermediate_output = ""  # For Gradio display
+    intermediate_output = ""
     previous_queries = research_data.get('previous_queries', [])
     failed_queries = research_data.get('failed_queries', [])
     reasoning_context = research_data.get('reasoning_context', [])
@@ -772,12 +768,11 @@ def deep_research(prompt):
     contradictions = research_data.get('contradictions', [])
     research_session_id = research_data.get('research_session_id', str(uuid4()))
     
-    # Restore or initialize FAISS index
     global index
     if research_data:
          logger.info("Restoring FAISS Index from loaded data.")
     else:
-        index.reset() #Start Fresh
+        index.reset()
         logger.info("Initialized a fresh FAISS Index")
 
     key_entities_with_descriptions = tool_extract_key_entities(prompt=prompt)
@@ -793,14 +788,13 @@ def deep_research(prompt):
             entity_progress[entity]['queries'] = research_data[entity]['queries']
             entity_progress[entity]['insights'] = research_data[entity]['insights']
 
-    if i == 0:
+    if not focus_areas:  # Corrected placement: outside the loop
         initial_focus_areas = tool_identify_focus_areas(prompt=prompt)
         research_plan = tool_draft_research_plan(prompt=prompt, entities=key_entities, focus_areas=initial_focus_areas)
         context.append(f"Initial Research Plan: {research_plan[:200]}...")
         intermediate_output += f"Initial Research Plan:\n{research_plan}\n\n"
         focus_areas = initial_focus_areas
-    elif not focus_areas:
-        focus_areas = tool_identify_focus_areas(prompt=prompt, insights=all_insights, failed_areas=failed_areas)
+
 
     for i in range(MAX_ITERATIONS):
         if key_entities and i > 0:
@@ -811,8 +805,7 @@ def deep_research(prompt):
 
         context.append(f"Current focus: {current_entity}")
         
-        # FAISS similarity search before web/arxiv/pubmed searches
-        if i > 0: # Don't do it on first iteration
+        if i > 0:
             faiss_results_indices = search_faiss_index(prompt if current_entity == 'general' else f"{prompt} {current_entity}")
             faiss_context = []
             for idx in faiss_results_indices:
@@ -852,7 +845,7 @@ def deep_research(prompt):
                         entity_progress['general']['insights'].append(reasoning_output)
                         reasoning_context.append(reasoning_output)
                         context.append(f"Initial Reasoning: {reasoning_output[:200]}...")
-                        add_to_faiss_index(reasoning_output) # Add reasoning to FAISS
+                        add_to_faiss_index(reasoning_output)
                 else:
                     failed_queries.append(initial_query)
                     context.append(f"Initial query yielded no relevant results: {initial_query}")
@@ -904,7 +897,7 @@ def deep_research(prompt):
                         entity_specific_insights[current_entity].append(entity_reasoning)
 
                         context.append(f"Reasoning about {current_entity}: {entity_reasoning[:200]}...")
-                        add_to_faiss_index(entity_reasoning)  # Add to FAISS
+                        add_to_faiss_index(entity_reasoning)
                 else:
                     failed_queries.append(entity_query)
                     context.append(f"Entity query for {current_entity} yielded no relevant results")
@@ -998,7 +991,7 @@ def deep_research(prompt):
                     entity_specific_insights[current_entity].append(result)
                 else:
                     reasoning_context.append(result)
-                add_to_faiss_index(result) # Add reasoning to FAISS
+                add_to_faiss_index(result)
                 all_insights.append(result)
 
             elif tool_name == "critique_reasoning":
@@ -1040,7 +1033,7 @@ def deep_research(prompt):
                     reasoning_about_article = tool_reason(prompt=prompt, search_results=[{"title": "Extracted Article", "snippet": result, "url": parameters['url']}])
                     if reasoning_about_article:
                         all_insights.append(reasoning_about_article)
-                        add_to_faiss_index(reasoning_about_article) # Add to FAISS
+                        add_to_faiss_index(reasoning_about_article)
 
 
             elif tool_name == "meta_analyze":
@@ -1052,7 +1045,7 @@ def deep_research(prompt):
                 if result:
                     all_insights.append(result)
                     context.append(f"Meta-analysis across entities: {result[:200]}...")
-                    add_to_faiss_index(result)  # Add to FAISS
+                    add_to_faiss_index(result)
 
             elif tool_name == "draft_research_plan":
                 result = "Research plan already generated."
@@ -1077,7 +1070,6 @@ def deep_research(prompt):
             intermediate_output += f"Iteration {i+1} - Error: {str(e)}\n"
             continue
 
-        # Save research data after each iteration
         research_data = {
             'context': context,
             'all_insights': all_insights,
@@ -1088,7 +1080,7 @@ def deep_research(prompt):
             'previous_critiques': previous_critiques,
             'focus_areas': focus_areas,
             'failed_areas': failed_areas,
-            'seen_snippets': list(seen_snippets),  # Convert set to list for pickling
+            'seen_snippets': list(seen_snippets),
             'contradictions': contradictions,
             'research_session_id': research_session_id
         }
@@ -1134,8 +1126,6 @@ def deep_research(prompt):
 
     return full_output
 
-# Gradio Interface
-
 custom_css = """
 .gradio-container {
     background-color: #f7f9fc;
@@ -1143,7 +1133,7 @@ custom_css = """
 .output-box {
     font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
     line-height: 1.5;
-    font-size: 14px; /* Increased font size */
+    font-size: 14px;
 }
 h3 {
     color: #2c3e50;
@@ -1177,7 +1167,7 @@ iface = gr.Interface(
     theme="default",
     cache_examples=False,
     css=custom_css,
-    allow_flagging="never",  # Disable flagging
+    allow_flagging="never",
 )
 
 if __name__ == "__main__":