Update app.py

app.py

@@ -20,6 +20,9 @@ from datetime import datetime
 import os
 from dotenv import load_dotenv
 import certifi
+from bs4 import BeautifulSoup
+from trafilatura import extract
+from trafilatura.htmlprocessing import convert_tree
 
 # Load environment variables from a .env file
 load_dotenv()
@@ -88,12 +91,24 @@ def scrape_with_bs4(url, session, max_chars=None):
         logger.error(f"Error scraping {url} with BeautifulSoup: {e}")
         return ""
 
-def scrape_with_trafilatura(url, max_chars=None, timeout=5):
+def scrape_with_trafilatura(url, max_chars=None, timeout=5, use_beautifulsoup=False):
     try:
         response = requests.get(url, timeout=timeout)
         response.raise_for_status()
         downloaded = response.text
-        content = extract(downloaded, include_comments=False, include_tables=True, no_fallback=False)
+        content = ""
+        if use_beautifulsoup:
+            soup = BeautifulSoup(downloaded, "lxml")
+            lxml_tree = convert_tree(soup)[0]
+            content = extract(lxml_tree, include_comments=False, include_tables=True, no_fallback=False)
+        
+        # Fallback mechanism: if BeautifulSoup didn't yield results, try without it
+        if not content and use_beautifulsoup:
+            logger.info("BeautifulSoup method failed to extract content. Trying without BeautifulSoup.")
+            content = extract(downloaded, include_comments=False, include_tables=True, no_fallback=False)
+        # If still no content, use the direct method
+        if not content:
+            content = extract(downloaded, include_comments=False, include_tables=True, no_fallback=False)
         return (content or "")[:max_chars] if max_chars else (content or "")
     except Timeout:
         logger.error(f"Timeout error while scraping {url} with Trafilatura")
@@ -252,7 +267,7 @@ def scrape_full_content(url, scraper="trafilatura", max_chars=3000, timeout=5):
             else:
                 content = soup.get_text(strip=True, separator='\n')
         else:  # trafilatura
-            content = scrape_with_trafilatura(url, max_chars, timeout)
+            content = scrape_with_trafilatura(url, max_chars, timeout, use_beautifulsoup=True)
         
         # Limit the content to max_chars
         return content[:max_chars] if content else ""
@@ -378,7 +393,7 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="trafilatura",
                 logger.warning(f"No more results returned from SearXNG on page {page}.")
                 break
 
-            for result in results:
+             for result in results:
                 if len(scraped_content) >= num_results:
                     break
         
@@ -415,7 +430,7 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="trafilatura",
                                 config = use_config()
                                 config.set("DEFAULT", "USER_AGENT", ua)
                                 
-                                content = scrape_with_trafilatura(url, max_chars, timeout=timeout)
+                                content = scrape_with_trafilatura(url, max_chars, timeout=timeout, use_beautifulsoup=True)
                             
                             if content:
                                 break