Update app.py

app.py

@@ -123,7 +123,7 @@ def scrape_with_bs4(url, session):
 @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
 def scrape_with_trafilatura(url):
     try:
-        downloaded = fetch_url(url, timeout=10)
+        downloaded = fetch_url(url)  # Remove the timeout parameter
         if downloaded is None:
             raise ScrapingError("Failed to download content")
         content = extract(downloaded)
@@ -433,11 +433,19 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="trafilatura",
                 else:  # trafilatura
                     content = scrape_with_trafilatura(url)
                 
-                # Limit content to max_chars
+                # Handle different types of content and limit to max_chars
+                if isinstance(content, dict) and 'content' in content:
+                    content['content'] = content['content'][:max_chars]
+                elif isinstance(content, str):
+                    content = content[:max_chars]
+                else:
+                    logger.warning(f"Unexpected content type for URL: {url}")
+                    content = str(content)[:max_chars]
+
                 scraped_content.append({
                     "title": title,
                     "url": url,
-                    "content": content[:max_chars],
+                    "content": content,
                     "scraper": scraper
                 })
             except requests.exceptions.RequestException as e: