Update application/utils/web_search.py

application/utils/web_search.py

@@ -1,7 +1,8 @@
-import requests
+# application/utils/web_search.py
+from duckduckgo_search import DDGS  # Simpler import
 from bs4 import BeautifulSoup
+import requests
 import re
-from duckduckgo_search import DDGS
 
 class WebScarper:
     def __init__(self):
@@ -19,16 +20,15 @@ class WebScarper:
                 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
             }
             response = requests.get(url, headers=headers, timeout=10)
-            response.raise_for_status()
+            response.raise_for_status()  # Raise HTTPError for bad responses (4xx or 5xx)
             return response.text
         except requests.exceptions.RequestException as e:
             print(f"Error fetching URL {url}: {e}")
             return None
-
     def get_text(self, data):
         soup = BeautifulSoup(data, 'html.parser')
         text = soup.get_text()
-        cleaned_text = re.sub(r'\s+', ' ', text).strip()
+        cleaned_text = re.sub(r'\s+', ' ', text).strip()  # Remove extra whitespace
         return cleaned_text[:4000] if len(cleaned_text) > 4000 else cleaned_text
 
     def scarpe(self, query):