Fix runtime error: Pin transformers, replace scraping with DDGS

models/evidence_retriever.py

@@ -1,10 +1,7 @@
-# models/evidence_retriever.py
 import wikipedia
 import requests
-from bs4 import BeautifulSoup
+from duckduckgo_search import DDGS
 import time
-from urllib.parse import quote_plus
-
 
 class EvidenceRetriever:
     def __init__(self):
@@ -24,7 +21,7 @@ class EvidenceRetriever:
         wiki_evidence = self._get_wikipedia_evidence(keywords)
         evidence.extend(wiki_evidence)
         
-        # Get evidence from web search (using Google Custom Search as fallback)
+        # Get evidence from web search (using DuckDuckGo API)
         web_evidence = self._get_web_search_evidence(keywords)
         evidence.extend(web_evidence)
         
@@ -89,118 +86,35 @@ class EvidenceRetriever:
         return evidence
     
     def _get_web_search_evidence(self, keywords):
-        """Retrieve evidence from web search using SearXNG metasearch"""
+        """Retrieve evidence from web search using DuckDuckGo library (Robust)"""
         evidence = []
         
         try:
             query = ' '.join(keywords[:5])
             
-            # Method 1: Try DuckDuckGo Lite (more reliable)
-            ddg_results = self._search_duckduckgo_lite(query)
-            evidence.extend(ddg_results)
-            
-            # Method 2: If DuckDuckGo fails, use direct scraping with Google
-            if len(evidence) < 3:
-                google_results = self._search_google_scrape(query)
-                evidence.extend(google_results)
-            
-        except Exception as e:
-            print(f"Web search error: {e}")
-        
-        return evidence[:5]
-    
-    def _search_duckduckgo_lite(self, query):
-        """Search using DuckDuckGo Lite (HTML version, more stable)"""
-        results = []
-        
-        try:
-            url = "https://lite.duckduckgo.com/lite/"
-            data = {"q": query}
-            headers = {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
-            }
-            
-            response = requests.post(url, data=data, headers=headers, timeout=10)
-            
-            if response.status_code == 200:
-                soup = BeautifulSoup(response.text, 'html.parser')
-                
-                # Find all result rows
-                result_table = soup.find_all('tr')
+            # Use DuckDuckGo Search library
+            # max_results=5 to get a good mix
+            with DDGS() as ddgs:
+                results = list(ddgs.text(query, max_results=5))
                 
-                for row in result_table[:10]:
+                for result in results:
                     try:
-                        link = row.find('a', class_='result-link')
-                        snippet_td = row.find('td', class_='result-snippet')
+                        title = result.get('title', 'Unknown')
+                        snippet = result.get('body', '')
+                        url = result.get('href', '')
                         
-                        if link and snippet_td:
-                            result_url = link.get('href', '')
-                            title = link.get_text(strip=True)
-                            snippet = snippet_td.get_text(strip=True)
-                            
-                            if result_url and snippet:
-                                results.append({
-                                    'content': snippet,
-                                    'source': f'Web - {title}',
-                                    'url': result_url,
-                                    'credibility_score': self._assign_credibility_score('web', result_url),
-                                    'source_type': 'web'
-                                })
-                                
-                                if len(results) >= 3:
-                                    break
-                    except:
+                        if url and snippet:
+                            evidence.append({
+                                'content': snippet,
+                                'source': f'Web - {title}',
+                                'url': url,
+                                'credibility_score': self._assign_credibility_score('web', url),
+                                'source_type': 'web'
+                            })
+                    except Exception as loop_e:
                         continue
-                        
-        except Exception as e:
-            print(f"DuckDuckGo Lite error: {e}")
-        
-        return results
-    
-    def _search_google_scrape(self, query):
-        """Search using Google scraping (fallback method)"""
-        results = []
-        
-        try:
-            url = f"https://www.google.com/search?q={quote_plus(query)}"
-            headers = {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
-            }
-            
-            response = requests.get(url, headers=headers, timeout=10)
             
-            if response.status_code == 200:
-                soup = BeautifulSoup(response.text, 'html.parser')
-                
-                # Find search results
-                search_results = soup.find_all('div', class_='g')
-                
-                for result in search_results[:5]:
-                    try:
-                        link = result.find('a')
-                        snippet_div = result.find('div', class_=['VwiC3b', 'lEBKkf'])
-                        
-                        if link and snippet_div:
-                            result_url = link.get('href', '')
-                            title = result.find('h3')
-                            title_text = title.get_text(strip=True) if title else 'Unknown'
-                            snippet = snippet_div.get_text(strip=True)
-                            
-                            if result_url.startswith('http') and snippet:
-                                results.append({
-                                    'content': snippet,
-                                    'source': f'Web - {title_text}',
-                                    'url': result_url,
-                                    'credibility_score': self._assign_credibility_score('web', result_url),
-                                    'source_type': 'web'
-                                })
-                                
-                                if len(results) >= 3:
-                                    break
-                    except:
-                        continue
-                        
         except Exception as e:
-            print(f"Google scrape error: {e}")
+            print(f"DuckDuckGo Search error: {e}")
         
-        return results
+        return evidence[:5]

requirements.txt

@@ -7,7 +7,8 @@ wikipedia
 requests
 beautifulsoup4
 sentence-transformers
-transformers
+transformers==4.36.2
 scikit-learn
 numpy
+duckduckgo-search
 https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl