models/evidence_retriever.py

import wikipedia
import requests
from duckduckgo_search import DDGS
import time

class EvidenceRetriever:
    def __init__(self):
        self.wikipedia_timeout = 10
        self.max_evidence_sources = 10
        self.trusted_domains = [
            'reuters.com', 'apnews.com', 'bbc.com', 'nature.com',
            'science.org', 'who.int', 'cdc.gov', 'nasa.gov', 
            'wikipedia.org', '.gov', '.edu'
        ]
        
    def get_evidence(self, keywords):
        """Retrieve evidence from multiple sources with credibility scoring"""
        evidence = []
        
        # Get evidence from Wikipedia (most reliable)
        wiki_evidence = self._get_wikipedia_evidence(keywords)
        evidence.extend(wiki_evidence)
        
        # Get evidence from web search (using DuckDuckGo API)
        web_evidence = self._get_web_search_evidence(keywords)
        evidence.extend(web_evidence)
        
        # Sort by credibility score and return top sources
        evidence.sort(key=lambda x: x.get('credibility_score', 0.0), reverse=True)
        
        return evidence[:10]
    
    def _assign_credibility_score(self, source_type, url):
        """Assign credibility scores based on source type and domain"""
        if 'wikipedia.org' in url:
            return 0.95
        elif any(url.endswith(gov) for gov in ['.gov', '.gov/']):
            return 0.92
        elif any(url.endswith(edu) for edu in ['.edu', '.edu/']):
            return 0.88
        elif any(trusted in url for trusted in ['reuters.com', 'apnews.com', 'bbc.com']):
            return 0.85
        elif any(sci in url for sci in ['nature.com', 'science.org', 'ncbi.nlm.nih.gov']):
            return 0.90
        elif source_type == 'wikipedia':
            return 0.95
        elif source_type == 'academic':
            return 0.88
        elif source_type == 'government':
            return 0.92
        elif source_type == 'news_trusted':
            return 0.80
        else:
            return 0.60
    
    def _get_wikipedia_evidence(self, keywords):
        """Retrieve evidence from Wikipedia"""
        evidence = []
        
        try:
            search_terms = ' '.join(keywords[:4])
            search_results = wikipedia.search(search_terms, results=5)
            
            for title in search_results:
                try:
                    summary = wikipedia.summary(title, sentences=7, auto_suggest=False)
                    url = f'https://en.wikipedia.org/wiki/{title.replace(" ", "_")}'
                    
                    evidence.append({
                        'content': summary,
                        'source': f'Wikipedia - {title}',
                        'url': url,
                        'credibility_score': self._assign_credibility_score('wikipedia', url),
                        'source_type': 'wikipedia'
                    })
                    
                    if len(evidence) >= 3:
                        break
                        
                except (wikipedia.DisambiguationError, wikipedia.PageError):
                    continue
                    
        except Exception as e:
            print(f"Wikipedia search error: {e}")
        
        return evidence
    
    def _get_web_search_evidence(self, keywords):
        """Retrieve evidence from web search using DuckDuckGo library (Robust)"""
        evidence = []
        
        try:
            query = ' '.join(keywords[:5])
            
            # Use DuckDuckGo Search library
            # max_results=5 to get a good mix
            with DDGS() as ddgs:
                results = list(ddgs.text(query, max_results=5))
                
                for result in results:
                    try:
                        title = result.get('title', 'Unknown')
                        snippet = result.get('body', '')
                        url = result.get('href', '')
                        
                        if url and snippet:
                            evidence.append({
                                'content': snippet,
                                'source': f'Web - {title}',
                                'url': url,
                                'credibility_score': self._assign_credibility_score('web', url),
                                'source_type': 'web'
                            })
                    except Exception as loop_e:
                        continue
            
        except Exception as e:
            print(f"DuckDuckGo Search error: {e}")
        
        return evidence[:5]