File size: 4,582 Bytes
622a0b7 4aa7201 622a0b7 4aa7201 622a0b7 4aa7201 622a0b7 4aa7201 622a0b7 4aa7201 622a0b7 4aa7201 622a0b7 4aa7201 622a0b7 4aa7201 622a0b7 4aa7201 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | import wikipedia
import requests
from duckduckgo_search import DDGS
import time
class EvidenceRetriever:
def __init__(self):
self.wikipedia_timeout = 10
self.max_evidence_sources = 10
self.trusted_domains = [
'reuters.com', 'apnews.com', 'bbc.com', 'nature.com',
'science.org', 'who.int', 'cdc.gov', 'nasa.gov',
'wikipedia.org', '.gov', '.edu'
]
def get_evidence(self, keywords):
"""Retrieve evidence from multiple sources with credibility scoring"""
evidence = []
# Get evidence from Wikipedia (most reliable)
wiki_evidence = self._get_wikipedia_evidence(keywords)
evidence.extend(wiki_evidence)
# Get evidence from web search (using DuckDuckGo API)
web_evidence = self._get_web_search_evidence(keywords)
evidence.extend(web_evidence)
# Sort by credibility score and return top sources
evidence.sort(key=lambda x: x.get('credibility_score', 0.0), reverse=True)
return evidence[:10]
def _assign_credibility_score(self, source_type, url):
"""Assign credibility scores based on source type and domain"""
if 'wikipedia.org' in url:
return 0.95
elif any(url.endswith(gov) for gov in ['.gov', '.gov/']):
return 0.92
elif any(url.endswith(edu) for edu in ['.edu', '.edu/']):
return 0.88
elif any(trusted in url for trusted in ['reuters.com', 'apnews.com', 'bbc.com']):
return 0.85
elif any(sci in url for sci in ['nature.com', 'science.org', 'ncbi.nlm.nih.gov']):
return 0.90
elif source_type == 'wikipedia':
return 0.95
elif source_type == 'academic':
return 0.88
elif source_type == 'government':
return 0.92
elif source_type == 'news_trusted':
return 0.80
else:
return 0.60
def _get_wikipedia_evidence(self, keywords):
"""Retrieve evidence from Wikipedia"""
evidence = []
try:
search_terms = ' '.join(keywords[:4])
search_results = wikipedia.search(search_terms, results=5)
for title in search_results:
try:
summary = wikipedia.summary(title, sentences=7, auto_suggest=False)
url = f'https://en.wikipedia.org/wiki/{title.replace(" ", "_")}'
evidence.append({
'content': summary,
'source': f'Wikipedia - {title}',
'url': url,
'credibility_score': self._assign_credibility_score('wikipedia', url),
'source_type': 'wikipedia'
})
if len(evidence) >= 3:
break
except (wikipedia.DisambiguationError, wikipedia.PageError):
continue
except Exception as e:
print(f"Wikipedia search error: {e}")
return evidence
def _get_web_search_evidence(self, keywords):
"""Retrieve evidence from web search using DuckDuckGo library (Robust)"""
evidence = []
try:
query = ' '.join(keywords[:5])
# Use DuckDuckGo Search library
# max_results=5 to get a good mix
with DDGS() as ddgs:
results = list(ddgs.text(query, max_results=5))
for result in results:
try:
title = result.get('title', 'Unknown')
snippet = result.get('body', '')
url = result.get('href', '')
if url and snippet:
evidence.append({
'content': snippet,
'source': f'Web - {title}',
'url': url,
'credibility_score': self._assign_credibility_score('web', url),
'source_type': 'web'
})
except Exception as loop_e:
continue
except Exception as e:
print(f"DuckDuckGo Search error: {e}")
return evidence[:5]
|