File size: 5,115 Bytes
7b96786 f17c710 7b96786 2451663 7b96786 243ed84 7b96786 243ed84 2451663 243ed84 7b96786 f17c710 7b96786 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 | # scraper.py
import asyncio
import aiohttp
import xml.etree.ElementTree as ET
import ssl
from email.utils import parsedate_to_datetime
class NewsScraper:
def __init__(self, limit=1000):
self.limit = limit
self.ssl_context = ssl.create_default_context()
self.ssl_context.check_hostname = False
self.ssl_context.verify_mode = ssl.CERT_NONE
async def fetch_feed(self, session, url):
try:
async with session.get(
url, timeout=aiohttp.ClientTimeout(total=8)
) as resp:
if resp.status == 200:
return await resp.text()
except:
pass
return ""
def parse_feed(self, xml_text, lookback_date):
articles = []
if not xml_text:
return articles
try:
lb = lookback_date.date()
root = ET.fromstring(xml_text)
for item in root.findall('.//item'):
t = item.findtext('title')
l = item.findtext('link')
p = item.findtext('pubDate')
s = item.find('source')
s_name = s.text if s is not None else "Unknown"
d = item.findtext('description')
if t and l and p:
try:
dt = parsedate_to_datetime(p)
if dt.date() >= lb:
articles.append({
'title': t,
'link': l,
'pub_date': p,
'timestamp': dt.isoformat(),
'source': s_name,
'description': d if d else ""
})
except:
pass
except:
pass
return articles
def _build_queries(self, ticker):
# Resolve aliases if it's a known ticker to expand search coverage
aliases = [ticker]
t_low = ticker.lower()
if t_low in ["^nsei", "nifty", "nifty 50"]:
aliases.extend(["nifty 50", "nifty50", "nifty index", "nse india", "nsei stocks", "indian market", "nifty 100", "nifty next 50"])
elif t_low in ["^bsesn", "sensex"]:
aliases.extend(["sensex", "bse sensex", "bombay stock exchange", "bse india", "sensex 30"])
elif t_low in ["^nsebank", "banknifty"]:
aliases.extend(["bank nifty", "nifty bank", "banknifty", "banking stocks india", "hdfc bank news", "icici bank news"])
queries = []
for a in aliases[:6]: # Use more aliases for broader coverage
queries.extend([
a, f"{a} stock", f"{a} news", f"{a} market",
f"{a} forecast", f"{a} predictions", f"{a} today",
f"{a} analysis", f"{a} outlook", f"{a} update",
f"{a} breakout", f"{a} technicals", f"{a} sentiment"
])
# Add high-yield generic financial terms for context if it's a major index
if t_low in ["^nsei", "^bsesn", "nifty", "sensex"]:
queries.extend([
"indian stock market news", "dalal street updates",
"rbi policy news", "fpi flows india", "nifty earnings season"
])
return list(dict.fromkeys(queries)) # Remove duplicates
async def scrape(self, ticker, lookback_date, progress_cb=None):
queries = self._build_queries(ticker)
all_articles = []
seen = set()
conn = aiohttp.TCPConnector(limit=50, ssl=self.ssl_context)
async with aiohttp.ClientSession(connector=conn) as session:
urls = []
for q in queries:
enc = q.replace(' ', '+')
urls.append(
f"https://news.google.com/rss/search?q={enc}"
f"&hl=en-US&gl=US&ceid=US:en"
)
for i in range(0, len(urls), 20):
if len(all_articles) >= self.limit:
break
batch = urls[i:i+20]
tasks = [self.fetch_feed(session, u) for u in batch]
results = await asyncio.gather(
*tasks, return_exceptions=True
)
for xml in results:
if isinstance(xml, Exception) or not xml:
continue
for a in self.parse_feed(xml, lookback_date):
if a['link'] not in seen:
seen.add(a['link'])
all_articles.append(a)
if progress_cb:
progress_cb(
min(len(all_articles)/self.limit, 1.0),
len(all_articles)
)
await asyncio.sleep(0.1)
print(f"[Scraper] {len(all_articles)} unique articles")
return all_articles[:self.limit]
|