#!/usr/bin/env python3 """ Eesha Search - Automatic Seed Generator ========================================= Pulls trending URLs from RSS feeds and appends them to Nutch's seed.txt file. Runs every hour via cron to keep the crawl queue fresh. RSS Sources: - Hacker News (tech trending) - Reddit (general trending) - BBC News (world news) - NYT (headlines) - Wikipedia (featured articles) - Product Hunt (new products) - Lobsters (tech discussion) """ import feedparser import urllib.request import json import os import sys import time import re from datetime import datetime from urllib.parse import urlparse # ─── Configuration ──────────────────────────────────────────────────────── SEED_OUTPUT = os.environ.get('SEED_OUTPUT', '/root/nutch/urls/seed.txt') SEED_ARCHIVE = os.environ.get('SEED_ARCHIVE', '/root/nutch/urls/archive.txt') SCAN_INTERVAL = int(os.environ.get('SCAN_INTERVAL', '3600')) MAX_SEEDS_PER_FEED = 50 MAX_TOTAL_SEEDS = 500 # RSS Feed URLs - curated for diversity and quality DEFAULT_FEEDS = [ # Tech 'https://hnrss.org/frontpage', 'https://lobste.rs/rss', 'https://www.producthunt.com/feed', 'https://techcrunch.com/feed/', # News 'https://feeds.bbci.co.uk/news/world/rss.xml', 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml', 'https://www.theguardian.com/world/rss', # Science & Knowledge 'https://en.wikipedia.org/w/api.php?action=featuredfeed&feed=featured', 'https://www.nature.com/nature.rss', # Culture 'https://www.reddit.com/r/all/.rss', 'https://www.reddit.com/r/science/.rss', 'https://www.reddit.com/r/technology/.rss', ] FEEDS = os.environ.get('RSS_FEEDS', ','.join(DEFAULT_FEEDS)).split(',') # ─── Helper Functions ───────────────────────────────────────────────────── def is_valid_url(url): """Validate URL format and filter out non-HTTP schemes.""" try: parsed = urlparse(url) if parsed.scheme not in ('http', 'https'): return False if not parsed.hostname: return False # Skip common non-indexable domains skip_domains = { 'localhost', '127.0.0.1', '0.0.0.0', 'youtube.com', 'youtu.be', # YouTube pages are poor text sources 'facebook.com', 'instagram.com', 'twitter.com', 'x.com', 'tiktok.com', 'linkedin.com', } if parsed.hostname in skip_domains: return False return True except Exception: return False def load_existing_seeds(): """Load URLs already in the seed file to avoid duplicates.""" existing = set() try: if os.path.exists(SEED_OUTPUT): with open(SEED_OUTPUT, 'r') as f: for line in f: url = line.strip() if url and not url.startswith('#'): existing.add(url) if os.path.exists(SEED_ARCHIVE): with open(SEED_ARCHIVE, 'r') as f: for line in f: url = line.strip() if url and not url.startswith('#'): existing.add(url) except Exception: pass return existing def archive_old_seeds(): """Move current seeds to archive to keep seed file fresh.""" try: if os.path.exists(SEED_OUTPUT): with open(SEED_OUTPUT, 'r') as src, open(SEED_ARCHIVE, 'a') as dst: dst.write(src.read()) # Clear the seed file for new URLs with open(SEED_OUTPUT, 'w') as f: f.write(f"# Eesha Search Seed URLs - Refreshed {datetime.utcnow().isoformat()}\n") except Exception: pass def fetch_feed_urls(feed_url): """Fetch and parse an RSS feed, extracting URLs.""" urls = [] try: headers = {'User-Agent': 'EeshaSearch/0.9.2 (Eesha Browser Search Crawler)'} req = urllib.request.Request(feed_url, headers=headers) with urllib.request.urlopen(req, timeout=15) as response: feed_data = feedparser.parse(response.read()) for entry in feed_data.entries[:MAX_SEEDS_PER_FEED]: url = entry.get('link', '') if url and is_valid_url(url): urls.append(url) except Exception as e: print(f"[WARN] Failed to fetch {feed_url}: {e}") return urls def inject_seeds_to_nutch(): """Tell Nutch to inject new seeds from the seed file.""" try: import subprocess result = subprocess.run( ['nutch', 'inject', '/root/nutch/crawl/crawldb', SEED_OUTPUT], capture_output=True, text=True, timeout=120 ) if result.returncode == 0: print(f"[OK] Seeds injected into Nutch crawl DB") else: print(f"[WARN] Nutch inject returned: {result.stderr}") except Exception as e: print(f"[WARN] Could not inject seeds into Nutch: {e}") def run_seed_cycle(): """Execute one full seed generation cycle.""" print(f"\n{'='*60}") print(f"Eesha Search - Seed Generation Cycle") print(f"Time: {datetime.utcnow().isoformat()}") print(f"{'='*60}") # Load existing seeds to avoid duplicates existing = load_existing_seeds() print(f"[INFO] Found {len(existing)} existing seed URLs") # Archive old seeds archive_old_seeds() # Collect new URLs from all feeds new_urls = [] for feed_url in FEEDS: feed_urls = fetch_feed_urls(feed_url) fresh = [u for u in feed_urls if u not in existing] new_urls.extend(fresh) print(f"[OK] {feed_url}: {len(feed_urls)} total, {len(fresh)} new") # Deduplicate and limit seen = set() unique_urls = [] for url in new_urls: if url not in seen: seen.add(url) unique_urls.append(url) unique_urls = unique_urls[:MAX_TOTAL_SEEDS] # Write new seeds if unique_urls: with open(SEED_OUTPUT, 'a') as f: for url in unique_urls: f.write(url + '\n') print(f"\n[OK] Added {len(unique_urls)} new seed URLs to {SEED_OUTPUT}") # Try to inject into Nutch if available inject_seeds_to_nutch() else: print(f"\n[INFO] No new seed URLs found this cycle") return len(unique_urls) def main(): """Main entry point - either run once or in continuous loop.""" single_run = '--once' in sys.argv if single_run: count = run_seed_cycle() print(f"\n[DONE] Single run complete. Added {count} seeds.") return # Continuous mode - run every SCAN_INTERVAL seconds print(f"Eesha Search Seed Generator starting...") print(f"Scan interval: {SCAN_INTERVAL}s ({SCAN_INTERVAL//60}m)") print(f"Seed output: {SEED_OUTPUT}") print(f"Monitoring {len(FEEDS)} RSS feeds") while True: try: run_seed_cycle() except Exception as e: print(f"[ERROR] Seed cycle failed: {e}") print(f"\n[NEXT] Sleeping {SCAN_INTERVAL}s until next cycle...") time.sleep(SCAN_INTERVAL) if __name__ == '__main__': main()