searcher.py

# core/searcher.py
import re
import time
import hashlib
import threading
import requests
from bs4 import BeautifulSoup
from ddgs import DDGS
from concurrent.futures import ThreadPoolExecutor, as_completed

_USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
]
_ua_index = 0
_ua_lock = threading.Lock()


def _next_ua():
    global _ua_index
    with _ua_lock:
        ua = _USER_AGENTS[_ua_index % len(_USER_AGENTS)]
        _ua_index += 1
    return ua


class WebSearcher:
    def __init__(self):
        self.ddgs = DDGS()
        self.scraper_pool = ThreadPoolExecutor(max_workers=3)
        self._cache = {}
        self._cache_ttl = 300  # 5 minutes
        self._cache_lock = threading.Lock()

    # ── Cache helpers ────────────────────────────────────────────────────────
    def _cache_key(self, query):
        return hashlib.md5(query.lower().strip().encode()).hexdigest()

    def _cache_get(self, key):
        with self._cache_lock:
            entry = self._cache.get(key)
            if entry and time.time() - entry["ts"] < self._cache_ttl:
                return entry["value"]
        return None

    def _cache_set(self, key, value):
        with self._cache_lock:
            self._cache[key] = {"value": value, "ts": time.time()}
            # Evict old entries if cache is too large
            if len(self._cache) > 100:
                oldest = min(self._cache.items(), key=lambda x: x[1]["ts"])
                del self._cache[oldest[0]]

    # ── Search ───────────────────────────────────────────────────────────────
    def search(self, query, max_results=6):
        cache_key = self._cache_key(f"search:{query}")
        cached = self._cache_get(cache_key)
        if cached:
            return cached

        results = []
        seen_urls = set()
        try:
            for r in self.ddgs.text(query, max_results=max_results + 2):
                url = r.get("href", "")
                if url in seen_urls:
                    continue
                # Skip low-quality sources
                if any(skip in url for skip in ["reddit.com/r/", "quora.com", "pinterest.com"]):
                    continue
                seen_urls.add(url)
                results.append({
                    "title": r.get("title", ""),
                    "url": url,
                    "snippet": r.get("body", ""),
                })
                if len(results) >= max_results:
                    break
        except Exception as e:
            print(f"⚠️ Search error: {e}")

        self._cache_set(cache_key, results)
        return results

    # ── Scrape ───────────────────────────────────────────────────────────────
    def scrape_page(self, url):
        try:
            headers = {
                "User-Agent": _next_ua(),
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                "Accept-Language": "en-US,en;q=0.5",
            }
            res = requests.get(url, headers=headers, timeout=6, allow_redirects=True)
            res.raise_for_status()

            soup = BeautifulSoup(res.text, "html.parser")

            # Remove boilerplate
            for tag in soup(["script", "style", "nav", "footer", "header",
                              "aside", "form", "noscript", "iframe", "ads",
                              "[class*='cookie']", "[class*='banner']",
                              "[id*='sidebar']", "[id*='nav']"]):
                tag.decompose()

            # Try to get the main content area
            main = (
                soup.find("main") or
                soup.find("article") or
                soup.find(id="content") or
                soup.find(class_="content") or
                soup.find(class_="post-content") or
                soup.body
            )

            if main:
                text = main.get_text(separator=" ", strip=True)
            else:
                text = soup.get_text(separator=" ", strip=True)

            # Clean up whitespace and repeated chars
            text = re.sub(r"\s{3,}", "  ", text)
            text = re.sub(r"\.{4,}", "...", text)
            return text[:5000]

        except requests.exceptions.Timeout:
            return None
        except requests.exceptions.HTTPError:
            return None
        except Exception as e:
            print(f"⚠️ Scrape error ({url[:40]}): {e}")
            return None

    # ── Parallel search + scrape ─────────────────────────────────────────────
    def search_and_read_parallel(self, query, max_scrape=3, max_results=8):
        """Enhanced search with deeper content extraction."""
        results = self.search(query, max_results=max_results)
        if not results:
            return []

        future_to_idx = {}
        for i, r in enumerate(results[:max_scrape]):
            future = self.scraper_pool.submit(self.scrape_page, r["url"])
            future_to_idx[future] = i

        for future in as_completed(future_to_idx, timeout=15):
            idx = future_to_idx[future]
            try:
                content = future.result()
                if content:
                    results[idx]["full_content"] = content
                    # Extract key facts
                    sentences = content.split('.')[:20]
                    results[idx]["key_points"] = [s.strip() for s in sentences if len(s) > 40 and len(s) < 200]
            except Exception:
                pass

        return results

    def search_and_read(self, query, deep=True):
        """Legacy sequential method for compatibility."""
        results = self.search(query)
        if deep:
            for r in results[:2]:
                content = self.scrape_page(r["url"])
                if content:
                    r["full_content"] = content
        return results