scraper.py

"""
scraper.py — fast server-side page fetching + clean text extraction.

Speed choices:
- requests.Session() reused across calls (connection pooling, keep-alive)
- gzip/deflate accepted so responses transfer smaller
- trafilatura for extraction (C-accelerated under the hood, much faster
  and cleaner than manual BeautifulSoup heuristics)
- short TTL in-memory cache so repeated fetches of the same URL (e.g.
  from /related hitting the same page twice) don't re-fetch over the wire
"""

import time
import requests
import trafilatura

SESSION = requests.Session()
SESSION.headers.update({
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
        "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
    ),
    "Accept-Encoding": "gzip, deflate",
})

_CACHE = {}
CACHE_TTL_SECONDS = 120


def _cache_get(url: str):
    entry = _CACHE.get(url)
    if not entry:
        return None
    value, expires_at = entry
    if time.time() > expires_at:
        _CACHE.pop(url, None)
        return None
    return value


def _cache_set(url: str, value):
    _CACHE[url] = (value, time.time() + CACHE_TTL_SECONDS)


def fetch_page(url: str, timeout: int = 8):
    """
    Fetches a URL and returns clean extracted content:
    { title, text, excerpt, url }
    Returns {"error": ...} on failure.
    """
    cached = _cache_get(url)
    if cached:
        return cached

    try:
        resp = SESSION.get(url, timeout=timeout)
        resp.raise_for_status()
    except requests.RequestException as e:
        return {"error": f"fetch failed: {e}"}

    html = resp.text

    text = trafilatura.extract(html, include_comments=False, include_tables=False) or ""
    metadata = trafilatura.extract_metadata(html)
    title = (metadata.title if metadata else "") or ""

    result = {
        "url": url,
        "title": title,
        "text": text.strip(),
        "excerpt": text.strip()[:300],
    }

    _cache_set(url, result)
    return result