crawler.py

import json
import os
import concurrent.futures
from typing import Generator, Optional
from urllib.parse import urljoin, urlparse
import requests
import trafilatura
from errors import get_logger, fmt_exc

log = get_logger("crawler")

CREDITS_FILE = "./database/credits.json"
DEFAULT_CREDITS = 10_000

SEED_DOMAINS = [
    "https://it.wikipedia.org",
    "https://en.wikipedia.org",
    "https://www.treccani.it",
    "https://www.ansa.it",
    "https://www.corriere.it",
]

_SKIP_DOMAINS = {
    "facebook.com", "twitter.com", "x.com", "instagram.com", "youtube.com",
    "tiktok.com", "pinterest.com", "linkedin.com", "amazon.com", "ebay.com",
    "google.com", "googleapis.com", "gstatic.com", "doubleclick.net",
}

# Link di licenza/boilerplate che compaiono nel footer di ogni pagina Wikipedia
# — non sono fonti citate, vanno esclusi dalle fonti in nota.
_CITATION_SKIP_DOMAINS = _SKIP_DOMAINS | {
    "creativecommons.org", "wikimediafoundation.org", "foundation.wikimedia.org",
}

_SKIP_EXTENSIONS = (".pdf", ".jpg", ".jpeg", ".png", ".gif", ".zip",
                    ".css", ".js", ".svg", ".ico", ".xml", ".rss", ".atom")

_HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; GenerAI-Spider/2.0; +https://amogaddy-generai.hf.space)"}

# Namespace/azioni tecniche di MediaWiki — non sono articoli, vanno scartati
# dai link normali (es. "Registrati", "Entra", "Discussione", "Modifica").
_WIKI_JUNK_MARKERS = (
    "action=edit", "veaction=", "action=history", "redlink=1",
    "/wiki/Speciale:", "/wiki/Special:", "/wiki/Discussione:", "/wiki/Talk:",
    "/wiki/Utente:", "/wiki/User:", "/wiki/Aiuto:", "/wiki/Help:",
    "/wiki/Wikipedia:", "/wiki/Progetto:", "/wiki/Portale:", "/wiki/Portal:",
    "/wiki/Template:", "/wiki/Categoria:", "/wiki/Category:",
    "/wiki/File:", "/wiki/Modulo:", "/wiki/Module:", "/wiki/MediaWiki:",
    "/w/index.php",
)

# Quante fonti citate in nota seguire al massimo, in aggiunta al budget scelto.
CITATION_BONUS_CAP = 4


def _is_junk_link(href: str) -> bool:
    return any(marker in href for marker in _WIKI_JUNK_MARKERS)


# ── Credit Ledger ──────────────────────────────────────────────────────────────

class CreditLedger:
    def __init__(self):
        os.makedirs("./database", exist_ok=True)
        if os.path.exists(CREDITS_FILE):
            try:
                with open(CREDITS_FILE) as f:
                    self._data = json.load(f)
            except Exception:
                self._data = {"remaining": DEFAULT_CREDITS, "total_used": 0}
        else:
            self._data = {"remaining": DEFAULT_CREDITS, "total_used": 0}
            self._save()

    def _save(self):
        try:
            with open(CREDITS_FILE, "w") as f:
                json.dump(self._data, f)
        except Exception as e:
            log.warning("Impossibile salvare crediti: %s", fmt_exc(e))

    @property
    def remaining(self) -> int:
        return int(self._data.get("remaining", 0))

    @property
    def total_used(self) -> int:
        return int(self._data.get("total_used", 0))

    def use(self, n: int = 1) -> bool:
        if self._data["remaining"] < n:
            return False
        self._data["remaining"] -= n
        self._data["total_used"] = self._data.get("total_used", 0) + n
        self._save()
        return True

    def add(self, n: int):
        self._data["remaining"] = self._data.get("remaining", 0) + n
        self._save()


# ── Helpers ────────────────────────────────────────────────────────────────────

def _score(text: str, query: str) -> float:
    words = [w for w in query.lower().split() if len(w) > 2]
    if not words:
        return 0.0
    t = text.lower()
    return sum(1 for w in words if w in t) / len(words)


def _extract_links(raw_html: str, base_url: str, query: str) -> list[tuple[str, str, float]]:
    links = []
    try:
        from lxml import html as lhtml
        tree = lhtml.fromstring(raw_html)
        tree.make_links_absolute(base_url)
        seen_urls: set[str] = set()
        for a in tree.xpath("//a[@href]"):
            href = a.get("href", "").split("#")[0]
            if not href.startswith("http"):
                continue
            if _is_junk_link(href):
                continue
            if any(href.lower().endswith(ext) for ext in _SKIP_EXTENSIONS):
                continue
            domain = urlparse(href).netloc.lower()
            if any(skip in domain for skip in _SKIP_DOMAINS):
                continue
            if href in seen_urls:
                continue
            seen_urls.add(href)
            anchor = (a.text_content() or "").strip()[:200]
            score = _score(anchor + " " + href, query)
            links.append((href, anchor, score))
    except Exception as e:
        log.debug("Link extraction error: %s", fmt_exc(e))
    links.sort(key=lambda x: -x[2])
    return links


def _extract_citation_links(raw_html: str, base_url: str, limit: int = CITATION_BONUS_CAP) -> list[tuple[str, str]]:
    """Estrae i link alle fonti esterne citate in nota/bibliografia.

    Su Wikipedia/MediaWiki ogni link esterno è marcato con class="external",
    sia dentro le sezioni Note/Bibliografia/Collegamenti esterni sia nel corpo
    del testo — è il modo più affidabile per trovare "le fonti dietro
    l'articolo" senza individuare i confini esatti delle sezioni.

    Su un sito qualsiasi (non Wikipedia) quel marcatore non esiste: si usa un
    criterio generale, cioè qualunque link che porta fuori dal dominio della
    pagina corrente è trattato come una possibile fonte esterna citata.
    """
    links: list[tuple[str, str]] = []
    try:
        from lxml import html as lhtml
        tree = lhtml.fromstring(raw_html)
        tree.make_links_absolute(base_url)
        base_domain = urlparse(base_url).netloc.lower()
        is_wiki = "wikipedia.org" in base_domain
        candidates = (
            tree.xpath('//a[contains(concat(" ", normalize-space(@class), " "), " external ")]')
            if is_wiki else tree.xpath("//a[@href]")
        )
        seen: set[str] = set()
        for a in candidates:
            href = a.get("href", "").split("#")[0]
            if not href.startswith("http") or href in seen:
                continue
            if _is_junk_link(href):
                continue
            if any(href.lower().endswith(ext) for ext in _SKIP_EXTENSIONS):
                continue
            domain = urlparse(href).netloc.lower()
            if any(skip in domain for skip in _CITATION_SKIP_DOMAINS):
                continue
            if not is_wiki and domain == base_domain:
                continue  # su un sito generico contano solo i link VERSO l'esterno
            seen.add(href)
            anchor = (a.text_content() or "").strip()[:200]
            links.append((href, anchor))
            if len(links) >= limit:
                break
    except Exception as e:
        log.debug("Citation extraction error: %s", fmt_exc(e))
    return links


def _get_title(raw_html: str) -> str:
    try:
        from lxml import html as lhtml
        tree = lhtml.fromstring(raw_html)
        t = tree.xpath("//title/text()")
        return (t[0].strip()[:80]) if t else ""
    except Exception:
        return ""


def _find_start_url(query: str) -> Optional[str]:
    for lang in ("it", "en"):
        try:
            resp = requests.get(
                f"https://{lang}.wikipedia.org/w/api.php",
                params={"action": "query", "list": "search", "srsearch": query,
                        "format": "json", "srlimit": 1},
                timeout=8, headers=_HEADERS,
            )
            if resp.status_code == 200:
                results = resp.json().get("query", {}).get("search", [])
                if results:
                    title = results[0]["title"]
                    url = f"https://{lang}.wikipedia.org/wiki/{title.replace(' ', '_')}"
                    log.info("Start URL (%s): %s", lang, url)
                    return url
        except Exception as e:
            log.debug("Wikipedia start fallita (%s): %s", lang, fmt_exc(e))

    # Fallback: opensearch di Wikipedia (suggerimenti, più permissivo della ricerca full-text)
    for lang in ("it", "en"):
        try:
            resp = requests.get(
                f"https://{lang}.wikipedia.org/w/api.php",
                params={"action": "opensearch", "search": query, "limit": 1, "format": "json"},
                timeout=8, headers=_HEADERS,
            )
            if resp.status_code == 200:
                data = resp.json()
                urls = data[3] if len(data) > 3 else []
                if urls:
                    log.info("Start URL opensearch (%s): %s", lang, urls[0])
                    return urls[0]
        except Exception as e:
            log.debug("Wikipedia opensearch fallita (%s): %s", lang, fmt_exc(e))

    return None


def _find_worldmonitor_start(query: str) -> Optional[tuple[str, str]]:
    """Cerca un secondo punto di partenza nei dati locali di World Monitor.
    Ritorna (url, title) oppure None se non trova nulla di pertinente."""
    try:
        import worldmonitor_client
        items = worldmonitor_client.find_relevant(query, max_items=1)
        if items and items[0].get("url"):
            return items[0]["url"], items[0].get("title", items[0]["url"])
    except Exception as e:
        log.debug("World Monitor start fallito: %s", fmt_exc(e))
    return None


# ── Motore di rendering (browser headless, per pagine basate su JavaScript) ────

class _Renderer:
    """Avvia un browser headless (Playwright/Chromium) solo se serve, e lo riusa
    per tutta la sessione di crawl. Si chiude con renderer.close()."""

    def __init__(self):
        self._pw = None
        self._browser = None
        self._failed = False

    def _ensure_browser(self):
        if self._browser is not None or self._failed:
            return self._browser
        try:
            from playwright.sync_api import sync_playwright
            self._pw = sync_playwright().start()
            self._browser = self._pw.chromium.launch(headless=True)
        except Exception as e:
            log.debug("Motore di rendering non disponibile: %s", fmt_exc(e))
            self._failed = True
        return self._browser

    def render(self, url: str) -> Optional[str]:
        browser = self._ensure_browser()
        if not browser:
            return None
        page = None
        try:
            page = browser.new_page(user_agent=_HEADERS["User-Agent"])
            page.goto(url, timeout=15000, wait_until="networkidle")
            return page.content()
        except Exception as e:
            log.debug("Rendering fallito %s: %s", url, fmt_exc(e))
            return None
        finally:
            if page:
                try:
                    page.close()
                except Exception:
                    pass

    def close(self):
        try:
            if self._browser:
                self._browser.close()
        except Exception:
            pass
        try:
            if self._pw:
                self._pw.stop()
        except Exception:
            pass


# ── Core Crawl ─────────────────────────────────────────────────────────────────

def crawl(query: str, budget: int = 10,
          use_credit_fn=None, credits_remaining_fn=None) -> Generator[dict, None, None]:
    """
    Web crawl puro partendo da Wikipedia.
    use_credit_fn()        → callable che scala 1 credito e ritorna bool (True = OK)
    credits_remaining_fn() → callable che ritorna i crediti rimasti (int)
    Se None usa CreditLedger globale.

    Le fonti citate in nota/bibliografia della pagina di partenza vengono seguite
    con un budget extra dedicato (CITATION_BONUS_CAP pagine), oltre al budget
    scelto dall'utente — marcate con is_citation=True negli eventi.

    Yield dicts:
      {"type": "visiting", "url", "depth", "credits_remaining", "is_citation"}
      {"type": "visit", "url", "title", "text", "depth", "parent", "relevance", "credits_remaining", "is_citation"}
      {"type": "done", "total_pages", "credits_used", "citations_used", "credits_remaining"}
      {"type": "error", "message", "credits_remaining"}
    """
    if use_credit_fn is None:
        ledger = CreditLedger()
        use_credit_fn = ledger.use
        credits_remaining_fn = lambda: ledger.remaining

    budget = max(1, min(budget, 100))
    current_credits = credits_remaining_fn()
    if current_credits <= 0:
        yield {"type": "error", "message": "Crediti esauriti.", "credits_remaining": 0}
        return

    start = _find_start_url(query)
    if not start:
        yield {"type": "error", "message": "Impossibile trovare punto di partenza.",
               "credits_remaining": credits_remaining_fn()}
        return

    visited: set[str] = set()
    queue: list[tuple[str, int, str, bool]] = [(start, 0, "query", False)]

    # Secondo nodo di partenza: se World Monitor ha qualcosa di pertinente nei
    # suoi dati locali (notizie/eventi), il ragno parte anche da lì, creando un
    # secondo ramo indipendente accanto a quello di Wikipedia.
    wm_start = _find_worldmonitor_start(query)
    if wm_start and wm_start[0] != start:
        queue.append((wm_start[0], 0, "query", False))
        log.info("Secondo punto di partenza (World Monitor): %s", wm_start[0])

    pages_found = 0
    credits_used = 0
    citations_used = 0
    citations_enqueued = 0

    renderer = _Renderer()
    try:
        while queue:
            url, depth, parent, is_citation = queue.pop(0)
            if url in visited:
                continue

            # Le fonti citate hanno un budget extra dedicato (CITATION_BONUS_CAP),
            # separato dal budget di pagine scelto dall'utente. Il controllo (e il
            # marcare l'URL come visitato) avviene PRIMA di aggiungerlo a visited:
            # se un URL compare sia come link normale che come fonte citata (caso
            # comune — una fonte in nota è spesso anche il link più pertinente nel
            # corpo del testo), la copia scartata per budget esaurito non deve
            # bloccare la copia "citazione" che ha un budget separato ancora libero.
            if is_citation:
                if citations_used >= CITATION_BONUS_CAP:
                    continue
            elif credits_used >= budget:
                continue

            visited.add(url)

            if not use_credit_fn(1):
                yield {"type": "error", "message": "Crediti esauriti.",
                       "credits_remaining": credits_remaining_fn()}
                break
            credits_used += 1
            if is_citation:
                citations_used += 1

            yield {"type": "visiting", "url": url, "depth": depth,
                   "credits_remaining": credits_remaining_fn(), "is_citation": is_citation}
            log.info("[crawl] depth=%d | %s", depth, url)

            raw_html = None
            try:
                resp = requests.get(url, timeout=10, headers=_HEADERS, allow_redirects=True)
                if resp.status_code == 200:
                    raw_html = resp.text
                else:
                    log.debug("HTTP %s per %s", resp.status_code, url)
            except Exception as e:
                log.debug("Fetch fallito %s: %s", url, fmt_exc(e))

            text = None
            if raw_html:
                try:
                    text = trafilatura.extract(
                        raw_html,
                        include_links=False, include_images=False,
                        include_tables=False, no_fallback=False, url=url,
                    )
                except Exception as e:
                    log.debug("trafilatura fallito %s: %s", url, fmt_exc(e))

            # Pagina vuota/JS-only → prova con il motore di rendering (browser headless)
            if not text or len(text) < 40:
                rendered_html = renderer.render(url)
                if rendered_html:
                    raw_html = rendered_html
                    try:
                        text = trafilatura.extract(
                            raw_html,
                            include_links=False, include_images=False,
                            include_tables=False, no_fallback=False, url=url,
                        )
                    except Exception as e:
                        log.debug("trafilatura (render) fallito %s: %s", url, fmt_exc(e))

            if not raw_html or not text or len(text) < 40:
                log.debug("Testo insufficiente per %s", url)
                continue

            title = _get_title(raw_html) or url
            relevance = _score(text[:1000], query)

            pages_found += 1
            yield {
                "type": "visit",
                "url": url,
                "title": title,
                "text": text[:2000],
                "depth": depth,
                "parent": parent,
                "relevance": round(relevance, 3),
                "credits_remaining": credits_remaining_fn(),
                "is_citation": is_citation,
            }

            # Aggiungi link rilevanti alla coda (max depth=2, max 4 link per pagina)
            if credits_used < budget and depth < 2:
                links = _extract_links(raw_html, url, query)
                added = 0
                for link_url, _, _ in links:
                    if link_url not in visited and added < 4:
                        queue.append((link_url, depth + 1, title, False))
                        added += 1

            # Fonti citate in nota — solo dalla pagina di partenza, budget extra dedicato
            if depth == 0 and citations_enqueued < CITATION_BONUS_CAP:
                for link_url, _ in _extract_citation_links(raw_html, url):
                    if link_url not in visited and citations_enqueued < CITATION_BONUS_CAP:
                        queue.append((link_url, depth + 1, title, True))
                        citations_enqueued += 1
    finally:
        renderer.close()

    yield {
        "type": "done",
        "total_pages": pages_found,
        "credits_used": credits_used,
        "citations_used": citations_used,
        "credits_remaining": credits_remaining_fn(),
    }