Sync aggiornamento

scraper.py

@@ -1,69 +1,125 @@
+import os
+import requests
 import trafilatura
-from duckduckgo_search import DDGS
 from errors import get_logger, GenerAIError, ErrorCode, fmt_exc
 
 log = get_logger("scraper")
 
+# Istanza SearXNG — configurabile via Secret SEARXNG_URL
+SEARXNG_URL = os.environ.get("SEARXNG_URL", "https://searx.be")
+_active_instance = SEARXNG_URL
+
+PUBLIC_INSTANCES = [
+    "https://searx.be",
+    "https://search.bus-hit.me",
+    "https://searxng.site",
+]
+
+# Motori attivi dentro SearXNG — brave, bing (Edge), torch, duckduckgo come fallback
+SEARXNG_ENGINES = os.environ.get("SEARXNG_ENGINES", "brave,bing,torch,duckduckgo")
+
+def get_active_instance() -> str:
+    return _active_instance
+
+def set_active_instance(url: str):
+    global _active_instance
+    _active_instance = url.rstrip("/")
+    log.info("Istanza SearXNG cambiata: %s", _active_instance)
+
+def get_engines() -> str:
+    return SEARXNG_ENGINES
+
+def set_engines(engines: str):
+    global SEARXNG_ENGINES
+    SEARXNG_ENGINES = engines
+    log.info("Motori SearXNG aggiornati: %s", SEARXNG_ENGINES)
+
+
+def _searxng_search(query: str, max_results: int) -> list[dict]:
+    """Cerca con SearXNG usando più motori. Prova più istanze se la prima fallisce."""
+    active = get_active_instance()
+    instances = [active] + [i for i in PUBLIC_INSTANCES if i != active]
+    engines = get_engines()
+
+    for instance in instances:
+        try:
+            resp = requests.get(
+                f"{instance}/search",
+                params={"q": query, "format": "json", "language": "it", "engines": engines},
+                timeout=8,
+                headers={"User-Agent": "GenerAI/1.0"},
+            )
+            if resp.status_code == 200:
+                data = resp.json()
+                hits = data.get("results", [])[:max_results * 2]
+                if hits:
+                    log.info("SearXNG (%s) [%s] → %d risultati", instance, engines, len(hits))
+                    return hits
+                # Nessun risultato con questi motori → riprova senza filtro motori
+                resp2 = requests.get(
+                    f"{instance}/search",
+                    params={"q": query, "format": "json", "language": "it"},
+                    timeout=8,
+                    headers={"User-Agent": "GenerAI/1.0"},
+                )
+                if resp2.status_code == 200:
+                    hits2 = resp2.json().get("results", [])[:max_results * 2]
+                    log.info("SearXNG (%s) [fallback tutti motori] → %d risultati", instance, len(hits2))
+                    return hits2
+        except Exception as e:
+            log.warning("SearXNG %s fallita: %s", instance, fmt_exc(e))
+    return []
+
 
 def search_and_extract(query: str, max_results: int = 3) -> list[dict]:
-    """Search DuckDuckGo and extract clean text. Returns [] on total failure."""
+    """Cerca con SearXNG ed estrae testo pulito. Returns [] on total failure."""
     log.info("Ricerca web per: %r", query)
 
-    # ── 1. DuckDuckGo search ───────────────────────────────────────────────────
-    try:
-        with DDGS() as ddgs:
-            hits = list(ddgs.text(query, max_results=max_results * 2))
-        log.debug("DuckDuckGo ha restituito %d risultati grezzi", len(hits))
-    except Exception as e:
-        err = GenerAIError(ErrorCode.WEB_SEARCH_FAILED, f"DuckDuckGo non raggiungibile: {fmt_exc(e)}", cause=e)
-        err.log(log)
-        return []
+    hits = _searxng_search(query, max_results)
 
     if not hits:
-        log.warning("[%s] Nessun risultato DuckDuckGo per: %r", ErrorCode.WEB_NO_RESULTS.value, query)
+        log.warning("[%s] Nessun risultato SearXNG per: %r", ErrorCode.WEB_NO_RESULTS.value, query)
         return []
 
-    # ── 2. Fetch + extract each URL ────────────────────────────────────────────
     results = []
     for hit in hits:
         if len(results) >= max_results:
             break
 
-        url = hit.get("href", "")
+        # SearXNG include già uno snippet — usalo se il fetch fallisce
+        url     = hit.get("url", "")
+        title   = hit.get("title", "")
+        snippet = hit.get("content", "")
+
         if not url:
-            log.debug("Hit senza URL, saltato: %s", hit)
             continue
 
         log.debug("Fetching: %s", url)
+        text = None
         try:
             downloaded = trafilatura.fetch_url(url, timeout=10)
+            if downloaded:
+                text = trafilatura.extract(
+                    downloaded,
+                    include_links=False,
+                    include_images=False,
+                    include_tables=False,
+                    no_fallback=False,
+                )
         except Exception as e:
-            log.warning("[%s] fetch_url fallito per %s — %s", ErrorCode.WEB_FETCH_FAILED.value, url, fmt_exc(e))
-            continue
+            log.warning("[%s] fetch fallito per %s — %s", ErrorCode.WEB_FETCH_FAILED.value, url, fmt_exc(e))
 
-        if not downloaded:
-            log.debug("fetch_url ha restituito None per: %s", url)
-            continue
-
-        try:
-            text = trafilatura.extract(
-                downloaded,
-                include_links=False,
-                include_images=False,
-                include_tables=False,
-                no_fallback=False,
-            )
-        except Exception as e:
-            log.warning("[%s] estrazione testo fallita per %s — %s", ErrorCode.WEB_EXTRACT_FAILED.value, url, fmt_exc(e))
-            continue
+        # Fallback allo snippet SearXNG se trafilatura non estrae nulla
+        if not text or len(text) < 100:
+            text = snippet
 
-        if not text or len(text) < 150:
-            log.debug("Testo troppo corto (%d chars) per: %s", len(text) if text else 0, url)
+        if not text or len(text) < 50:
+            log.debug("Testo troppo corto per: %s", url)
             continue
 
         log.info("Estratti %d chars da: %s", len(text), url)
         results.append({
-            "title": hit.get("title", ""),
+            "title": title,
             "url": url,
             "text": text[:2000],
         })