api/search.py

"""backend/api/search.py — Search, fetch-page, analyze-image proxy routes (S358).

S358: proxy_search web branch ora usa asyncio.gather per eseguire Wikipedia, DDG HTML,
SearXNG e DDG Instant Answer in parallelo invece di in cascata. Worst case: ~8s → ~8s
(stesso bound sul provider più lento), ma caso medio: ~3s invece di ~20s.
"""
import os, asyncio, html, json
import re as _re
import urllib.request, urllib.parse
from typing import Optional
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel

import logging
_logger = logging.getLogger("api.search")

router = APIRouter()


class SearchRequest(BaseModel):
    query:  str
    type:   str   = "web"
    limit:  int   = 10
    lang:   Optional[str] = None


class FetchPageRequest(BaseModel):
    url: str


class AnalyzeImageRequest(BaseModel):
    dataUrl:  str
    filename: Optional[str] = "image"
    question: Optional[str] = None


# ── Helpers ────────────────────────────────────────────────────────────────────

def _clean_html(raw: str, max_chars: int = 20_000) -> tuple[str, list[str]]:
    code_blocks: list[str] = []
    for m in _re.findall(r'<(?:pre|code)[^>]*>(.*?)</(?:pre|code)>', raw, _re.S | _re.I):
        cleaned = _re.sub(r'<[^>]+>', '', m)
        decoded = html.unescape(cleaned).strip()
        if decoded:
            code_blocks.append(decoded[:2000])
    raw = _re.sub(r'<(script|style)[^>]*>.*?</\1>', '', raw, flags=_re.S | _re.I)
    text = _re.sub(r'<[^>]+>', ' ', raw)
    text = _re.sub(r'\s+', ' ', html.unescape(text)).strip()
    return text[:max_chars], code_blocks[:20]


def _get_title(raw: str) -> str:
    m = _re.search(r'<title[^>]*>(.*?)</title>', raw, _re.S | _re.I)
    if m:
        return html.unescape(_re.sub(r'<[^>]+>', '', m.group(1))).strip()[:200]  # S580: 120→200
    return ""


# ── Web search coroutines (S358: run in parallel via asyncio.gather) ───────────

async def _wiki_search(q: str, limit: int) -> list[dict]:
    try:
        wiki_qs = urllib.parse.urlencode({
            'action': 'query', 'list': 'search', 'srsearch': q,
            'format': 'json', 'utf8': '1', 'srlimit': min(limit, 5), 'srnamespace': '0',
        })
        wiki_req = urllib.request.Request(
            f'https://en.wikipedia.org/w/api.php?{wiki_qs}',
            headers={'User-Agent': 'agente-ai/3.1'},
        )
        wiki_data = await asyncio.to_thread(
            lambda: json.loads(urllib.request.urlopen(wiki_req, timeout=6).read())
        )
        results = []
        for item in wiki_data.get('query', {}).get('search', [])[:limit]:
            snippet = _re.sub(r'<[^>]+>', '', item.get('snippet', ''))
            results.append({
                'title':   item.get('title', ''),
                'url':     'https://en.wikipedia.org/wiki/' + item.get('title', '').replace(' ', '_'),
                # S598: snippet 300→500 — snippet Wikipedia troncati (spesso > 300 chars)
                'snippet': html.unescape(snippet)[:500],
                'source':  'web',
            })
        return results
    except Exception:
        return []


async def _ddg_html_search(q: str, limit: int) -> list[dict]:
    try:
        ddg_qs = urllib.parse.urlencode({'q': q, 'kl': 'it-it'})
        ddg_req = urllib.request.Request(
            f'https://html.duckduckgo.com/html/?{ddg_qs}',
            headers={
                'User-Agent':      'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 Safari/604.1',
                'Accept':          'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                'Accept-Language': 'it-IT,it;q=0.9,en;q=0.8',
            },
        )
        raw_ddg       = await asyncio.to_thread(
            lambda: urllib.request.urlopen(ddg_req, timeout=10).read().decode('utf-8', errors='replace')
        )
        links         = _re.findall(r'class="result__a"[^>]+href="([^"]+)"[^>]*>(.*?)</a>', raw_ddg, _re.S)
        snippets_raw  = _re.findall(r'class="result__snippet"[^>]*>(.*?)</a>', raw_ddg, _re.S)
        results = []
        for i, (href, title_html) in enumerate(links[:limit]):
            title   = html.unescape(_re.sub(r'<[^>]+>', '', title_html)).strip()
            snippet = html.unescape(_re.sub(r'<[^>]+>', '', snippets_raw[i] if i < len(snippets_raw) else '')).strip()
            if title and href.startswith('http'):
                # S598: snippet 300→500 — snippet DDG HTML troncati
                results.append({'title': title, 'url': href, 'snippet': snippet[:500], 'source': 'web'})
        return results
    except Exception:
        return []


async def _searxng_search(q: str, limit: int) -> list[dict]:
    try:
        sx_qs  = urllib.parse.urlencode({'q': q, 'format': 'json', 'language': 'en', 'safesearch': '0', 'categories': 'general'})
        sx_req = urllib.request.Request(
            f'https://searx.be/search?{sx_qs}',
            headers={'User-Agent': 'agente-ai/3.1', 'Accept': 'application/json'},
        )
        sx_data = await asyncio.to_thread(
            lambda: json.loads(urllib.request.urlopen(sx_req, timeout=8).read())
        )
        return [
            {
                'title':   item.get('title', '')[:200],  # S580: 100→200
                'url':     item.get('url', ''),
                # S598: snippet 300→500 — SearXNG content troncato
                'snippet': item.get('content', '')[:500],
                'source':  'web',
            }
            for item in sx_data.get('results', [])[:limit]
        ]
    except Exception:
        return []


async def _brave_search(q: str, limit: int) -> list[dict]:
    """S601: Brave Search API — attivo solo se BRAVE_SEARCH_API_KEY impostata."""
    api_key = os.environ.get("BRAVE_SEARCH_API_KEY", "")
    if not api_key:
        return []
    try:
        import httpx as _httpx_b
        qs = urllib.parse.urlencode({"q": q, "count": min(limit, 20), "search_lang": "it"})
        async with _httpx_b.AsyncClient(timeout=8.0) as c:
            r = await c.get(
                f"https://api.search.brave.com/res/v1/web/search?{qs}",
                headers={
                    "Accept":              "application/json",
                    "Accept-Encoding":     "gzip",
                    "X-Subscription-Token": api_key,
                },
            )
            data = r.json()
        return [
            {
                "title":   item.get("title", "")[:200],
                "url":     item.get("url", ""),
                "snippet": item.get("description", "")[:500],
                "source":  "web",
            }
            for item in data.get("web", {}).get("results", [])[:limit]
            if item.get("url")
        ]
    except Exception:
        return []


async def _tavily_search(q: str, limit: int) -> list[dict]:
    """S601: Tavily Search API — attivo solo se TAVILY_API_KEY impostata."""
    api_key = os.environ.get("TAVILY_API_KEY", "")
    if not api_key:
        return []
    try:
        import httpx as _httpx_t
        async with _httpx_t.AsyncClient(timeout=10.0) as c:
            r = await c.post(
                "https://api.tavily.com/search",
                json={
                    "api_key":      api_key,
                    "query":        q,
                    "max_results":  min(limit, 10),
                    "search_depth": "basic",
                },
                headers={"Content-Type": "application/json"},
            )
            data = r.json()
        return [
            {
                "title":   item.get("title", "")[:200],
                "url":     item.get("url", ""),
                "snippet": item.get("content", "")[:500],
                "source":  "web",
            }
            for item in data.get("results", [])[:limit]
            if item.get("url")
        ]
    except Exception:
        return []


async def _ddg_instant_search(q: str, limit: int) -> list[dict]:
    try:
        qs  = urllib.parse.urlencode({'q': q, 'format': 'json', 'no_html': '1', 'no_redirect': '1', 'skip_disambig': '1'})
        req = urllib.request.Request(
            f'https://api.duckduckgo.com/?{qs}',
            headers={'User-Agent': 'agente-ai/3.1'},
        )
        data = await asyncio.to_thread(
            lambda: json.loads(urllib.request.urlopen(req, timeout=8).read())
        )
        results = []
        if data.get('AbstractText'):
            results.append({
                'title':   data.get('Heading', q),
                'url':     data.get('AbstractURL', ''),
                # S598: snippet 300→500 — DDG instant AbstractText troncato
                'snippet': data['AbstractText'][:500],
                'source':  'web',
            })
        for topic in data.get('RelatedTopics', [])[:limit]:
            if isinstance(topic, dict) and topic.get('Text'):
                results.append({
                    'title':   topic.get('Text', '')[:200],  # S580: 80→200
                    'url':     topic.get('FirstURL', ''),
                    'snippet': topic.get('Text', '')[:300],  # S580: 200→300
                    'source':  'web',
                })
        return results
    except Exception:
        return []


# ── Search ─────────────────────────────────────────────────────────────────────

@router.post('/api/search')
async def proxy_search(req: SearchRequest):
    q = req.query.strip()[:200]
    if not q:
        return {'results': []}

    results = []

    try:
        if req.type == "github_repo":
            qs = urllib.parse.urlencode({'q': q + (f' language:{req.lang}' if req.lang else ''), 'per_page': min(req.limit, 10)})
            url = f'https://api.github.com/search/repositories?{qs}'
            gh_req = urllib.request.Request(url, headers={'User-Agent': 'agente-ai/3.1', 'Accept': 'application/vnd.github+json'})
            gh_token = os.getenv('GITHUB_TOKEN', '')
            if gh_token:
                gh_req.add_header('Authorization', f'Bearer {gh_token}')
            def _do_gh():
                with urllib.request.urlopen(gh_req, timeout=8) as r:
                    return json.loads(r.read())
            data = await asyncio.to_thread(_do_gh)
            for item in data.get('items', [])[:req.limit]:
                results.append({
                    'title':   item.get('full_name', ''),
                    'url':     item.get('html_url', ''),
                    'snippet': (item.get('description') or '')[:300],  # S584: 200→300
                    'source':  'github',
                })

        elif req.type == "npm":
            qs = urllib.parse.urlencode({'text': q, 'size': min(req.limit, 20)})
            url = f'https://registry.npmjs.org/-/v1/search?{qs}'
            _npm_r = urllib.request.Request(url, headers={'User-Agent': 'agente-ai/3.1'})
            data = await asyncio.to_thread(lambda: json.loads(urllib.request.urlopen(_npm_r, timeout=8).read()))
            for obj in data.get('objects', [])[:req.limit]:
                pkg = obj.get('package', {})
                results.append({
                    'title':   pkg.get('name', ''),
                    'url':     f"https://www.npmjs.com/package/{pkg.get('name', '')}",
                    'snippet': pkg.get('description', '')[:300],  # S584: 200→300
                    'source':  'docs',
                })

        elif req.type == "pypi":
            encoded = urllib.parse.quote(q)
            url = f'https://pypi.org/pypi/{encoded}/json'
            try:
                _pypi_r = urllib.request.Request(url, headers={'User-Agent': 'agente-ai/3.1'})
                data = await asyncio.to_thread(lambda: json.loads(urllib.request.urlopen(_pypi_r, timeout=8).read()))
                info = data.get('info', {})
                results.append({
                    'title':   info.get('name', q),
                    'url':     info.get('project_url') or f'https://pypi.org/project/{q}/',
                    'snippet': (info.get('summary') or '')[:300],  # S584: 200→300
                    'source':  'docs',
                })
            except Exception as _exc:
                _logger.debug("[search] silenced %s", type(_exc).__name__)  # noqa: BLE001

        else:
            # S601: Brave+Tavily come provider primari se API key disponibile;
            # fallback automatico a Wikipedia+DDG+SearXNG se non configurati.
            _brave_res = await _brave_search(q, req.limit)
            _tavily_res = await _tavily_search(q, req.limit)
            if _brave_res or _tavily_res:
                # Provider premium disponibili → usa solo loro + Wikipedia per contesto enciclopedico
                batches = await asyncio.gather(
                    asyncio.sleep(0, result=_brave_res),
                    asyncio.sleep(0, result=_tavily_res),
                    _wiki_search(q, min(req.limit, 3)),
                    return_exceptions=True,
                )
            else:
                # Fallback: Wikipedia + DDG + SearXNG (free tier)
                batches = await asyncio.gather(
                    _wiki_search(q, req.limit),
                    _ddg_html_search(q, req.limit),
                    _searxng_search(q, req.limit),
                    _ddg_instant_search(q, req.limit),
                    return_exceptions=True,
                )
            seen_urls: set[str] = set()
            for batch in batches:
                if isinstance(batch, Exception):
                    continue
                for r in batch:
                    url = r.get('url', '')
                    if url and url not in seen_urls:
                        seen_urls.add(url)
                        results.append(r)
                    elif not url:
                        results.append(r)

    except Exception as e:
        return {'results': [], 'error': str(e)}

    return {'results': results[:req.limit]}


# ── Fetch page ─────────────────────────────────────────────────────────────────

@router.post('/api/fetch-page')
async def proxy_fetch_page(req: FetchPageRequest):
    url = req.url.strip()
    if not url.startswith(('http://', 'https://')):
        raise HTTPException(400, detail={'error': 'url_invalido'})
    try:
        import httpx as _httpx_fp
        async with _httpx_fp.AsyncClient(timeout=12, follow_redirects=True) as _hc_fp:
            _r_fp = await _hc_fp.get(url, headers={
                'User-Agent':      'Mozilla/5.0 (compatible; AgenteAI/3.1; +https://github.com/Baida98/AI)',
                'Accept':          'text/html,application/xhtml+xml,*/*;q=0.8',
                'Accept-Language': 'it-IT,it;q=0.9,en;q=0.8',
            })
        raw_bytes = _r_fp.content[:500_000]
        try:
            raw = raw_bytes.decode('utf-8', errors='replace')
        except Exception:
            raw = raw_bytes.decode('latin-1', errors='replace')
        title, code_bks = _get_title(raw), _clean_html(raw)[1]
        text            = _clean_html(raw)[0]
        return {'url': url, 'title': title, 'text': text, 'code_blocks': code_bks}
    except Exception as e:
        return {'error': str(e), 'url': url, 'title': '', 'text': '', 'code_blocks': []}


# ── Analyze image ──────────────────────────────────────────────────────────────

@router.post('/api/analyze-image')
async def analyze_image(body: AnalyzeImageRequest, request: Request):
    """
    Vision AI — OpenRouter free VL models → Gemini fallback.
    [S191] rimossi modelli deprecated.
    [S-GAP23] X-Internal-Token guard — protegge consumi LLM da abusi esterni.
    """
    _internal_token = os.getenv('INTERNAL_TOKEN', '')
    if _internal_token and request.headers.get('X-Internal-Token') != _internal_token:
        raise HTTPException(401, 'Unauthorized')
    import re as _re2, json as _json
    from openai import OpenAI as _OAI

    VISION_MATRIX = []
    or_key  = os.getenv('OPENROUTER_API_KEY', '')
    or_base = 'https://openrouter.ai/api/v1'
    if or_key:
        for model in [
            'nvidia/nemotron-nano-12b-v2-vl:free',
            'nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free',
        ]:
            VISION_MATRIX.append(('openrouter', or_key, or_base, model))

    gemini_key  = os.getenv('GEMINI_API_KEY') or os.getenv('GOOGLE_API_KEY', '')
    gemini_base = 'https://generativelanguage.googleapis.com/v1beta/openai/'
    if gemini_key:
        for model in [
            os.getenv('VISION_MODEL', ''),
            'gemini-2.5-flash',
            'gemini-2.0-flash',
            'gemini-2.0-flash-lite',
        ]:
            if model:
                VISION_MATRIX.append(('gemini', gemini_key, gemini_base, model))

    if not VISION_MATRIX:
        return {'ok': False, 'error': 'Nessun provider vision disponibile. Configura GEMINI_API_KEY o OPENROUTER_API_KEY.'}

    question = body.question or (
        "Analizza questa immagine. Rispondi in italiano con SOLO questo JSON valido:\n"
        "{\n"
        "  \"description\": \"descrizione breve e chiara (max 60 parole)\",\n"
        "  \"tags\": [\"tag1\", \"tag2\", \"tag3\"],\n"
        "  \"objects\": [\"oggetto1\", \"oggetto2\"],\n"
        "  \"text_in_image\": \"testo visibile nell'immagine, o null\",\n"
        "  \"mood\": \"atmosfera es: professionale, casual, naturale, vivace\"\n"
        "}\n"
        "Solo JSON. Nessun testo aggiuntivo prima o dopo."
    )

    messages = [{
        "role": "user",
        "content": [
            {"type": "text", "text": question},
            {"type": "image_url", "image_url": {"url": body.dataUrl, "detail": "low"}},
        ],
    }]

    last_err = None
    tried    = []

    for (pname, api_key, base_url, model) in VISION_MATRIX:
        try:
            vclient = _OAI(api_key=api_key, base_url=base_url)
            resp    = vclient.chat.completions.create(
                model=model, messages=messages, max_tokens=400, temperature=0.1,
            )
            raw = (resp.choices[0].message.content or "").strip()
            m   = _re2.search(r'\{[\s\S]*\}', raw)
            if m:
                try:
                    result = _json.loads(m.group())
                    return {
                        'ok':            True,
                        'description':   str(result.get('description', '')),
                        'tags':          list(result.get('tags', [])),
                        'objects':       list(result.get('objects', [])),
                        'text_in_image': result.get('text_in_image') or None,
                        'mood':          result.get('mood') or None,
                        'provider':      f'{pname}/{model}',
                        'filename':      body.filename,
                    }
                except _json.JSONDecodeError as _exc:
                    _logger.debug("[search] silenced %s", type(_exc).__name__)  # noqa: BLE001
            if raw:
                return {
                    'ok': True, 'description': raw[:300],  # S584: 250→300
                    'tags': [], 'objects': [], 'text_in_image': None, 'mood': None,
                    'provider': f'{pname}/{model}', 'filename': body.filename,
                }
        except Exception as exc:
            last_err = str(exc)
            tried.append(f'{pname}/{model}: {str(exc)[:200]}')  # S608: 60→200
            continue

    return {'ok': False, 'error': f'Tutti i provider vision falliti. Ultimo: {last_err}', 'tried': tried[:5]}