sync: 141 file da Baida98/AI@32e8386b (2026-08-06 20:21 UTC) [deploy-all]

README.md

@@ -19,7 +19,8 @@ Backend Python FastAPI per Agente AI. Streaming LLM, memoria, esecuzione codice,
 - `POST /api/reason/loop`     — agent reasoning loop
 - `POST /api/exec`            — esecuzione codice Python
 - `POST /api/execute-shell`   — shell commands
-- `POST /api/analyze-image`   — analisi immagine via Vision AI
+- `POST /api/search`          — web search proxy
+- `POST /api/fetch-page`      — page fetch proxy
 - `WS   /ws/terminal`         — PTY WebSocket terminal
 
 ## Stack

api/search.py

@@ -1,8 +1,12 @@
-"""backend/api/search.py — Image analysis routes.
-Rimosse funzionalità di ricerca e fetch proxy per conformità alle policy Hugging Face (Reverse Proxy).
+"""backend/api/search.py — Search, fetch-page, analyze-image proxy routes (S358).
+
+S358: proxy_search web branch ora usa asyncio.gather per eseguire Wikipedia, DDG HTML,
+SearXNG e DDG Instant Answer in parallelo invece di in cascata. Worst case: ~8s → ~8s
+(stesso bound sul provider più lento), ma caso medio: ~3s invece di ~20s.
 """
-import os, asyncio, json
+import os, asyncio, html, json
 import re as _re
+import urllib.request, urllib.parse
 from typing import Optional
 from fastapi import APIRouter, Depends, HTTPException, Request
 from pydantic import BaseModel
@@ -13,20 +17,362 @@ _logger = logging.getLogger("api.search")
 
 router = APIRouter()
 
+
+class SearchRequest(BaseModel):
+    query:  str
+    type:   str   = "web"
+    limit:  int   = 10
+    lang:   Optional[str] = None
+
+
+class FetchPageRequest(BaseModel):
+    url: str
+
+
 class AnalyzeImageRequest(BaseModel):
     dataUrl:  str
     filename: Optional[str] = "image"
     question: Optional[str] = None
 
+
+# ── Helpers ────────────────────────────────────────────────────────────────────
+
+def _clean_html(raw: str, max_chars: int = 20_000) -> tuple[str, list[str]]:
+    code_blocks: list[str] = []
+    for m in _re.findall(r'<(?:pre|code)[^>]*>(.*?)</(?:pre|code)>', raw, _re.S | _re.I):
+        cleaned = _re.sub(r'<[^>]+>', '', m)
+        decoded = html.unescape(cleaned).strip()
+        if decoded:
+            code_blocks.append(decoded[:2000])
+    raw = _re.sub(r'<(script|style)[^>]*>.*?</\1>', '', raw, flags=_re.S | _re.I)
+    text = _re.sub(r'<[^>]+>', ' ', raw)
+    text = _re.sub(r'\s+', ' ', html.unescape(text)).strip()
+    return text[:max_chars], code_blocks[:20]
+
+
+def _get_title(raw: str) -> str:
+    m = _re.search(r'<title[^>]*>(.*?)</title>', raw, _re.S | _re.I)
+    if m:
+        return html.unescape(_re.sub(r'<[^>]+>', '', m.group(1))).strip()[:200]  # S580: 120→200
+    return ""
+
+
+# ── Web search coroutines (S358: run in parallel via asyncio.gather) ───────────
+
+async def _wiki_search(q: str, limit: int) -> list[dict]:
+    try:
+        wiki_qs = urllib.parse.urlencode({
+            'action': 'query', 'list': 'search', 'srsearch': q,
+            'format': 'json', 'utf8': '1', 'srlimit': min(limit, 5), 'srnamespace': '0',
+        })
+        wiki_req = urllib.request.Request(
+            f'https://en.wikipedia.org/w/api.php?{wiki_qs}',
+            headers={'User-Agent': 'agente-ai/3.1'},
+        )
+        wiki_data = await asyncio.to_thread(
+            lambda: json.loads(urllib.request.urlopen(wiki_req, timeout=6).read())
+        )
+        results = []
+        for item in wiki_data.get('query', {}).get('search', [])[:limit]:
+            snippet = _re.sub(r'<[^>]+>', '', item.get('snippet', ''))
+            results.append({
+                'title':   item.get('title', ''),
+                'url':     'https://en.wikipedia.org/wiki/' + item.get('title', '').replace(' ', '_'),
+                # S598: snippet 300→500 — snippet Wikipedia troncati (spesso > 300 chars)
+                'snippet': html.unescape(snippet)[:500],
+                'source':  'web',
+            })
+        return results
+    except Exception:
+        return []
+
+
+async def _ddg_html_search(q: str, limit: int) -> list[dict]:
+    try:
+        ddg_qs = urllib.parse.urlencode({'q': q, 'kl': 'it-it'})
+        ddg_req = urllib.request.Request(
+            f'https://html.duckduckgo.com/html/?{ddg_qs}',
+            headers={
+                'User-Agent':      'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 Safari/604.1',
+                'Accept':          'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                'Accept-Language': 'it-IT,it;q=0.9,en;q=0.8',
+            },
+        )
+        raw_ddg       = await asyncio.to_thread(
+            lambda: urllib.request.urlopen(ddg_req, timeout=10).read().decode('utf-8', errors='replace')
+        )
+        links         = _re.findall(r'class="result__a"[^>]+href="([^"]+)"[^>]*>(.*?)</a>', raw_ddg, _re.S)
+        snippets_raw  = _re.findall(r'class="result__snippet"[^>]*>(.*?)</a>', raw_ddg, _re.S)
+        results = []
+        for i, (href, title_html) in enumerate(links[:limit]):
+            title   = html.unescape(_re.sub(r'<[^>]+>', '', title_html)).strip()
+            snippet = html.unescape(_re.sub(r'<[^>]+>', '', snippets_raw[i] if i < len(snippets_raw) else '')).strip()
+            if title and href.startswith('http'):
+                # S598: snippet 300→500 — snippet DDG HTML troncati
+                results.append({'title': title, 'url': href, 'snippet': snippet[:500], 'source': 'web'})
+        return results
+    except Exception:
+        return []
+
+
+async def _searxng_search(q: str, limit: int) -> list[dict]:
+    try:
+        sx_qs  = urllib.parse.urlencode({'q': q, 'format': 'json', 'language': 'en', 'safesearch': '0', 'categories': 'general'})
+        sx_req = urllib.request.Request(
+            f'https://searx.be/search?{sx_qs}',
+            headers={'User-Agent': 'agente-ai/3.1', 'Accept': 'application/json'},
+        )
+        sx_data = await asyncio.to_thread(
+            lambda: json.loads(urllib.request.urlopen(sx_req, timeout=8).read())
+        )
+        return [
+            {
+                'title':   item.get('title', '')[:200],  # S580: 100→200
+                'url':     item.get('url', ''),
+                # S598: snippet 300→500 — SearXNG content troncato
+                'snippet': item.get('content', '')[:500],
+                'source':  'web',
+            }
+            for item in sx_data.get('results', [])[:limit]
+        ]
+    except Exception:
+        return []
+
+
+async def _brave_search(q: str, limit: int) -> list[dict]:
+    """S601: Brave Search API — attivo solo se BRAVE_SEARCH_API_KEY impostata."""
+    api_key = os.environ.get("BRAVE_SEARCH_API_KEY", "")
+    if not api_key:
+        return []
+    try:
+        import httpx as _httpx_b
+        qs = urllib.parse.urlencode({"q": q, "count": min(limit, 20), "search_lang": "it"})
+        async with _httpx_b.AsyncClient(timeout=8.0) as c:
+            r = await c.get(
+                f"https://api.search.brave.com/res/v1/web/search?{qs}",
+                headers={
+                    "Accept":              "application/json",
+                    "Accept-Encoding":     "gzip",
+                    "X-Subscription-Token": api_key,
+                },
+            )
+            data = r.json()
+        return [
+            {
+                "title":   item.get("title", "")[:200],
+                "url":     item.get("url", ""),
+                "snippet": item.get("description", "")[:500],
+                "source":  "web",
+            }
+            for item in data.get("web", {}).get("results", [])[:limit]
+            if item.get("url")
+        ]
+    except Exception:
+        return []
+
+
+async def _tavily_search(q: str, limit: int) -> list[dict]:
+    """S601: Tavily Search API — attivo solo se TAVILY_API_KEY impostata."""
+    api_key = os.environ.get("TAVILY_API_KEY", "")
+    if not api_key:
+        return []
+    try:
+        import httpx as _httpx_t
+        async with _httpx_t.AsyncClient(timeout=10.0) as c:
+            r = await c.post(
+                "https://api.tavily.com/search",
+                json={
+                    "api_key":      api_key,
+                    "query":        q,
+                    "max_results":  min(limit, 10),
+                    "search_depth": "basic",
+                },
+                headers={"Content-Type": "application/json"},
+            )
+            data = r.json()
+        return [
+            {
+                "title":   item.get("title", "")[:200],
+                "url":     item.get("url", ""),
+                "snippet": item.get("content", "")[:500],
+                "source":  "web",
+            }
+            for item in data.get("results", [])[:limit]
+            if item.get("url")
+        ]
+    except Exception:
+        return []
+
+
+async def _ddg_instant_search(q: str, limit: int) -> list[dict]:
+    try:
+        qs  = urllib.parse.urlencode({'q': q, 'format': 'json', 'no_html': '1', 'no_redirect': '1', 'skip_disambig': '1'})
+        req = urllib.request.Request(
+            f'https://api.duckduckgo.com/?{qs}',
+            headers={'User-Agent': 'agente-ai/3.1'},
+        )
+        data = await asyncio.to_thread(
+            lambda: json.loads(urllib.request.urlopen(req, timeout=8).read())
+        )
+        results = []
+        if data.get('AbstractText'):
+            results.append({
+                'title':   data.get('Heading', q),
+                'url':     data.get('AbstractURL', ''),
+                # S598: snippet 300→500 — DDG instant AbstractText troncato
+                'snippet': data['AbstractText'][:500],
+                'source':  'web',
+            })
+        for topic in data.get('RelatedTopics', [])[:limit]:
+            if isinstance(topic, dict) and topic.get('Text'):
+                results.append({
+                    'title':   topic.get('Text', '')[:200],  # S580: 80→200
+                    'url':     topic.get('FirstURL', ''),
+                    'snippet': topic.get('Text', '')[:300],  # S580: 200→300
+                    'source':  'web',
+                })
+        return results
+    except Exception:
+        return []
+
+
+# ── Search ─────────────────────────────────────────────────────────────────────
+
+@router.post('/api/search')
+async def proxy_search(req: SearchRequest, role: AuthRole = Depends(require_role(AuthRole.MACHINE))):
+    q = req.query.strip()[:200]
+    if not q:
+        return {'results': []}
+
+    results = []
+
+    try:
+        if req.type == "github_repo":
+            qs = urllib.parse.urlencode({'q': q + (f' language:{req.lang}' if req.lang else ''), 'per_page': min(req.limit, 10)})
+            url = f'https://api.github.com/search/repositories?{qs}'
+            gh_req = urllib.request.Request(url, headers={'User-Agent': 'agente-ai/3.1', 'Accept': 'application/vnd.github+json'})
+            gh_token = os.getenv('GITHUB_TOKEN', '')
+            if gh_token:
+                gh_req.add_header('Authorization', f'Bearer {gh_token}')
+            def _do_gh():
+                with urllib.request.urlopen(gh_req, timeout=8) as r:
+                    return json.loads(r.read())
+            data = await asyncio.to_thread(_do_gh)
+            for item in data.get('items', [])[:req.limit]:
+                results.append({
+                    'title':   item.get('full_name', ''),
+                    'url':     item.get('html_url', ''),
+                    'snippet': (item.get('description') or '')[:300],  # S584: 200→300
+                    'source':  'github',
+                })
+
+        elif req.type == "npm":
+            qs = urllib.parse.urlencode({'text': q, 'size': min(req.limit, 20)})
+            url = f'https://registry.npmjs.org/-/v1/search?{qs}'
+            _npm_r = urllib.request.Request(url, headers={'User-Agent': 'agente-ai/3.1'})
+            data = await asyncio.to_thread(lambda: json.loads(urllib.request.urlopen(_npm_r, timeout=8).read()))
+            for obj in data.get('objects', [])[:req.limit]:
+                pkg = obj.get('package', {})
+                results.append({
+                    'title':   pkg.get('name', ''),
+                    'url':     f"https://www.npmjs.com/package/{pkg.get('name', '')}",
+                    'snippet': pkg.get('description', '')[:300],  # S584: 200→300
+                    'source':  'docs',
+                })
+
+        elif req.type == "pypi":
+            encoded = urllib.parse.quote(q)
+            url = f'https://pypi.org/pypi/{encoded}/json'
+            try:
+                _pypi_r = urllib.request.Request(url, headers={'User-Agent': 'agente-ai/3.1'})
+                data = await asyncio.to_thread(lambda: json.loads(urllib.request.urlopen(_pypi_r, timeout=8).read()))
+                info = data.get('info', {})
+                results.append({
+                    'title':   info.get('name', q),
+                    'url':     info.get('project_url') or f'https://pypi.org/project/{q}/',
+                    'snippet': (info.get('summary') or '')[:300],  # S584: 200→300
+                    'source':  'docs',
+                })
+            except Exception as _exc:
+                _logger.debug("[search] silenced %s", type(_exc).__name__)  # noqa: BLE001
+
+        else:
+            # S601: Brave+Tavily come provider primari se API key disponibile;
+            # fallback automatico a Wikipedia+DDG+SearXNG se non configurati.
+            _brave_res = await _brave_search(q, req.limit)
+            _tavily_res = await _tavily_search(q, req.limit)
+            if _brave_res or _tavily_res:
+                # Provider premium disponibili → usa solo loro + Wikipedia per contesto enciclopedico
+                batches = await asyncio.gather(
+                    asyncio.sleep(0, result=_brave_res),
+                    asyncio.sleep(0, result=_tavily_res),
+                    _wiki_search(q, min(req.limit, 3)),
+                    return_exceptions=True,
+                )
+            else:
+                # Fallback: Wikipedia + DDG + SearXNG (free tier)
+                batches = await asyncio.gather(
+                    _wiki_search(q, req.limit),
+                    _ddg_html_search(q, req.limit),
+                    _searxng_search(q, req.limit),
+                    _ddg_instant_search(q, req.limit),
+                    return_exceptions=True,
+                )
+            seen_urls: set[str] = set()
+            for batch in batches:
+                if isinstance(batch, Exception):
+                    continue
+                for r in batch:
+                    url = r.get('url', '')
+                    if url and url not in seen_urls:
+                        seen_urls.add(url)
+                        results.append(r)
+                    elif not url:
+                        results.append(r)
+
+    except Exception as e:
+        return {'results': [], 'error': str(e)}
+
+    return {'results': results[:req.limit]}
+
+
+# ── Fetch page ─────────────────────────────────────────────────────────────────
+
+@router.post('/api/fetch-page')
+async def proxy_fetch_page(req: FetchPageRequest, role: AuthRole = Depends(require_role(AuthRole.MACHINE))):
+    url = req.url.strip()
+    if not url.startswith(('http://', 'https://')):
+        raise HTTPException(400, detail={'error': 'url_invalido'})
+    try:
+        import httpx as _httpx_fp
+        async with _httpx_fp.AsyncClient(timeout=12, follow_redirects=True) as _hc_fp:
+            _r_fp = await _hc_fp.get(url, headers={
+                'User-Agent':      'Mozilla/5.0 (compatible; AgenteAI/3.1; +https://github.com/Baida98/AI)',
+                'Accept':          'text/html,application/xhtml+xml,*/*;q=0.8',
+                'Accept-Language': 'it-IT,it;q=0.9,en;q=0.8',
+            })
+        raw_bytes = _r_fp.content[:500_000]
+        try:
+            raw = raw_bytes.decode('utf-8', errors='replace')
+        except Exception:
+            raw = raw_bytes.decode('latin-1', errors='replace')
+        title, code_bks = _get_title(raw), _clean_html(raw)[1]
+        text            = _clean_html(raw)[0]
+        return {'url': url, 'title': title, 'text': text, 'code_blocks': code_bks}
+    except Exception as e:
+        return {'error': str(e), 'url': url, 'title': '', 'text': '', 'code_blocks': []}
+
+
 # ── Analyze image ──────────────────────────────────────────────────────────────
 
 @router.post('/api/analyze-image')
 async def analyze_image(
     body: AnalyzeImageRequest, request: Request,
-    role: AuthRole = Depends(require_role(AuthRole.MACHINE)),
+    role: AuthRole = Depends(require_role(AuthRole.MACHINE)),  # P19-SEC2-1: era fail-open
 ):
     """
     Vision AI — OpenRouter free VL models → Gemini fallback.
+    [S191] rimossi modelli deprecated.
     [S-GAP23] X-Internal-Token guard — protegge consumi LLM da abusi esterni.
     """
     import re as _re2, json as _json
@@ -78,6 +424,8 @@ async def analyze_image(
     }]
 
     last_err = None
+    tried    = []
+
     for (pname, api_key, base_url, model) in VISION_MATRIX:
         try:
             vclient = _OAI(api_key=api_key, base_url=base_url)
@@ -99,10 +447,17 @@ async def analyze_image(
                         'provider':      f'{pname}/{model}',
                         'filename':      body.filename,
                     }
-                except _json.JSONDecodeError:
-                    continue
-        except Exception as e:
-            last_err = str(e)
+                except _json.JSONDecodeError as _exc:
+                    _logger.debug("[search] silenced %s", type(_exc).__name__)  # noqa: BLE001
+            if raw:
+                return {
+                    'ok': True, 'description': raw[:300],  # S584: 250→300
+                    'tags': [], 'objects': [], 'text_in_image': None, 'mood': None,
+                    'provider': f'{pname}/{model}', 'filename': body.filename,
+                }
+        except Exception as exc:
+            last_err = str(exc)
+            tried.append(f'{pname}/{model}: {str(exc)[:200]}')  # S608: 60→200
             continue
 
-    return {'ok': False, 'error': last_err or 'Analisi fallita su tutti i provider.'}
+    return {'ok': False, 'error': f'Tutti i provider vision falliti. Ultimo: {last_err}', 'tried': tried[:5]}

api/web.py

@@ -18,6 +18,14 @@ class SearchReq(BaseModel):
 class FetchReq(BaseModel):
   url:str; query:Optional[str]=None; max_chars:Optional[int]=5000
 
+class ProxyFetchReq(BaseModel):
+  """V7-2 GAP-2b: server-side proxy fetch — zero CORS, nessun WAF client-side."""
+  url:       str
+  method:    Optional[str]   = "GET"
+  headers:   Optional[dict]  = None
+  body:      Optional[str]   = None
+  max_chars: Optional[int]   = 50_000
+
 @router.post("/search")
 async def search(req:SearchReq, role:AuthRole=Depends(require_role(AuthRole.MACHINE))):  # GAP-1-fix
   raw=await web_search(req.query,req.focus or "general",req.max_results or 6)
@@ -33,4 +41,35 @@ async def fetch(req:FetchReq, role:AuthRole=Depends(require_role(AuthRole.MACHIN
       raw["content"]=cleaned["content"]; raw["words"]=cleaned["words"]
   return raw
 
-# Endpoint /proxy-fetch rimosso per conformità alle policy Hugging Face (Reverse Proxy)
+@router.post("/proxy-fetch")
+async def proxy_fetch(req: ProxyFetchReq, role:AuthRole=Depends(require_role(AuthRole.MACHINE))):  # GAP-1-fix SSRF
+  """
+  V7-2 GAP-2b: server-side proxy fetch.
+
+  Il client iPhone Safari fallisce su ~40-60% dei fetch diretti (CORS, WAF, TLS fingerprint).
+  Questo endpoint esegue il fetch lato server (Railway) e lo ritorna al client:
+  - Zero CORS (server-to-server)
+  - Bypassa WAF client-side
+  - httpx con timeout 7s e max_chars 50K
+  - Response: {ok:bool, status:int, body:str, url:str} o {ok:false, error:str, url:str}
+
+  Cold-start Railway free tier: fino a 5s → il client usa timeout 7s.
+  """
+  try:
+    import httpx
+    max_chars = min(req.max_chars or 50_000, 100_000)  # hard cap 100K
+    hdrs = req.headers or {}
+    # Imposta un User-Agent realistico per evitare blocchi anti-bot
+    if "user-agent" not in {k.lower() for k in hdrs}:
+      hdrs = {**hdrs, "User-Agent": "Mozilla/5.0 (compatible; AgentBot/1.0)"}
+    async with httpx.AsyncClient(timeout=7.0, follow_redirects=True) as client:
+      resp = await client.request(
+        method  = (req.method or "GET").upper(),
+        url     = req.url,
+        headers = hdrs,
+        content = req.body.encode("utf-8") if req.body else None,
+      )
+    body = resp.text[:max_chars]
+    return {"ok": True, "status": resp.status_code, "body": body, "url": req.url}
+  except Exception as e:
+    return {"ok": False, "error": str(e)[:300], "url": req.url}