api/web.py

"""api/web.py — Web search + fetch endpoints
V7-2 GAP-2b: aggiunto /proxy-fetch — server-side fetch zero CORS via Railway.
"""
from fastapi import APIRouter
from pydantic import BaseModel
from typing import Optional
from tools.web_search import web_search
from tools.web_fetch import fetch_page
from tools.ranking import rank_results, format_for_llm
from tools.content_cleaner import clean_and_structure

router=APIRouter(prefix="/web",tags=["web"])

class SearchReq(BaseModel):
  query:str; focus:Optional[str]="general"; max_results:Optional[int]=6; format:Optional[str]="json"

class FetchReq(BaseModel):
  url:str; query:Optional[str]=None; max_chars:Optional[int]=5000

class ProxyFetchReq(BaseModel):
  """V7-2 GAP-2b: server-side proxy fetch — zero CORS, nessun WAF client-side."""
  url:       str
  method:    Optional[str]   = "GET"
  headers:   Optional[dict]  = None
  body:      Optional[str]   = None
  max_chars: Optional[int]   = 50_000

@router.post("/search")
async def search(req:SearchReq):
  raw=await web_search(req.query,req.focus or "general",req.max_results or 6)
  ranked=rank_results(raw.get("results",[]),req.query,top_k=req.max_results or 6)
  if req.format=="text": return {"text":format_for_llm(ranked,req.query),"query":req.query}
  return {**raw,"results":ranked}

@router.post("/fetch")
async def fetch(req:FetchReq):
  raw=await fetch_page(req.url,max_chars=req.max_chars or 5000)
  if req.query and raw.get("content"):
      cleaned=clean_and_structure(raw["content"],req.query)
      raw["content"]=cleaned["content"]; raw["words"]=cleaned["words"]
  return raw

@router.post("/proxy-fetch")
async def proxy_fetch(req: ProxyFetchReq):
  """
  V7-2 GAP-2b: server-side proxy fetch.

  Il client iPhone Safari fallisce su ~40-60% dei fetch diretti (CORS, WAF, TLS fingerprint).
  Questo endpoint esegue il fetch lato server (Railway) e lo ritorna al client:
  - Zero CORS (server-to-server)
  - Bypassa WAF client-side
  - httpx con timeout 7s e max_chars 50K
  - Response: {ok:bool, status:int, body:str, url:str} o {ok:false, error:str, url:str}

  Cold-start Railway free tier: fino a 5s → il client usa timeout 7s.
  """
  try:
    import httpx
    max_chars = min(req.max_chars or 50_000, 100_000)  # hard cap 100K
    hdrs = req.headers or {}
    # Imposta un User-Agent realistico per evitare blocchi anti-bot
    if "user-agent" not in {k.lower() for k in hdrs}:
      hdrs = {**hdrs, "User-Agent": "Mozilla/5.0 (compatible; AgentBot/1.0)"}
    async with httpx.AsyncClient(timeout=7.0, follow_redirects=True) as client:
      resp = await client.request(
        method  = (req.method or "GET").upper(),
        url     = req.url,
        headers = hdrs,
        content = req.body.encode("utf-8") if req.body else None,
      )
    body = resp.text[:max_chars]
    return {"ok": True, "status": resp.status_code, "body": body, "url": req.url}
  except Exception as e:
    return {"ok": False, "error": str(e)[:300], "url": req.url}