""" ai_client.py — Cloud AI Client for free remote deployments Backend-first model router for mobile/no-PC usage. It prefers free or low-cost OpenAI-compatible providers when configured and falls back across providers before returning an error. The iPhone remains only a control surface; all model calls run from the deployed backend/HF Space. v2 — timeout + asyncio.to_thread per ogni chiamata sync (non blocca l'event loop), fallback immediato al provider successivo su timeout o errore. S752-C: _ProviderHealth — corruption rate tracking + auto-cooldown per provider. """ from __future__ import annotations import asyncio import os import time as _time_mod from collections import deque from dataclasses import dataclass, field from typing import AsyncIterator, Optional from openai import OpenAI import logging _logger = logging.getLogger("agente_ai") # S-BUGFIX # Gap 2.3: Distributed LLM cache — importato con guard per graceful degradation try: from api.llm_cache import ( cache_key as _lc_key, should_cache as _lc_should, get_cached as _lc_get, set_cached as _lc_set, ) _LLM_CACHE_AVAILABLE = True except ImportError: _LLM_CACHE_AVAILABLE = False def _lc_should(*_a, **_kw): return False # type: ignore[misc] def _lc_key(*_a, **_kw): return "" # type: ignore[misc] async def _lc_get(*_a, **_kw): return None # type: ignore[misc] async def _lc_set(*_a, **_kw): pass # type: ignore[misc] # Timeout per provider — abbassabile via env per reti lente PROVIDER_TIMEOUT: float = float(os.getenv("PROVIDER_TIMEOUT", "15")) STREAM_TIMEOUT: float = float(os.getenv("STREAM_TIMEOUT", "45")) # S391: Cerebras free tier ctx limit — ~8K token * ~4 chars/token. CEREBRAS_CTX_LIMIT_CHARS: int = int(os.getenv("CEREBRAS_CTX_LIMIT_CHARS", "32000")) # S416-Fix5: limiti reali output_token per modello → evita truncation silenziosa. _MODEL_OUTPUT_LIMITS: dict[str, int] = { "llama3-8b-8192": 8192, "gemini-2.5-flash": 65536, "gemini-2.5-flash-lite": 32768, "gemini-2.5-flash-preview-04-17": 65536, "deepseek-r1:free": 16000, "deepseek/deepseek-r1:free": 16000, "qwen/qwen3-30b-a3b:free": 8192, "qwen/qwen3-235b-a22b:free": 16384, "meta-llama/llama-4-scout:free": 8192, "meta-llama/llama-4-maverick:free": 8192, "meta-llama/llama-4-scout": 8192, "meta-llama/llama-4-maverick": 8192, "llama3.1-8b": 2048, "gemini-2.5-pro": 65536, "cerebras/gpt-oss-120b": 8192, "gpt-oss-120b": 32768, # S-LOOP1: Cerebras reasoning model (thinking tokens → alto consumo, serve spazio) "qwen/qwen3.6-27b": 32768, # S-LOOP1: Groq Qwen3.6 27B (sostituisce qwen3-32b deprecato) "Meta-Llama-3.3-70B-Instruct": 8192, # S-LOOP1: SambaNova Llama 3.3 70B "openai/gpt-oss-20b:free": 4096, # S-LOOP1: OpenRouter gpt-oss-20b free # S-2026-06: nuovi modelli OpenRouter free verificati live 2026-06-14 "nvidia/nemotron-3-ultra-550b-a55b:free": 16384, # 1M ctx, ARCHITECT role "nvidia/nemotron-3-super-120b-a12b:free": 16384, # 1M ctx, 120B "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free": 8192, "nvidia/nemotron-3-nano-30b-a3b:free": 8192, "qwen/qwen3-coder:free": 32768, # 1M ctx, CODER role — 480B "qwen/qwen3-next-80b-a3b-instruct:free": 16384, # 262K ctx "poolside/laguna-m.1:free": 16384, # 262K ctx, coding "poolside/laguna-xs.2:free": 8192, # 262K ctx, lightweight "nex-agi/nex-n2-pro:free": 16384, # 262K ctx "google/gemma-4-31b-it:free": 16384, # 262K ctx "google/gemma-4-26b-a4b-it:free": 16384, # 262K ctx "openai/gpt-oss-120b:free": 16384, # 131K ctx "meta-llama/llama-3.3-70b-instruct:free": 16384, # 131K ctx "meta-llama/llama-3.2-3b-instruct:free": 4096, # SambaNova — modelli verificati live 2026-06-14 "DeepSeek-V3.1": 32768, # 131K ctx "DeepSeek-V3.2": 8192, # 32K ctx "MiniMax-M2.7": 32768, # 196K ctx! "gemma-4-31B-it": 16384, # 131K ctx # Gemini 3.x — disponibili live 2026-06-14 (flash = free) "gemini-3-flash-preview": 32768, "gemini-3.5-flash": 32768, "gemini-3.1-flash-lite": 16384, "gemini-3.1-flash-lite-preview": 16384, # Groq 2026 — verificati live "openai/gpt-oss-120b": 8192, # Groq GPT-OSS 120B "openai/gpt-oss-20b": 4096, # Groq GPT-OSS 20B # Groq 2026 — live verificato 2026-06-14 /v1/models "groq/compound": 8192, # Groq Compound (web + reasoning) "groq/compound-mini": 4096, # Groq Compound Mini "openai/gpt-oss-safeguard-20b": 4096, # Groq GPT-OSS Safeguard 20B # Cerebras — reasoning model con thinking tokens "zai-glm-4.7": 4096, # Cerebras GLM 4.7 } def _safe_max_tokens(requested: int, model_id: str) -> int: """S416-Fix5: cappa max_tokens al limite output reale del modello (margine 512 token).""" limit = _MODEL_OUTPUT_LIMITS.get(model_id, requested) return min(requested, max(512, limit - 512)) # ── S752-C: Provider Health — corruption rate tracking + auto-cooldown ───────── # Ogni provider ha una finestra rolling di 10 esiti ('ok'|'corrupt'|'timeout'). # Se corruption_rate >= 40% su >= 5 campioni → cooldown automatico 60s. # Il cooldown si azzera da solo allo scadere — nessun intervento manuale necessario. _CORRUPT_RATE_THRESHOLD: float = 0.40 # soglia per attivare cooldown _COOLDOWN_SECONDS: float = 60.0 # durata cooldown in secondi _CORRUPT_MIN_LEN: int = 5 # risposta < 5 chars = corrupted @dataclass class _ProviderHealth: outcomes: "deque[str]" = field(default_factory=lambda: deque(maxlen=10)) cooldown_until: float = 0.0 call_timestamps: "deque[float]" = field(default_factory=lambda: deque(maxlen=120)) # P19-B1: RPM sliding window _PROVIDER_HEALTH: dict[str, _ProviderHealth] = {} # P19-B1: Limiti RPM per provider — sliding window 60s. # Fonte: free tier ufficiali 2026. Override via env var (RPM_GEMINI, RPM_GROQ, ecc.) # 0 = nessun limite (provider a pagamento o senza cap documentato). _PROVIDER_RPM_LIMITS: dict[str, int] = { "gemini": int(os.getenv("RPM_GEMINI", "15")), # Google AI free: 15 RPM "groq": int(os.getenv("RPM_GROQ", "30")), # Groq free: 30 RPM/model "groq-fast": int(os.getenv("RPM_GROQ_FAST", "30")), "groq-qwen": int(os.getenv("RPM_GROQ_QWEN", "30")), "groq-scout": int(os.getenv("RPM_GROQ_SCOUT", "30")), "groq-compound": int(os.getenv("RPM_GROQ_COMPOUND", "20")), "groq-b": int(os.getenv("RPM_GROQ_B", "30")), # Groq key B slot 1 "groq-fast-b": int(os.getenv("RPM_GROQ_FAST_B", "30")), # Groq key B slot 2 "groq-c": int(os.getenv("RPM_GROQ_C", "30")), # Groq key C slot 1 "groq-fast-c": int(os.getenv("RPM_GROQ_FAST_C", "30")), # Groq key C slot 2 "cerebras": int(os.getenv("RPM_CEREBRAS", "30")), "cerebras-b": int(os.getenv("RPM_CEREBRAS_B", "30")), # Cerebras key B "sambanova": int(os.getenv("RPM_SAMBANOVA", "60")), "sambanova-b": int(os.getenv("RPM_SAMBANOVA_B", "60")), # SambaNova key B "nvidia": int(os.getenv("RPM_NVIDIA", "30")), # NVIDIA NIM free tier "nvidia-b": int(os.getenv("RPM_NVIDIA_B", "30")), # NVIDIA NIM key B "openrouter": int(os.getenv("RPM_OPENROUTER", "20")), # free tier "huggingface": int(os.getenv("RPM_HF", "10")), # HF router free: ~10 RPM "cloudflare": int(os.getenv("RPM_CLOUDFLARE", "300")), # CF Workers AI "cloudflare-b": int(os.getenv("RPM_CLOUDFLARE_B", "300")), # CF Workers AI Account B # openai_compatible: 0 = nessun limite (account a pagamento) } def _record_provider_outcome(name: str, outcome: str) -> None: """ Registra esito ('ok' | 'corrupt' | 'timeout') e attiva cooldown automatico se la corruption rate supera la soglia su almeno 5 campioni. Mai rilancia eccezioni. """ try: rec = _PROVIDER_HEALTH.setdefault(name, _ProviderHealth()) rec.outcomes.append(outcome) if len(rec.outcomes) >= 5: n_corrupt = sum(1 for o in rec.outcomes if o == 'corrupt') rate = n_corrupt / len(rec.outcomes) if rate >= _CORRUPT_RATE_THRESHOLD and rec.cooldown_until < _time_mod.monotonic(): rec.cooldown_until = _time_mod.monotonic() + _COOLDOWN_SECONDS import logging as _log_cd _log_cd.getLogger("agente_ai").warning( "S752 provider '%s' cooldown %ds — corruption_rate=%.0f%%", name, int(_COOLDOWN_SECONDS), rate * 100, ) except Exception: pass # health tracking mai blocca il path principale def _provider_in_cooldown(name: str) -> bool: """True se il provider è in cooldown e il tempo non è ancora scaduto.""" rec = _PROVIDER_HEALTH.get(name) return rec is not None and rec.cooldown_until > _time_mod.monotonic() def _classify_llm_response(text: str) -> str: """ Classifica la risposta come 'ok' | 'corrupt'. Corrupt = vuota, troppo corta, o raw API error JSON trapelato come testo. """ if not text or len(text.strip()) < _CORRUPT_MIN_LEN: return 'corrupt' s = text.strip() # Raw API error JSON passato come testo dal provider if s.startswith('{"error"') or s.startswith('{"message"'): return 'corrupt' # choices[]/delta leak — streaming non gestito correttamente if '"choices"' in s and '"delta"' in s: return 'corrupt' return 'ok' def get_provider_health_snapshot() -> dict[str, dict]: """ Ritorna snapshot dello stato di salute di tutti i provider. Usato da /api/ai/health per esporre dati di osservabilità. """ now = _time_mod.monotonic() result: dict[str, dict] = {} for name, rec in _PROVIDER_HEALTH.items(): total = len(rec.outcomes) corrupt = sum(1 for o in rec.outcomes if o == 'corrupt') timeout = sum(1 for o in rec.outcomes if o == 'timeout') # P19-B1: snapshot finestra RPM corrente (ultimi 60s) _rpm_win = sum(1 for ts in rec.call_timestamps if (now - ts) <= 60.0) _rpm_lim = _PROVIDER_RPM_LIMITS.get(name, 0) result[name] = { "total_samples": total, "corrupt_count": corrupt, "timeout_count": timeout, "corruption_rate": round(corrupt / total, 3) if total > 0 else 0.0, "in_cooldown": rec.cooldown_until > now, "cooldown_remaining_s": max(0.0, round(rec.cooldown_until - now, 1)), "rpm_window_count": _rpm_win, "rpm_limit": _rpm_lim, "rpm_saturated": bool(_rpm_lim) and _rpm_win >= _rpm_lim, } return result # ── P19-B1: RPM sliding-window check/record ───────────────────────────────────── def _rpm_allowed(name: str) -> bool: """ True se il provider ha ancora capacità nella finestra 60s. Se il limite è 0 → sempre True. Fail-open: eccezione → True. """ try: limit = _PROVIDER_RPM_LIMITS.get(name, 0) if limit == 0: return True rec = _PROVIDER_HEALTH.get(name) if rec is None: return True now = _time_mod.monotonic() # Rimuovi timestamps fuori finestra while rec.call_timestamps and (now - rec.call_timestamps[0]) > 60.0: rec.call_timestamps.popleft() allowed = len(rec.call_timestamps) < limit if not allowed: _logger.warning( "[P19-B1] provider '%s' al limite %d RPM — skip immediato verso provider successivo", name, limit, ) return allowed except Exception: return True # fail-open: mai blocca il path principale def _rpm_record(name: str) -> None: """Registra timestamp chiamata nella sliding window. Fail-safe.""" try: rec = _PROVIDER_HEALTH.setdefault(name, _ProviderHealth()) rec.call_timestamps.append(_time_mod.monotonic()) except Exception: pass @dataclass(frozen=True) class ProviderConfig: name: str api_key: str base_url: str default_model: str embedding_model: Optional[str] = None class AIClient: def __init__(self) -> None: self.providers = self._discover_providers() self._client_cache: dict[str, OpenAI] = {} self.provider_name = self.providers[0].name if self.providers else "unconfigured" self.default_model = self.providers[0].default_model if self.providers else os.getenv("LLM_MODEL", "openrouter/auto") self.client = self._client_for(self.providers[0]) if self.providers else None # ── Discovery ──────────────────────────────────────────────────────────── def _discover_providers(self) -> list[ProviderConfig]: """ S387 — Provider chain aggiornata al 2026-06-03 con i modelli disponibili su ogni piattaforma. GROQ (3 slot, bucket TPM separati per modello — stessa chiave): groq → openai/gpt-oss-120b (benchmark #1: 100%, 290ms, ctx 131K) groq-qwen → qwen/qwen3.6-27b (Qwen3 32B, ~14K TPM, ctx 131K — ragionamento/math) groq-fast → openai/gpt-oss-20b (~100K TPM, ctx 131K — emergenza rate-limit) GEMINI (1 slot): gemini → gemini-2.5-flash-lite (S435: 2.0 spento, 2.5-flash-lite ✓) OPENROUTER (1 slot, modello :free aggiornato): openrouter → openai/gpt-oss-20b:free (S435: nemotron rate-limit esaurito) HUGGINGFACE (opzionale, disabilitato di default: 402 free tier esaurito): huggingface → Qwen/Qwen2.5-Coder-32B-Instruct OPENAI-COMPAT (opzionale, fallback finale): openai_compatible → gpt-4o-mini S752-C: Logica bucket TPM Groq: i rate limit sono per-modello su Groq → anche se groq (Scout) è a 429, groq-qwen (Qwen3) e groq-fast (8b) rispondono. Provider in cooldown S752 vengono skippati nel sequential (non nella race). """ providers: list[ProviderConfig] = [] groq_key = os.getenv("GROQ_API_KEY") # ── GROQ SLOT 1: Llama 4 Scout — modello primario 2026 ─────────────── if groq_key: providers.append(ProviderConfig( name="groq", api_key=groq_key, base_url="https://api.groq.com/openai/v1", default_model=os.getenv("GROQ_MODEL", "openai/gpt-oss-120b"), )) # ── GROQ SLOT 2: Llama 3.1 8B Instant — fast race partner ───────────── if groq_key and not os.getenv("DISABLE_GROQ_FAST"): providers.append(ProviderConfig( name="groq-fast", api_key=groq_key, base_url="https://api.groq.com/openai/v1", default_model=os.getenv("GROQ_FAST_MODEL", "openai/gpt-oss-20b"), )) # ── GROQ SLOT 3: Qwen3-32B — fallback qualità ragionamento/math ────── if groq_key and not os.getenv("DISABLE_GROQ_QWEN"): providers.append(ProviderConfig( name="groq-qwen", api_key=groq_key, base_url="https://api.groq.com/openai/v1", default_model=os.getenv("GROQ_QWEN_MODEL", "qwen/qwen3.6-27b"), )) # ── GROQ SLOT 4: Llama 4 Scout — 10M ctx, emergenza rate-limit versatile ───── if groq_key and not os.getenv("DISABLE_GROQ_SCOUT"): providers.append(ProviderConfig( name="groq-scout", api_key=groq_key, base_url="https://api.groq.com/openai/v1", default_model=os.getenv("GROQ_SCOUT_MODEL", "openai/gpt-oss-120b"), )) # ── GROQ SLOT 5: Compound — web search + reasoning integrati ───────────────── if groq_key and not os.getenv("DISABLE_GROQ_COMPOUND"): providers.append(ProviderConfig( name="groq-compound", api_key=groq_key, base_url="https://api.groq.com/openai/v1", default_model=os.getenv("GROQ_COMPOUND_MODEL", "groq/compound"), )) # ── GROQ KEY B — slot 1: llama-3.3-70b con chiave secondaria ───────────── # GAP-API-LB: seconda chiave Groq per raddoppiare il rate-limit giornaliero. # Attivato da GROQ_API_KEY_B env var. DISABLE_GROQ_B per disabilitare. groq_key_b = os.getenv("GROQ_API_KEY_B") if groq_key_b and not os.getenv("DISABLE_GROQ_B"): providers.append(ProviderConfig( name="groq-b", api_key=groq_key_b, base_url="https://api.groq.com/openai/v1", default_model=os.getenv("GROQ_MODEL", "openai/gpt-oss-120b"), )) # ── GROQ KEY B — slot 2: openai/gpt-oss-20b con chiave secondaria ──── if groq_key_b and not os.getenv("DISABLE_GROQ_B"): providers.append(ProviderConfig( name="groq-fast-b", api_key=groq_key_b, base_url="https://api.groq.com/openai/v1", default_model=os.getenv("GROQ_FAST_MODEL", "openai/gpt-oss-20b"), )) # ── GROQ KEY C — slot 1: terza chiave, raddoppia ancora il budget giornaliero ─ groq_key_c = os.getenv("GROQ_API_KEY_C") if groq_key_c and not os.getenv("DISABLE_GROQ_C"): providers.append(ProviderConfig( name="groq-c", api_key=groq_key_c, base_url="https://api.groq.com/openai/v1", default_model=os.getenv("GROQ_MODEL", "openai/gpt-oss-120b"), )) # ── GROQ KEY C — slot 2: fast model con chiave C ───────────────────────── if groq_key_c and not os.getenv("DISABLE_GROQ_C"): providers.append(ProviderConfig( name="groq-fast-c", api_key=groq_key_c, base_url="https://api.groq.com/openai/v1", default_model=os.getenv("GROQ_FAST_MODEL", "openai/gpt-oss-20b"), )) # ── CEREBRAS: gpt-oss-120b — reasoning model con "reasoning" field ────────── # IMPORTANTE: gpt-oss-120b usa thinking tokens interni → max_tokens≥500 richiesto # Con max_tokens<200 il content è "" (reasoning consuma tutti i token) cerebras_key = os.getenv("CEREBRAS_API_KEY") if cerebras_key: providers.append(ProviderConfig( name="cerebras", api_key=cerebras_key, base_url="https://api.cerebras.ai/v1", default_model=os.getenv("CEREBRAS_MODEL", "gpt-oss-120b"), )) # ── CEREBRAS KEY B — seconda chiave, raddoppia quota oraria ────────────── cerebras_key_b = os.getenv("CEREBRAS_API_KEY_B") if cerebras_key_b and not os.getenv("DISABLE_CEREBRAS_B"): providers.append(ProviderConfig( name="cerebras-b", api_key=cerebras_key_b, base_url="https://api.cerebras.ai/v1", default_model=os.getenv("CEREBRAS_MODEL", "gpt-oss-120b"), )) # ── SAMBANOVA: Llama 3.3 70B — 2200+ tok/s, 60 req/min, no daily cap ──── sn_key = os.getenv("SAMBANOVA_API_KEY", "") if sn_key: providers.append(ProviderConfig( name="sambanova", api_key=sn_key, base_url="https://api.sambanova.ai/v1", default_model=os.getenv("SAMBANOVA_MODEL", "DeepSeek-V3.1"), )) # ── SAMBANOVA KEY B — seconda chiave, raddoppia 60 req/min ─────────────── sn_key_b = os.getenv("SAMBANOVA_API_KEY_B", "") if sn_key_b and not os.getenv("DISABLE_SAMBANOVA_B"): providers.append(ProviderConfig( name="sambanova-b", api_key=sn_key_b, base_url="https://api.sambanova.ai/v1", default_model=os.getenv("SAMBANOVA_MODEL", "DeepSeek-V3.1"), )) # ── NVIDIA NIM: nemotron-3-ultra-550b — 1M ctx, 550B params, gratuito ────── # endpoint OpenAI-compatible: integrate.api.nvidia.com/v1 # modello primario: nvidia/nemotron-3-ultra-550b-a55b (ARCHITECT, 16K output) nvidia_key = os.getenv("NVIDIA_API_KEY") if nvidia_key: providers.append(ProviderConfig( name="nvidia", api_key=nvidia_key, base_url="https://integrate.api.nvidia.com/v1", default_model=os.getenv("NVIDIA_MODEL", "nvidia/nemotron-3-super-120b-a12b"), )) # ── NVIDIA NIM key B — slot secondario (raddoppia rate-limit gratuito NIM) ──────────── # Modello B: meta/llama-3.3-70b-instruct (veloce, stabile) — complementare a nvidia (120B) nvidia_key_b = os.getenv("NVIDIA_API_KEY_B") if nvidia_key_b and not os.getenv("DISABLE_NVIDIA_B"): providers.append(ProviderConfig( name="nvidia-b", api_key=nvidia_key_b, base_url="https://integrate.api.nvidia.com/v1", default_model=os.getenv("NVIDIA_B_MODEL", "meta/llama-3.3-70b-instruct"), )) # ── GEMINI: gemini-2.5-flash-lite — S435: 2.0-flash-lite spento dal 1-giu-2026 ──────── gemini_key = os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") if gemini_key: providers.append(ProviderConfig( name="gemini", api_key=gemini_key, base_url="https://generativelanguage.googleapis.com/v1beta/openai/", default_model=os.getenv("GEMINI_MODEL", "gemini-2.5-flash"), )) # GEMINI B-E (Failover A-E) for suffix in ['B', 'C', 'E']: g_key = os.getenv(f'GEMINI_API_KEY_{suffix}') if g_key: providers.append(ProviderConfig( name=f'gemini-{suffix.lower()}', api_key=g_key, base_url='https://generativelanguage.googleapis.com/v1beta/openai/', default_model=os.getenv('GEMINI_MODEL', 'gemini-2.5-flash'), )) # ── OPENROUTER: gpt-oss-20b:free — S435: nemotron rate-limit esaurito ───────── openrouter_key = os.getenv("OPENROUTER_API_KEY") if openrouter_key: providers.append(ProviderConfig( name="openrouter", api_key=openrouter_key, base_url="https://openrouter.ai/api/v1", default_model=os.getenv("OPENROUTER_MODEL", "openai/gpt-oss-120b:free"), )) # OPENROUTER B-E (Failover A-E) for suffix in ['B', 'C', 'D', 'E']: or_key = os.getenv(f'OPENROUTER_API_KEY_{suffix}') if or_key: providers.append(ProviderConfig( name=f'openrouter-{suffix.lower()}', api_key=or_key, base_url='https://openrouter.ai/api/v1', default_model=os.getenv('OPENROUTER_MODEL', 'openai/gpt-oss-120b:free'), )) # ── HUGGINGFACE: Qwen2.5-Coder-32B ──────────────────────────────────── hf_key = os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACE_API_KEY") or os.getenv("HUGGINGFACE_TOKEN") # RF-3: rimosso gate ENABLE_HF_PROVIDER — si attiva automaticamente se HF_TOKEN presente if hf_key: providers.append(ProviderConfig( name="huggingface", api_key=hf_key, base_url=os.getenv("HF_OPENAI_BASE_URL", "https://router.huggingface.co/v1"), default_model=os.getenv("HF_MODEL", "Qwen/Qwen2.5-Coder-32B-Instruct"), )) # 5. OpenAI compat last openai_key = os.getenv("OPENAI_API_KEY") if openai_key: providers.append(ProviderConfig( name="openai_compatible", api_key=openai_key, base_url=os.getenv("OPENAI_API_BASE", "https://api.openai.com/v1"), default_model=os.getenv("OPENAI_MODEL", os.getenv("LLM_MODEL", "gpt-4o-mini")), embedding_model=os.getenv("EMBEDDING_MODEL", "text-embedding-3-small"), )) # ── CLOUDFLARE WORKERS AI: llama-3.3-70b-instruct-fp8-fast ───────────────── # RF-3: attivo se CF_API_TOKEN + CF_ACCOUNT_ID entrambi presenti. # Endpoint OpenAI-compatible: /accounts/{id}/ai/v1 # Free: 10K req/giorno, nessun account separato — usa il token Cloudflare. cf_token = os.getenv("CF_API_TOKEN") cf_account = os.getenv("CF_ACCOUNT_ID") if cf_token and cf_account and not os.getenv("DISABLE_CF_PROVIDER"): providers.append(ProviderConfig( name="cloudflare", api_key=cf_token, base_url=f"https://api.cloudflare.com/client/v4/accounts/{cf_account}/ai/v1", default_model=os.getenv("CF_AI_MODEL", "@cf/meta/llama-3.3-70b-instruct-fp8-fast"), )) # ── CLOUDFLARE WORKERS AI (Account B): Fallback secondario ───────────────── # S408-DUAL: attivo se CF_API_TOKEN_B + CF_ACCOUNT_ID_B presenti. # Raddoppia la quota giornaliera a 20k request totali. cf_token_b = os.getenv("CF_API_TOKEN_B") cf_account_b = os.getenv("CF_ACCOUNT_ID_B") if cf_token_b and cf_account_b and not os.getenv("DISABLE_CF_B"): providers.append(ProviderConfig( name="cloudflare-b", api_key=cf_token_b, base_url=f"https://api.cloudflare.com/client/v4/accounts/{cf_account_b}/ai/v1", default_model=os.getenv("CF_AI_MODEL_B", "@cf/meta/llama-3.3-70b-instruct-fp8-fast"), )) return providers # ── Client factory ──────────────────────────────────────────────────────── def _client_for(self, provider: ProviderConfig) -> OpenAI: """B11: Cache client per provider — evita re-istanziazione OpenAI() a ogni LLM call.""" if provider.name not in self._client_cache: self._client_cache[provider.name] = OpenAI( api_key=provider.api_key, base_url=provider.base_url, timeout=PROVIDER_TIMEOUT, max_retries=0, ) return self._client_cache[provider.name] def _model_for(self, provider: ProviderConfig, requested: Optional[str]) -> str: """ S195-Robust: resolve correct model ID for this provider. 1. Strips litellm provider prefix (e.g. 'groq/openai/gpt-oss-20b' -> 'openai/gpt-oss-20b') 2. Blocks Groq-only model IDs from being sent to other providers. """ if not requested: return provider.default_model _KNOWN_PREFIXES = { "groq", "openrouter", "openai", "anthropic", "cohere", "mistral", "huggingface", "gemini", "azure", "bedrock", "vertex_ai", } _segs = requested.split("/", 1) bare = _segs[1] if len(_segs) > 1 and _segs[0].lower() in _KNOWN_PREFIXES else requested _GROQ_ONLY: set[str] = { "openai/gpt-oss-20b", "openai/gpt-oss-120b", "openai/gpt-oss-120b", "qwen/qwen3.6-27b", "groq/compound", "groq/compound-mini", "openai/gpt-oss-20b", "openai/gpt-oss-120b", "openai/gpt-4o-mini", "groq/compound", "groq/compound-mini", "openai/gpt-oss-safeguard-20b", "allam-2-7b", "whisper-large-v3", "whisper-large-v3-turbo", } if not provider.name.startswith("groq") and bare in _GROQ_ONLY: return provider.default_model return bare # ── Qwen3 /no_think helper ─────────────────────────────────────────────── @staticmethod def _inject_no_think(messages: list) -> list: """ S436: Qwen3 thinking disable via system message prefix /no_think. Ritorna una NUOVA lista — non muta messages originale. """ NO_THINK = "/no_think" msgs = list(messages) for i, m in enumerate(msgs): if m.get("role") == "system": content = m.get("content") or "" if not content.startswith(NO_THINK): msgs[i] = {**m, "content": f"{NO_THINK}\n{content}"} return msgs msgs.insert(0, {"role": "system", "content": NO_THINK}) return msgs # ── Race-to-first helper ───────────────────────────────────────────────── async def _try_chat_once( self, provider: ProviderConfig, messages: list, model: Optional[str], temperature: float, max_tokens: int, timeout: float, ) -> str: """ S356: Tenta una singola chiamata a un provider con timeout fisso. S752-C: Registra esito ('ok'|'corrupt'|'timeout') nel health tracker. """ client = self._client_for(provider) if provider.name == "cerebras": _est = sum(len(str(m.get("content", "") or "")) for m in messages) if _est > CEREBRAS_CTX_LIMIT_CHARS: raise ValueError(f"cerebras: ctx troppo lungo ({_est} chars > {CEREBRAS_CTX_LIMIT_CHARS})") _model = model _msgs = self._inject_no_think(messages) if provider.name == "groq-qwen" else messages for attempt in range(2): try: _extra = {} if provider.name == "gemini": _extra = {"extra_body": {"thinking": {"type": "disabled"}}} _resolved_model = self._model_for(provider, _model) response = await asyncio.wait_for( asyncio.to_thread( client.chat.completions.create, model=_resolved_model, messages=_msgs, temperature=temperature, max_tokens=_safe_max_tokens(max_tokens, _resolved_model), stream=False, **_extra, ), timeout=timeout, ) result = response.choices[0].message.content or "" # P16-B4: cattura finish_reason per segnalare truncation al caller self._last_finish_reason: str = ( getattr(response.choices[0], "finish_reason", None) or "stop" ) # S752-C: registra esito nel health tracker _record_provider_outcome(provider.name, _classify_llm_response(result)) return result except asyncio.TimeoutError: _record_provider_outcome(provider.name, 'timeout') raise TimeoutError(f"{provider.name} timeout {timeout:.0f}s") except Exception as exc: exc_str = str(exc) is_bad_model = ( "not a valid model" in exc_str or ("400" in exc_str and "BadRequest" in exc_str) ) if is_bad_model and _model is not None and attempt == 0: _model = None continue raise raise RuntimeError(f"{provider.name}: tentativi esauriti") # ── Health ──────────────────────────────────────────────────────────────── async def health(self) -> dict: if not self.providers: return { "available": False, "provider": "none", "error": "No remote provider key configured. Set OPENROUTER_API_KEY, GEMINI_API_KEY, GROQ_API_KEY, HF_TOKEN or OPENAI_API_KEY.", "models": [], } checks: list[dict] = [] for provider in self.providers: client = self._client_for(provider) try: await asyncio.wait_for( asyncio.to_thread( client.chat.completions.create, model=provider.default_model, messages=[{"role": "user", "content": "ping"}], max_tokens=1, temperature=0, ), timeout=PROVIDER_TIMEOUT, ) checks.append({"provider": provider.name, "available": True, "model": provider.default_model}) return { "available": True, "provider": provider.name, "models": [p.default_model for p in self.providers], "default": provider.default_model, "checks": checks, # S752-C: include health snapshot nell'endpoint /api/ai/health "provider_health": get_provider_health_snapshot(), } except Exception as exc: checks.append({"provider": provider.name, "available": False, "model": provider.default_model, "error": str(exc)}) return { "available": False, "provider": "configured_but_unavailable", "models": [p.default_model for p in self.providers], "checks": checks, "provider_health": get_provider_health_snapshot(), } # ── Chat (non-stream) ───────────────────────────────────────────────────── async def chat( self, messages: list, *, model: Optional[str] = None, temperature: float = 0.7, max_tokens: int = 4096, timeout: Optional[float] = None, ) -> str: """ S356: Race-to-first sui primi 2 provider con timeout aggressivo (5s). Se entrambi falliscono, fallback sequenziale completo con timeout pieno. S752-C: nel sequential, skippa provider in cooldown (se almeno 1 altro non è in cooldown). Registra 'ok'/'corrupt'/'timeout' sul health tracker. """ per_provider_timeout = timeout or PROVIDER_TIMEOUT last_error: Exception | None = None # Gap 2.3: Distributed LLM cache — check prima di qualsiasi chiamata provider. # Solo chat() non-streaming, temperature ≤ 0.5, senza tool results. # get_cached() timeout 2s → miss trasparente se Upstash non risponde. _lc_ck: str | None = None if _LLM_CACHE_AVAILABLE and _lc_should(messages, temperature): _lc_ck = _lc_key(model or "", messages) _lc_cached = await _lc_get(_lc_ck) if _lc_cached is not None: return _lc_cached # ── Fase 1: Race-to-first sui primi 2 provider ─────────────────────── # Race NON controlla cooldown — è già fast (5s) e cancella il perdente. RACE_TIMEOUT = min(per_provider_timeout * 0.4, 5.0) _race_hard_fail: set[str] = set() if len(self.providers) >= 2: _race_n = min(2, len(self.providers)) # P19-B1: preferisce provider con capacità RPM disponibile _race_cands = [p for p in self.providers[:_race_n] if _rpm_allowed(p.name)] _race_providers = _race_cands if _race_cands else self.providers[:1] for _rp in _race_providers: _rpm_record(_rp.name) race_tasks = [ asyncio.create_task( self._try_chat_once(p, messages, model, temperature, max_tokens, RACE_TIMEOUT) ) for p in _race_providers ] done, pending = await asyncio.wait(race_tasks, return_when=asyncio.FIRST_COMPLETED) for t in pending: t.cancel() try: await t except (asyncio.CancelledError, Exception) as _exc: _logger.debug("[ai_client] silenced %s", type(_exc).__name__) # noqa: BLE001 for p, t in zip(_race_providers, race_tasks): if t.done() and not t.cancelled(): _exc = t.exception() if _exc is not None: _es = str(_exc) if "429" in _es or "402" in _es or "rate_limit" in _es.lower() or "depleted" in _es.lower(): _race_hard_fail.add(p.name) for t in sorted(done, key=lambda x: len(x.result()) if x.exception() is None else 0, reverse=True): exc = t.exception() if exc is None: result = t.result() if result and len(result.strip()) >= 30: if _lc_ck: asyncio.create_task(_lc_set(_lc_ck, result)) return result last_error = RuntimeError(f"risposta troppo corta ({len(result)} chars)") else: last_error = exc # ── Fase 2: Fallback sequenziale su tutti i provider ───────────────── # S752-C: skippa provider in cooldown — a meno che TUTTI siano in cooldown _all_in_cooldown = all(_provider_in_cooldown(p.name) for p in self.providers) # P19-B1: fallback se TUTTI sono al limite RPM → non skippiamo nessuno (fail-open) _all_at_rpm = all(not _rpm_allowed(p.name) for p in self.providers) for provider in self.providers: if provider.name in _race_hard_fail: last_error = RuntimeError(f"{provider.name}: rate-limit/no-credits (già fallito in race)") continue # S752-C: cooldown check — bypass se tutti i provider sono in cooldown if not _all_in_cooldown and _provider_in_cooldown(provider.name): last_error = RuntimeError(f"{provider.name}: in cooldown (corruption_rate >= 40%)") continue # P19-B1: RPM limit — skip immediato, nessuna HTTP call, nessun timeout wait if not _all_at_rpm and not _rpm_allowed(provider.name): last_error = RuntimeError( f"{provider.name}: RPM limit {_PROVIDER_RPM_LIMITS.get(provider.name, '?')}/min — attendi 60s" ) continue _rpm_record(provider.name) client = self._client_for(provider) _model = model _msgs = self._inject_no_think(messages) if provider.name == "groq-qwen" else messages for attempt in range(3): try: _extra: dict = {} if provider.name == "gemini": _extra = {"extra_body": {"thinking": {"type": "disabled"}}} _resolved_model_seq = self._model_for(provider, _model) response = await asyncio.wait_for( asyncio.to_thread( client.chat.completions.create, model=_resolved_model_seq, messages=_msgs, temperature=temperature, max_tokens=_safe_max_tokens(max_tokens, _resolved_model_seq), stream=False, **_extra, ), timeout=per_provider_timeout, ) result = response.choices[0].message.content or "" # S752-C: registra esito nel sequential path _record_provider_outcome(provider.name, _classify_llm_response(result)) if _lc_ck and result: asyncio.create_task(_lc_set(_lc_ck, result)) return result except asyncio.TimeoutError: _record_provider_outcome(provider.name, 'timeout') last_error = TimeoutError(f"{provider.name} non ha risposto entro {per_provider_timeout}s") break except Exception as exc: last_error = exc exc_str = str(exc) is_rate_limit = "429" in exc_str or "rate_limit" in exc_str.lower() is_no_credits = "402" in exc_str or "depleted" in exc_str.lower() is_bad_model = ( "not a valid model" in exc_str or ("400" in exc_str and "BadRequest" in exc_str) ) if is_no_credits: _record_provider_outcome(provider.name, "timeout") # 402/no-credits break if is_bad_model and _model is not None and attempt == 0: _model = None continue if is_rate_limit: _record_provider_outcome(provider.name, "timeout") # S-LOOP4: registra rate-limit (era dead code dopo break) break # S-LOOP1: 429/rate-limit -> passa subito al provider successivo (no sleep+retry) _record_provider_outcome(provider.name, "corrupt") # errore generico non classificato break # GAP-FALLBACK-GRACEFUL: Se TUTTI i provider falliscono, restituisci un messaggio utile # invece di crashare. Questo permette al frontend di mostrare un errore chiaro. _logger.error("[ai_client] All providers exhausted: %s", last_error) raise RuntimeError( f"🔴 Tutti i provider AI sono temporaneamente non disponibili. " f"Ultimo errore: {str(last_error)[:100]}. " f"Riprova tra qualche minuto o contatta il supporto." ) # ── Stream chat ─────────────────────────────────────────────────────────── async def stream_chat( self, messages: list, *, model: Optional[str] = None, temperature: float = 0.7, max_tokens: int = 4096, ) -> AsyncIterator[str]: """ Streaming con fallback automatico tra provider. S752-C: skippa provider in cooldown (se almeno 1 altro non è in cooldown). """ last_error: Exception | None = None _all_in_cooldown = all(_provider_in_cooldown(p.name) for p in self.providers) _all_at_rpm_s = all(not _rpm_allowed(p.name) for p in self.providers) # P19-B1 for provider in self.providers: # S752-C: cooldown check nel path streaming if not _all_in_cooldown and _provider_in_cooldown(provider.name): last_error = RuntimeError(f"{provider.name}: in cooldown") continue # P19-B1: RPM limit check — skip immediato nel path streaming if not _all_at_rpm_s and not _rpm_allowed(provider.name): last_error = RuntimeError( f"{provider.name}: RPM limit {_PROVIDER_RPM_LIMITS.get(provider.name, '?')}/min" ) continue _rpm_record(provider.name) client = self._client_for(provider) if provider.name == "cerebras": _est = sum(len(str(m.get("content", "") or "")) for m in messages) if _est > CEREBRAS_CTX_LIMIT_CHARS: last_error = ValueError(f"cerebras: ctx troppo lungo ({_est} chars)") continue try: q: asyncio.Queue[str | None] = asyncio.Queue() _loop = asyncio.get_running_loop() _msgs = self._inject_no_think(messages) if provider.name == "groq-qwen" else messages _stream_model = self._model_for(provider, model) _stream_max_tokens = _safe_max_tokens(max_tokens, _stream_model) _finish_reason_holder: list[str] = ['stop'] # P16-B4: thread→coroutine handoff def _stream_to_queue() -> None: try: _stream_extra: dict = {} if provider.name == "gemini": _stream_extra = {"extra_body": {"thinking": {"type": "disabled"}}} stream = client.chat.completions.create( model=_stream_model, messages=_msgs, temperature=temperature, max_tokens=_stream_max_tokens, stream=True, **_stream_extra, ) for chunk in stream: if chunk.choices and chunk.choices[0].delta.content: _loop.call_soon_threadsafe( q.put_nowait, chunk.choices[0].delta.content ) # P16-B4: cattura finish_reason dall'ultimo chunk streaming if chunk.choices and chunk.choices[0].finish_reason: _finish_reason_holder[0] = chunk.choices[0].finish_reason finally: _loop.call_soon_threadsafe(q.put_nowait, None) import threading t = threading.Thread(target=_stream_to_queue, daemon=True) t.start() deadline = _loop.time() + STREAM_TIMEOUT _stream_buf: list[str] = [] while True: remaining = deadline - _loop.time() if remaining <= 0: _record_provider_outcome(provider.name, 'timeout') raise TimeoutError(f"stream timeout {STREAM_TIMEOUT}s") try: token = await asyncio.wait_for(q.get(), timeout=min(remaining, 5.0)) except asyncio.TimeoutError: _record_provider_outcome(provider.name, 'timeout') raise TimeoutError(f"stream timeout {STREAM_TIMEOUT}s") if token is None: break _stream_buf.append(token) yield token # S752-C: registra esito stream completato _record_provider_outcome( provider.name, _classify_llm_response(''.join(_stream_buf)) ) # P16-B4: espone finish_reason al caller (unified_loop.py legge _last_finish_reason) self._last_finish_reason = _finish_reason_holder[0] return except asyncio.TimeoutError: last_error = TimeoutError(f"{provider.name} stream timeout dopo {STREAM_TIMEOUT}s") continue except Exception as exc: last_error = exc exc_str = str(exc) is_bad_model_stream = ( "not a valid model" in exc_str or ("400" in exc_str and "BadRequest" in exc_str) ) if is_bad_model_stream and model is not None: try: _fallback_model = self._model_for(provider, None) _fallback_msgs = self._inject_no_think(messages) if provider.name == "groq-qwen" else messages _fallback_max = _safe_max_tokens(max_tokens, _fallback_model) _extra_fb: dict = {} if provider.name == "gemini": _extra_fb = {"extra_body": {"thinking": {"type": "disabled"}}} _fb_resp = await asyncio.wait_for( asyncio.to_thread( client.chat.completions.create, model=_fallback_model, messages=_fallback_msgs, temperature=temperature, max_tokens=_fallback_max, stream=False, **_extra_fb, ), timeout=STREAM_TIMEOUT, ) _fb_text = _fb_resp.choices[0].message.content or "" if _fb_text: _record_provider_outcome(provider.name, _classify_llm_response(_fb_text)) yield _fb_text return except Exception as _exc: _logger.debug("[ai_client] silenced %s", type(_exc).__name__) # noqa: BLE001 # S-LOOP3: registra esito streaming nel health tracker (era mancante nel path Exception) # 429/rate_limit → timeout, altri errori → corrupt _sl3_exc_str = str(exc) _sl3_is_rl = "429" in _sl3_exc_str or "rate_limit" in _sl3_exc_str.lower() or "402" in _sl3_exc_str _sl3_outcome = "timeout" if _sl3_is_rl or "timeout" in _sl3_exc_str.lower() else "corrupt" _record_provider_outcome(provider.name, _sl3_outcome) continue raise RuntimeError(f"Nessun provider streaming disponibile: {last_error}") # ── Embeddings ──────────────────────────────────────────────────────────── async def embed(self, text: str, model: str = "text-embedding-3-small") -> list[float]: for provider in self.providers: embedding_model = provider.embedding_model or os.getenv("EMBEDDING_MODEL") if not embedding_model: continue client = self._client_for(provider) try: response = await asyncio.wait_for( asyncio.to_thread( client.embeddings.create, input=[text], model=embedding_model or model, ), timeout=PROVIDER_TIMEOUT, ) return response.data[0].embedding except Exception: continue return []