sync: 166 file da Baida98/AI@a6ac2424e11e5c320c5ff688e1ce7addac64cdab (local-fallback deploy-all)

agents/unified_loop.py

@@ -2231,6 +2231,58 @@ class UnifiedAgentLoop(DirectToolsMixin, PromptBuilderMixin, LLMSelectionMixin,
                     _rec_timing("coder_ms",  _llm_elapsed)  # Sprint 5 ITEM 14: phase timing
                 except Exception as _exc:
                     _logger.debug("[unified_loop] silenced %s", type(_exc).__name__)  # noqa: BLE001
+                # BENCH-SHADOW: validator osservazionale MMLU/coding. Fail-open: non
+                # modifica answer, retry, provider routing o scoring.
+                try:
+                    from benchmarks.shadow_telemetry import validate_and_record_shadow
+                    validate_and_record_shadow(
+                        goal=state.goal,
+                        answer=answer,
+                        metadata={
+                            "provider": getattr(_active_llm, "provider", None),
+                            "model": getattr(_active_llm, "model", None),
+                            "profile": getattr(_active_llm, "profile", None),
+                            "attempt": _llm_try,
+                            "latency_ms": round(_llm_elapsed, 2),
+                            "source": "unified_loop",
+                        },
+                    )
+                except Exception as _exc:
+                    _logger.debug("[unified_loop] shadow telemetry silenced %s", type(_exc).__name__)
+
+                # BENCH-CODE-RETRY: retry strutturato solo per output TypeScript
+                # non estraibile/non conforme. Non aggiunge tentativi oltre il budget
+                # esistente e non scatta su goal non-coding.
+                if not _is_last:
+                    try:
+                        from benchmarks.validators import validate_coding_retry
+                        _code_validation = validate_coding_retry(
+                            state.goal,
+                            answer,
+                            is_last_attempt=_is_last,
+                        )
+                        if _code_validation is not None:
+                            state.steps.append({
+                                "action": f"typescript_contract_retry_{_llm_try}",
+                                "failure_code": _code_validation.failure_code,
+                            })
+                            _code_repair = (
+                                "CONTRATTO TYPESCRIPT FALLITO: "
+                                f"{_code_validation.failure_code}.\n"
+                                "Ripeti ora la risposta da zero. Restituisci ESATTAMENTE un solo blocco "
+                                "```typescript ... ``` non vuoto, completo e compilabile. "
+                                "Mantieni la firma e tutti i simboli richiesti dal task. "
+                                "Non usare pseudocodice, Python, testo al posto del codice, TODO o placeholder."
+                            )
+                            messages = [
+                                messages[0],
+                                {"role": "system", "content": _code_repair},
+                                *messages[1:],
+                            ]
+                            _error_severity = "syntax"
+                            continue
+                    except Exception as _exc:
+                        _logger.debug("[unified_loop] coding validator retry silenced %s", type(_exc).__name__)
                 # P16-B4: segnala truncation SSE se finish_reason == "length"
                 _fr = getattr(_active_llm, '_last_finish_reason', 'stop')
                 if _fr == 'length' and on_step:

benchmark-extended.mjs

@@ -314,23 +314,33 @@ FSH.autonomy =
 // e ritenta 1 volta se la risposta è troppo corta (cold start / timeout transitorio)
 async function callAgentWithRetry(task, timeoutMs) {
   const hint = FSH[task.category];
-  // Append FSH hint so the agent knows the expected output format
+  // Append FSH hint so the agent knows the expected output format.
   const goal = hint
-    ? `${task.prompt}
-
----
-📌 FORMATO RISPOSTA ATTESO:
-${hint}`
+    ? `${task.prompt}\n\n---\n📌 FORMATO RISPOSTA ATTESO:\n${hint}`
     : task.prompt;
   const a = await callAgent(goal, timeoutMs);
   // Infrastructure failures are not model answers and must not be retried/scored.
   if (a.failed) return a;
-  // Retry once if response is too short (transient failure / cold start)
+
+  // Reasoning retry: one hidden-contract recalculation before the final attempt.
+  const reasoningFailure = reasoningRetryFailure(task, a.output);
+  if (reasoningFailure) {
+    const repairGoal = `${goal}\n\n---\n🔁 CONTROLLO DI CALCOLO:\nLa risposta numerica finale non è stata accettata dal controllo deterministico (${reasoningFailure}). Ricalcola il problema indipendentemente, verifica ogni passaggio e restituisci una sola risposta finale nel formato #### N. Non assumere il risultato precedente.`;
+    if(!F_JSON) process.stdout.write(`    ⟳ retry reasoning (${reasoningFailure})... `);
+    await new Promise(r => setTimeout(r, 1000));
+    const a2 = await callAgent(repairGoal, timeoutMs);
+    if (!F_JSON) process.stdout.write("ok\\n");
+    const secondFailure = reasoningRetryFailure(task, a2.output);
+    if (!a2.failed && !secondFailure) return a2;
+    return (a2.output||"").length > (a.output||"").length ? a2 : a;
+  }
+
+  // Retry once if response is too short (transient failure / cold start).
   if ((a.output||"").length < 40 && !(a.output||"").includes("TIMEOUT")) {
     if(!F_JSON) process.stdout.write("    ⟳ retry (risposta vuota)... ");
     await new Promise(r => setTimeout(r, 4000));
     const a2 = await callAgent(goal, timeoutMs);
-    if (!F_JSON) process.stdout.write("ok\n");
+    if (!F_JSON) process.stdout.write("ok\\n");
     return (a2.output||"").length > (a.output||"").length ? a2 : a;
   }
   return a;
@@ -535,13 +545,25 @@ async function runCmd(cmd,args=[],cwd="/tmp",ms=20000){
   });
 }
 function extractCode(out,langs){
-  const re=/```(\w*)\r?\n([\s\S]*?)```/g; let best={code:"",len:0};
+  // Accetta fence canonicali su nuova riga e fence compatti con contenuto sulla
+  // stessa riga; il backend validator usa la stessa tolleranza.
+  const re=/```([A-Za-z0-9_+#.-]*)[ \t]*(?:\r?\n|[ \t]+)([\s\S]*?)```/g; let best={code:"",len:0};
   for(const m of out.matchAll(re)){
     const l=(m[1]||"").toLowerCase(),c=m[2].trimEnd();
     if((langs.includes(l)||langs.includes("*"))&&c.length>best.len) best={code:c,len:c.length};
   }
   return best.code;
 }
+
+// MMLU parser canonico — mantenere sincronizzato con backend/benchmarks/validators.py.
+// La priorità replica il contratto v5, esteso con ANSWER/FINAL per risposte
+// strutturate; il fallback isolato resta compatibile con il runner storico.
+function extractMMLUChoice(out){
+  const text=String(out||"");
+  const m=text.match(/\*\*\(?([A-D])\)?\*\*|\b(?:ANSWER|FINAL(?:\s+ANSWER)?|RISPOSTA|CHOICE|SCELTA)\b\s*(?:IS|=|:)\s*[*`_\[(]*([A-D])[*`_\])]*|\bRisposta[:\s]+([A-D])\b|\b([A-D])\)/i)
+    || text.match(/\b([A-D])\b/i);
+  return m ? String(m[1]||m[2]||m[3]||m[4]||"").toUpperCase() || null : null;
+}
 // ── LLM-as-Judge (GROQ/Cerebras semantic scoring) ────────────────────────────
 const _JUDGE_CACHE = new Map();
 
@@ -633,6 +655,27 @@ async function fetchGSM8K(seed){
   return{question:row.question,expectedAnswer:parseInt(m[1].replace(/,/g,"")),
     offset:hfOffset(seed,HF_GSM8K),source:"openai/gsm8k"};
 }
+function extractReasoningNumbers(text){
+  const source=String(text||"");
+  const explicit=[...source.matchAll(/(?:####|final\s+answer|answer|risposta|risultato|result|total|totale)\s*[:=]?\s*(-?\d[\d,]*(?:\.\d+)?)/gi)]
+    .map(m=>Number(m[1].replace(/,/g,"")));
+  if(explicit.length)return explicit;
+  const bold=[...source.matchAll(/\*\*\s*(-?\d[\d,]*(?:\.\d+)?)\s*\*\*/g)]
+    .map(m=>Number(m[1].replace(/,/g,"")));
+  if(bold.length)return bold;
+  const lines=[...source.matchAll(/^\s*(-?\d[\d,]*(?:\.\d+)?)\s*$/gm)]
+    .map(m=>Number(m[1].replace(/,/g,"")));
+  return lines.length?lines.slice(-1):[];
+}
+function reasoningRetryFailure(task,output){
+  if(task.category!=="reasoning"||!Number.isFinite(task.expectedAnswer))return null;
+  const candidates=extractReasoningNumbers(output);
+  const distinct=[...new Set(candidates)];
+  if(!distinct.length)return"answer_missing";
+  if(distinct.length>1)return"calculation_conflict";
+  return distinct[0]!==task.expectedAnswer?"wrong_numeric_answer":null;
+}
+
 async function fetchBBH(seed){
   const row=await hfRow("lukaemon/bbh","logical_deduction_three_objects","test",hfOffset(seed,HF_BBH,3571));
   if(!row?.input)return null;
@@ -672,35 +715,51 @@ async function callAgent(goal,timeoutMs=90000){
     if(!res.body)return{ok:false,output:"",engine,ttfa,toolCalls,durationMs:Date.now()-t0,failed:true,failureReason:"stream body mancante"};
     const dec=new TextDecoder(); let buf="";
     const rd=res.body.getReader();
-    loop:while(true){
-      const{done:d,value}=await rd.read(); if(d)break;
-      buf+=dec.decode(value,{stream:true});
-      const lines=buf.split("\n"); buf=lines.pop()??"";
-      for(const ln of lines){
-        if(!ln.startsWith("data:"))continue;
-        const raw=ln.slice(5).trim();
-        if(raw==="[DONE]")break loop;
-        try{
-          const ev=JSON.parse(raw),now=Date.now()-t0;
-          if(ev.type==="tool_use"){toolCalls++;ttfa=ttfa??now;}
-          else if(ev.type==="step_done"){ttfa=ttfa??now;}
-          else if(ev.type==="text_chunk"&&(ev.token||ev.content)&&!done){
-            const chunk=String(ev.token||ev.content); const trimmed=chunk.trim();
+    let stopStream=false;
+    const processSSEData=(raw)=>{
+      if(!raw||raw==="[DONE]")return raw==="[DONE]";
+      try{
+        const ev=JSON.parse(raw),now=Date.now()-t0;
+        // Backend emits `event`; tolerate legacy/proxy clients that emit `type`.
+        const eventType=ev.type||ev.event||"";
+        if(eventType==="tool_use"){toolCalls++;ttfa=ttfa??now;}
+        else if(eventType==="step_done"){ttfa=ttfa??now;}
+        else if(eventType==="text_chunk"&&!done){
+          const value=ev.token??ev.content;
+          if(value!==undefined&&value!==null){
+            const chunk=String(value),trimmed=chunk.trim();
             if(!(trimmed.startsWith("{")&&trimmed.endsWith("}")))out+=chunk;
             ttfa=ttfa??now;
-          } else if(ev.type==="task_error"){
-            failed=true;failureReason=String(ev.error||"task_error");done=true;break loop;
-          } else if(ev.type==="task_done"){
-            const result=typeof ev.result==="string"?ev.result:"";
-            if(ev.success===false||result.startsWith("[LLM_UNAVAILABLE]")||result.includes("tutti i provider configurati sono falliti")){
-              failed=true;failureReason=result||"provider_unavailable";engine=ev.engine??engine;done=true;break loop;
-            }
-            if(result&&result.length>(out||"").length)out=result;
-            engine=ev.engine??engine;done=true;break loop;
           }
-        }catch{}
+        } else if(eventType==="task_error"){
+          failed=true;failureReason=String(ev.error||"task_error");done=true;return true;
+        } else if(eventType==="task_done"){
+          const rawResult=ev.result??ev.output??ev.answer??ev.content??"";
+          const result=typeof rawResult==="string"?rawResult:String(rawResult||"");
+          if(ev.success===false||result.startsWith("[LLM_UNAVAILABLE]")||result.includes("tutti i provider configurati sono falliti")){
+            failed=true;failureReason=result||"provider_unavailable";engine=ev.engine??engine;done=true;return true;
+          }
+          if(result&&result.length>(out||"").length)out=result;
+          engine=ev.engine??engine;done=true;return true;
+        }
+      }catch{}
+      return false;
+    };
+    while(!stopStream){
+      const{done:d,value}=await rd.read();
+      if(d){
+        buf+=dec.decode();
+        break;
+      }
+      buf+=dec.decode(value,{stream:true});
+      const lines=buf.split(/\r?\n/); buf=lines.pop()??"";
+      for(const ln of lines){
+        if(!ln.startsWith("data:"))continue;
+        if(processSSEData(ln.slice(5).trim())){stopStream=true;break;}
       }
     }
+    // A proxy may close after a final unterminated data line.
+    if(!stopStream&&buf.trim().startsWith("data:"))processSSEData(buf.trim().slice(5).trim());
     rd.cancel?.();
   }catch(e){
     if(taskId){
@@ -1357,8 +1416,7 @@ async function makeMMLU(rng,seed){
       hfSource:"cais/mmlu/"+subject,hfOffset:offset,targetMs:30000,ref:REF.mmlu,
       prompt:"Domanda di informatica a scelta multipla:\n\n"+row.question+"\n\nA) "+row.choices[0]+"\nB) "+row.choices[1]+"\nC) "+row.choices[2]+"\nD) "+row.choices[3]+"\n\nRispondi con la lettera **(A/B/C/D)** e spiega brevemente il ragionamento.",
       verify:async function(o){
-        const m=o.match(/\*\*\(?([A-D])\)?\*\*|\bRisposta[:\s]+([A-D])\b|\b([A-D])\)/)||o.match(/\b([A-D])\b/);
-        const got=m?(m[1]||m[2]||m[3]||m[4]):null;
+        const got=extractMMLUChoice(o);
         const correct=got===ansLetter;
         const hasExp=o.length>60&&/perch|quindi|perche|because|quindi|questo|in quanto/i.test(o);
         const judge=await judgeWithLLM(row.question,o,{
@@ -1378,8 +1436,7 @@ async function makeMMLU(rng,seed){
     hfSource:"local-cs-fundamentals",targetMs:30000,ref:REF.mmlu,
     prompt:"Domanda di informatica a scelta multipla:\n\n"+f.q+"\n\nA) "+f.choices[0]+"\nB) "+f.choices[1]+"\nC) "+f.choices[2]+"\nD) "+f.choices[3]+"\n\nRispondi con la lettera **(A/B/C/D)** e spiega brevemente il ragionamento.",
     verify:async function(o){
-      const m=o.match(/\*\*\(?([A-D])\)?\*\*|\bRisposta[:\s]+([A-D])\b/)||o.match(/\b([A-D])\b/);
-      const got=m?(m[1]||m[2]||m[3]):null;
+      const got=extractMMLUChoice(o);
       const correct=got===ansLetter;
       const hasExp=o.length>60;
       const judge=await judgeWithLLM(f.q,o,{
@@ -1402,7 +1459,7 @@ async function makeReasoning(rng,seed){
   const gsm=await fetchGSM8K(seed);
   if(gsm){
     return{id:"RS",category:"reasoning",label:`GSM8K #${gsm.offset}: ${gsm.question.slice(0,50)}…`,
-      hfSource:gsm.source,hfOffset:gsm.offset,targetMs:50000,ref:REF.reasoning,
+      hfSource:gsm.source,hfOffset:gsm.offset,expectedAnswer:gsm.expectedAnswer,targetMs:50000,ref:REF.reasoning,
       prompt:`Risolvi il problema matematico passo per passo.\n\n${gsm.question}\n\nFormato risposta: **#### N** (N = numero intero).`,
       verify:async(o)=>{
         const m=o.match(/####\s*(-?\d[\d,]*)/)||o.match(/(?:answer|risposta|risultato|total|totale|result)[:\s=]+(-?\d[\d,]+)/i)||o.match(/\*\*(-?\d[\d,]+)\*\*/);

benchmarks/__init__.py

@@ -0,0 +1,41 @@
+"""Deterministic benchmark validators and observe-only model discovery."""
+
+from .model_watch_adapter import (
+    CatalogResult,
+    CatalogStatus,
+    GeminiModelsAdapter,
+    ObserveOnlyModelsAdapter,
+    ProfileScan,
+    ProviderProfile,
+    scan_profiles,
+)
+from .shadow_telemetry import (
+    shadow_enabled,
+    validate_and_record_shadow,
+)
+from .validators import (
+    ValidationResult,
+    validate_coding_output,
+    validate_coding_retry,
+    validate_mmlu_output,
+    validate_reasoning_output,
+    validate_reasoning_retry,
+)
+
+__all__ = [
+    "CatalogResult",
+    "CatalogStatus",
+    "GeminiModelsAdapter",
+    "ObserveOnlyModelsAdapter",
+    "ProfileScan",
+    "ProviderProfile",
+    "scan_profiles",
+    "ValidationResult",
+    "shadow_enabled",
+    "validate_and_record_shadow",
+    "validate_coding_output",
+    "validate_coding_retry",
+    "validate_mmlu_output",
+    "validate_reasoning_output",
+    "validate_reasoning_retry",
+]

benchmarks/model_watch_adapter.py

@@ -0,0 +1,279 @@
+"""Observe-only model catalog adapters.
+
+This module performs discovery only. It never updates ai_providers, selects a
+fallback, or persists credentials. Callers can use CatalogResult as an audit
+record and decide separately whether a later approval/apply phase is allowed.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field, replace
+import asyncio
+from enum import Enum
+import json
+import re
+from typing import Any, Mapping, Optional
+
+import httpx
+
+
+class CatalogStatus(str, Enum):
+    AVAILABLE = "available"
+    UNAUTHORIZED = "unauthorized"
+    FORBIDDEN = "forbidden"
+    RATE_LIMITED = "rate_limited"
+    PROVIDER_ERROR = "provider_error"
+    TIMEOUT = "timeout"
+    NETWORK_ERROR = "network_error"
+    MALFORMED = "malformed"
+
+
+@dataclass(frozen=True)
+class ProviderProfile:
+    provider: str
+    profile: str
+    base_url: str
+    api_key: str
+    default_model: str
+    auth_mode: str = "bearer"  # bearer | query_key
+
+
+@dataclass(frozen=True)
+class CatalogResult:
+    provider: str
+    profile: str
+    status: CatalogStatus
+    http_status: Optional[int] = None
+    models: tuple[str, ...] = ()
+    default_available: Optional[bool] = None
+    retry_after_seconds: Optional[int] = None
+    detail: str = ""
+    checked_url: str = ""
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+    @property
+    def should_auto_apply(self) -> bool:
+        """Observe-only invariant: this adapter can never authorize a write."""
+        return False
+
+    def as_audit_record(self) -> dict[str, Any]:
+        return {
+            "provider": self.provider,
+            "profile": self.profile,
+            "status": self.status.value,
+            "http_status": self.http_status,
+            "model_count": len(self.models),
+            "default_available": self.default_available,
+            "retry_after_seconds": self.retry_after_seconds,
+            "detail": self.detail[:240],
+            "checked_url": self.checked_url,
+            "metadata": dict(self.metadata),
+        }
+
+
+_RETRY_AFTER_SECONDS = re.compile(r"^\s*(\d+)\s*$")
+
+
+def models_url(base_url: str) -> str:
+    """Normalize common OpenAI-compatible base URLs to a models endpoint."""
+    value = base_url.rstrip("/")
+    for suffix in ("/chat/completions", "/completions"):
+        if value.endswith(suffix):
+            value = value[: -len(suffix)]
+    if not value.endswith("/models"):
+        value += "/models"
+    return value
+
+
+def _retry_after(headers: Mapping[str, str]) -> Optional[int]:
+    raw = headers.get("retry-after") or headers.get("Retry-After")
+    if not raw:
+        return None
+    match = _RETRY_AFTER_SECONDS.match(raw)
+    return int(match.group(1)) if match else None
+
+
+def _safe_detail(response: httpx.Response) -> str:
+    """Return bounded provider detail without authorization headers or secrets."""
+    try:
+        payload = response.json()
+        if isinstance(payload, Mapping):
+            for key in ("error", "message", "detail", "code"):
+                value = payload.get(key)
+                if value is not None:
+                    return str(value)[:240]
+        return json.dumps(payload, ensure_ascii=True)[:240]
+    except Exception:
+        return response.text[:240]
+
+
+def _models_from_gemini_payload(payload: Any) -> tuple[str, ...] | None:
+    """Parse Gemini's native {models: [{name: 'models/<id>'}]} payload."""
+    if not isinstance(payload, Mapping) or not isinstance(payload.get("models"), list):
+        return None
+    models: list[str] = []
+    for item in payload["models"]:
+        if not isinstance(item, Mapping):
+            continue
+        name = item.get("name") or item.get("baseModelId")
+        if isinstance(name, str) and name.strip():
+            normalized = name.strip()
+            if normalized.startswith("models/"):
+                normalized = normalized[len("models/"):]
+            models.append(normalized)
+    return tuple(dict.fromkeys(models))
+
+
+def _models_from_payload(payload: Any) -> tuple[str, ...] | None:
+    if isinstance(payload, Mapping):
+        items = payload.get("data")
+    else:
+        items = payload
+    if not isinstance(items, list):
+        return None
+    models: list[str] = []
+    for item in items:
+        if isinstance(item, Mapping) and isinstance(item.get("id"), str) and item["id"].strip():
+            models.append(item["id"].strip())
+    return tuple(dict.fromkeys(models))
+
+
+@dataclass(frozen=True)
+class ProfileScan:
+    """Results plus profiles skipped because their provider returned 429."""
+    results: tuple[CatalogResult, ...]
+    skipped_rate_limited: tuple[CatalogResult, ...] = ()
+
+
+class ObserveOnlyModelsAdapter:
+    """Fetch a provider catalog and classify the result; never mutates state."""
+
+    def __init__(self, *, timeout_seconds: float = 8.0, client: httpx.AsyncClient | None = None):
+        self.timeout_seconds = timeout_seconds
+        self._client = client
+
+    async def list_models(self, profile: ProviderProfile) -> CatalogResult:
+        url = models_url(profile.base_url)
+        headers = {"Accept": "application/json"}
+        params: dict[str, str] = {}
+        if profile.auth_mode == "query_key":
+            params["key"] = profile.api_key
+        else:
+            headers["Authorization"] = f"Bearer {profile.api_key}"
+
+        owns_client = self._client is None
+        client = self._client or httpx.AsyncClient(timeout=self.timeout_seconds)
+        try:
+            response = await client.get(url, headers=headers, params=params)
+            status = response.status_code
+            if status == 401:
+                return self._result(profile, url, CatalogStatus.UNAUTHORIZED, response)
+            if status == 403:
+                return self._result(profile, url, CatalogStatus.FORBIDDEN, response)
+            if status == 429:
+                return self._result(profile, url, CatalogStatus.RATE_LIMITED, response)
+            if 500 <= status <= 599:
+                return self._result(profile, url, CatalogStatus.PROVIDER_ERROR, response)
+            if status != 200:
+                return self._result(profile, url, CatalogStatus.NETWORK_ERROR, response)
+            try:
+                payload = response.json()
+            except (ValueError, json.JSONDecodeError):
+                return self._result(profile, url, CatalogStatus.MALFORMED, response, detail="invalid JSON")
+            models = _models_from_payload(payload)
+            if models is None:
+                return self._result(profile, url, CatalogStatus.MALFORMED, response, detail="missing data list")
+            return CatalogResult(
+                provider=profile.provider,
+                profile=profile.profile,
+                status=CatalogStatus.AVAILABLE,
+                http_status=status,
+                models=models,
+                default_available=profile.default_model in models,
+                checked_url=url,
+                detail="catalog fetched",
+            )
+        except httpx.TimeoutException as exc:
+            return CatalogResult(profile.provider, profile.profile, CatalogStatus.TIMEOUT, detail=str(exc)[:240], checked_url=url)
+        except httpx.RequestError as exc:
+            return CatalogResult(profile.provider, profile.profile, CatalogStatus.NETWORK_ERROR, detail=str(exc)[:240], checked_url=url)
+        finally:
+            if owns_client:
+                await client.aclose()
+
+    @staticmethod
+    def _result(profile: ProviderProfile, url: str, status: CatalogStatus, response: httpx.Response, *, detail: str = "") -> CatalogResult:
+        return CatalogResult(
+            provider=profile.provider,
+            profile=profile.profile,
+            status=status,
+            http_status=response.status_code,
+            retry_after_seconds=_retry_after(response.headers) if status == CatalogStatus.RATE_LIMITED else None,
+            detail=detail or _safe_detail(response),
+            checked_url=url,
+        )
+
+
+class GeminiModelsAdapter(ObserveOnlyModelsAdapter):
+    """Observe-only adapter for Gemini's native ``models`` catalog."""
+
+    async def list_models(self, profile: ProviderProfile) -> CatalogResult:
+        url = models_url(profile.base_url)
+        headers = {"Accept": "application/json"}
+        params = {"key": profile.api_key}
+        owns_client = self._client is None
+        client = self._client or httpx.AsyncClient(timeout=self.timeout_seconds)
+        try:
+            response = await client.get(url, headers=headers, params=params)
+            status = response.status_code
+            if status == 401:
+                return self._result(profile, url, CatalogStatus.UNAUTHORIZED, response)
+            if status == 403:
+                return self._result(profile, url, CatalogStatus.FORBIDDEN, response)
+            if status == 429:
+                return self._result(profile, url, CatalogStatus.RATE_LIMITED, response)
+            if 500 <= status <= 599:
+                return self._result(profile, url, CatalogStatus.PROVIDER_ERROR, response)
+            if status != 200:
+                return self._result(profile, url, CatalogStatus.NETWORK_ERROR, response)
+            try:
+                payload = response.json()
+            except (ValueError, json.JSONDecodeError):
+                return self._result(profile, url, CatalogStatus.MALFORMED, response, detail="invalid JSON")
+            models = _models_from_gemini_payload(payload)
+            if models is None:
+                return self._result(profile, url, CatalogStatus.MALFORMED, response, detail="missing models list")
+            return CatalogResult(
+                provider=profile.provider,
+                profile=profile.profile,
+                status=CatalogStatus.AVAILABLE,
+                http_status=status,
+                models=models,
+                default_available=profile.default_model in models,
+                checked_url=url,
+                detail="Gemini catalog fetched",
+                metadata={"catalog_format": "gemini_native"},
+            )
+        except httpx.TimeoutException as exc:
+            return CatalogResult(profile.provider, profile.profile, CatalogStatus.TIMEOUT, detail=str(exc)[:240], checked_url=url)
+        except httpx.RequestError as exc:
+            return CatalogResult(profile.provider, profile.profile, CatalogStatus.NETWORK_ERROR, detail=str(exc)[:240], checked_url=url)
+        finally:
+            if owns_client:
+                await client.aclose()
+
+
+async def scan_profiles(
+    profiles: list[ProviderProfile],
+    *,
+    adapter: ObserveOnlyModelsAdapter | None = None,
+) -> ProfileScan:
+    """Scan a mixed pool and isolate 429 profiles without blocking healthy ones."""
+    adapter = adapter or ObserveOnlyModelsAdapter()
+    results = await asyncio.gather(*(adapter.list_models(profile) for profile in profiles))
+    skipped = tuple(
+        replace(result, metadata={"skipped": True, "skip_reason": "rate_limited"})
+        for result in results
+        if result.status == CatalogStatus.RATE_LIMITED
+    )
+    active = tuple(result for result in results if result.status != CatalogStatus.RATE_LIMITED)
+    return ProfileScan(results=active, skipped_rate_limited=skipped)

benchmarks/shadow_telemetry.py

@@ -0,0 +1,125 @@
+"""Fail-open shadow telemetry for benchmark output validators.
+
+Shadow mode records validator outcomes only. It never changes the answer, retry
+budget, provider selection, or benchmark score. Raw model output is deliberately
+not persisted; only length and normalized validator evidence are stored.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import json
+import os
+from pathlib import Path
+import threading
+from typing import Any, Mapping, Optional
+
+from .validators import ValidationResult, validate_coding_output, validate_mmlu_output
+
+
+_ENABLED_VALUES = frozenset({"1", "true", "yes", "on"})
+_WRITE_LOCK = threading.Lock()
+
+
+def shadow_enabled() -> bool:
+    return os.getenv("BENCHMARK_SHADOW_MODE", "0").strip().lower() in _ENABLED_VALUES
+
+
+def infer_benchmark_category(goal: Any) -> Optional[str]:
+    """Infer only the two supported benchmark categories from explicit markers."""
+
+    text = str(goal or "")
+    lowered = text.lower()
+    if "mmlu" in lowered or "scelta multipla" in lowered or "a/b/c/d" in lowered:
+        return "mmlu"
+    if "code_correct" in lowered or "typescript" in lowered or "```typescript" in lowered:
+        return "coding"
+    return None
+
+
+def _safe_metadata(metadata: Optional[Mapping[str, Any]]) -> dict[str, Any]:
+    allowed = {
+        "provider",
+        "model",
+        "profile",
+        "attempt",
+        "latency_ms",
+        "first_token_ms",
+        "task_id",
+        "source",
+    }
+    safe: dict[str, Any] = {}
+    for key in allowed:
+        value = (metadata or {}).get(key)
+        if value is None:
+            continue
+        if isinstance(value, (str, int, float, bool)):
+            safe[key] = value
+        else:
+            safe[key] = str(value)[:120]
+    return safe
+
+
+def _evidence_for_log(result: ValidationResult) -> dict[str, Any]:
+    evidence: dict[str, Any] = {}
+    for key, value in result.evidence.items():
+        if key == "source_length":
+            evidence[key] = value
+        elif key in {"candidates", "distinct_candidates", "required_symbols", "missing_symbols", "declarations", "fence_count", "languages", "extraction", "significant_lines", "correct", "expected", "has_import_or_export", "has_syntax_tokens"}:
+            evidence[key] = value
+    return evidence
+
+
+def _log_path() -> Path:
+    return Path(os.getenv("BENCHMARK_SHADOW_LOG_PATH", "/tmp/baida98-benchmark-shadow.jsonl"))
+
+
+def _append_event(event: dict[str, Any]) -> None:
+    path = _log_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with _WRITE_LOCK:
+        with path.open("a", encoding="utf-8") as handle:
+            handle.write(json.dumps(event, ensure_ascii=False, separators=(",", ":")) + "\n")
+
+
+def validate_and_record_shadow(
+    *,
+    goal: Any,
+    answer: Any,
+    metadata: Optional[Mapping[str, Any]] = None,
+) -> Optional[ValidationResult]:
+    """Validate and record a supported benchmark response in fail-open shadow mode."""
+
+    if not shadow_enabled():
+        return None
+
+    category = infer_benchmark_category(goal)
+    if category is None:
+        return None
+
+    if category == "mmlu":
+        result = validate_mmlu_output(answer)
+        validator = "mmlu_v1"
+    else:
+        result = validate_coding_output(answer)
+        validator = "coding_v1"
+
+    text = answer if isinstance(answer, str) else str(answer or "")
+    event = {
+        "schema_version": 1,
+        "event": "benchmark_shadow_validation",
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "category": category,
+        "validator": validator,
+        "valid": result.valid,
+        "failure_code": result.failure_code,
+        "response_chars": len(text),
+        "evidence": _evidence_for_log(result),
+        "metadata": _safe_metadata(metadata),
+    }
+    try:
+        _append_event(event)
+    except Exception:
+        # Shadow telemetry must never break the agent loop.
+        return result
+    return result

benchmarks/validators.py

@@ -0,0 +1,351 @@
+"""Deterministic validators for benchmark outputs.
+
+The validators in this module deliberately do not call an LLM or a provider. They
+only normalize an output when the evidence is unambiguous and otherwise return a
+stable failure code that the retry layer can act on.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import json
+import re
+from typing import Any, Iterable, Mapping, Optional
+
+
+_MMLU_LETTERS = frozenset("ABCD")
+_MMLU_EXPLICIT = re.compile(
+    r"\b(?:answer|答案|risposta|final(?:\s+answer)?|choice|scelta)\b\s*"
+    r"(?:is|=|:)\s*[*`_\[\(]*([A-D])[*`_\]\)]*",
+    re.IGNORECASE,
+)
+_MMLU_MARKED = re.compile(r"(?:^|\n|\s)(?:\(?([A-D])\)?)[\.:\)](?:\s|$)", re.IGNORECASE)
+_MMLU_ISOLATED = re.compile(r"(?<![A-Za-z])([A-D])(?![A-Za-z])", re.IGNORECASE)
+
+_CODE_FENCE = re.compile(
+    r"```\s*([A-Za-z0-9_+#.-]*)\s*\n?(.*?)```", re.IGNORECASE | re.DOTALL
+)
+_CODE_JSON_KEYS = ("code", "typescript", "source", "implementation")
+_TS_DECLARATION = re.compile(
+    r"\b(?:export\s+)?(?:async\s+)?(?:function|class|interface|type|const|let|var)\s+([A-Za-z_$][\w$]*)",
+    re.MULTILINE,
+)
+_TS_IMPORT_EXPORT = re.compile(r"\b(?:import|export)\b")
+_TS_SYNTAX_TOKENS = re.compile(r"[{}();]|=>|:\s*[A-Za-z_$][\w$<>,\[\]| ]*")
+_PLACEHOLDER = re.compile(r"\b(?:TODO|TBD|your implementation|implement here)\b", re.IGNORECASE)
+_REASONING_EXPLICIT = re.compile(
+    r"(?:####|final\s+answer|answer|risposta|risultato|result|total|totale)\s*[:=]?\s*"
+    r"(-?\d[\d,]*(?:\.\d+)?)",
+    re.IGNORECASE,
+)
+_REASONING_BOLD = re.compile(r"\*\*\s*(-?\d[\d,]*(?:\.\d+)?)\s*\*\*")
+_REASONING_LINE_NUMBER = re.compile(r"(?m)^\s*(-?\d[\d,]*(?:\.\d+)?)\s*$")
+_REASONING_FAILURES = frozenset({"answer_missing", "wrong_numeric_answer", "calculation_conflict"})
+
+
+@dataclass(frozen=True)
+class ValidationResult:
+    """Stable validator result consumed by shadow mode and retry logic."""
+
+    valid: bool
+    normalized: Optional[str] = None
+    failure_code: Optional[str] = None
+    evidence: dict[str, Any] = field(default_factory=dict)
+    repair_hint: Optional[str] = None
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "valid": self.valid,
+            "normalized": self.normalized,
+            "failure_code": self.failure_code,
+            "evidence": self.evidence,
+            "repair_hint": self.repair_hint,
+        }
+
+
+def _failure(code: str, *, evidence: Optional[dict[str, Any]] = None, hint: str = "") -> ValidationResult:
+    return ValidationResult(
+        valid=False,
+        failure_code=code,
+        evidence=evidence or {},
+        repair_hint=hint or None,
+    )
+
+
+def _success(normalized: str, *, evidence: Optional[dict[str, Any]] = None) -> ValidationResult:
+    return ValidationResult(valid=True, normalized=normalized, evidence=evidence or {})
+
+
+def _clean_text(raw: Any) -> str:
+    if raw is None:
+        return ""
+    if isinstance(raw, str):
+        return raw.strip()
+    return str(raw).strip()
+
+
+def _mmlu_candidates(text: str) -> list[str]:
+    """Return candidates in evidence order, preserving duplicates for ambiguity checks."""
+    explicit = [m.group(1).upper() for m in _MMLU_EXPLICIT.finditer(text)]
+    if explicit:
+        return explicit
+    marked = [m.group(1).upper() for m in _MMLU_MARKED.finditer(text)]
+    if marked:
+        return marked
+    return [m.group(1).upper() for m in _MMLU_ISOLATED.finditer(text)]
+
+
+def validate_mmlu_output(raw: Any, *, expected: Optional[str] = None) -> ValidationResult:
+    """Validate a multiple-choice answer without guessing from explanation prose.
+
+    Accepted outputs contain one unambiguous A/B/C/D choice. Explicit labels such
+    as ``ANSWER: C`` have priority over marked choices and isolated letters. If
+    multiple distinct candidates are present, the result is ambiguous and fails.
+    ``expected`` is optional and is only used to expose correctness in evidence; it
+    never changes the parsing result.
+    """
+
+    text = _clean_text(raw)
+    if not text:
+        return _failure(
+            "answer_missing",
+            hint="Return exactly one canonical choice using ANSWER: A, B, C, or D.",
+        )
+
+    candidates = _mmlu_candidates(text)
+    distinct = sorted(set(candidates))
+    evidence: dict[str, Any] = {
+        "candidates": candidates,
+        "distinct_candidates": distinct,
+        "source_length": len(text),
+    }
+    if expected is not None:
+        normalized_expected = str(expected).strip().upper()
+        evidence["expected"] = normalized_expected
+        if normalized_expected in _MMLU_LETTERS:
+            evidence["correct"] = len(distinct) == 1 and distinct[0] == normalized_expected
+
+    if not candidates:
+        return _failure(
+            "answer_missing",
+            evidence=evidence,
+            hint="Return exactly one canonical choice using ANSWER: A, B, C, or D.",
+        )
+    if len(distinct) != 1 or distinct[0] not in _MMLU_LETTERS:
+        return _failure(
+            "answer_ambiguous",
+            evidence=evidence,
+            hint="Remove competing choices and return one letter: A, B, C, or D.",
+        )
+
+    return _success(distinct[0], evidence=evidence)
+
+
+def _extract_code(raw: Any) -> tuple[Optional[str], str, dict[str, Any]]:
+    """Extract code from a TypeScript fence or a JSON envelope."""
+
+    text = _clean_text(raw)
+    if not text:
+        return None, "none", {"source_length": 0}
+
+    try:
+        decoded = json.loads(text)
+    except (TypeError, json.JSONDecodeError):
+        decoded = None
+    if isinstance(decoded, Mapping):
+        for key in _CODE_JSON_KEYS:
+            value = decoded.get(key)
+            if isinstance(value, str) and value.strip():
+                return value.strip(), f"json:{key}", {"source_length": len(text)}
+
+    fences = _CODE_FENCE.findall(text)
+    if fences:
+        typed = [body.strip() for language, body in fences if language.lower() in {"ts", "typescript"}]
+        if typed:
+            return max(typed, key=len), "fence:typescript", {"fence_count": len(fences)}
+        return None, "fence:wrong-language", {"languages": [language.lower() for language, _ in fences]}
+
+    return None, "none", {"source_length": len(text)}
+
+
+def _normalize_symbols(required_symbols: Iterable[str]) -> list[str]:
+    return [symbol.strip() for symbol in required_symbols if str(symbol).strip()]
+
+
+def _parse_numeric_token(value: str) -> int | float:
+    normalized = value.replace(",", "").strip()
+    number = float(normalized) if "." in normalized else int(normalized)
+    return number
+
+
+def _reasoning_candidates(text: str) -> tuple[list[int | float], str]:
+    """Extract answer candidates conservatively, preferring explicit final markers."""
+    explicit = [_parse_numeric_token(match.group(1)) for match in _REASONING_EXPLICIT.finditer(text)]
+    if explicit:
+        return explicit, "explicit"
+    bold = [_parse_numeric_token(match.group(1)) for match in _REASONING_BOLD.finditer(text)]
+    if bold:
+        return bold, "bold"
+    lines = [_parse_numeric_token(match.group(1)) for match in _REASONING_LINE_NUMBER.finditer(text)]
+    if lines:
+        return lines[-1:], "final_line"
+    return [], "none"
+
+
+def validate_reasoning_output(raw: Any, *, expected: Optional[int | float] = None) -> ValidationResult:
+    """Validate a numeric reasoning answer without calling an LLM.
+
+    Explicit final markers have priority over intermediate arithmetic. Multiple
+    distinct explicit answers are classified as a conflict rather than guessed.
+    """
+    text = _clean_text(raw)
+    if not text:
+        return _failure(
+            "answer_missing",
+            hint="Show the calculation and finish with #### N, where N is the final integer.",
+        )
+
+    candidates, source = _reasoning_candidates(text)
+    distinct = list(dict.fromkeys(candidates))
+    evidence: dict[str, Any] = {
+        "candidates": candidates,
+        "distinct_candidates": distinct,
+        "source": source,
+        "source_length": len(text),
+    }
+    if expected is not None:
+        try:
+            normalized_expected = _parse_numeric_token(str(expected))
+            evidence["expected"] = normalized_expected
+        except ValueError:
+            normalized_expected = expected
+
+    if not candidates:
+        return _failure(
+            "answer_missing",
+            evidence=evidence,
+            hint="Show the calculation and finish with #### N, where N is the final integer.",
+        )
+    if len(distinct) > 1:
+        return _failure(
+            "calculation_conflict",
+            evidence=evidence,
+            hint="Recalculate the final value and provide exactly one final numeric answer.",
+        )
+
+    normalized = distinct[0]
+    if expected is not None and normalized != normalized_expected:
+        evidence["correct"] = False
+        return _failure(
+            "wrong_numeric_answer",
+            evidence=evidence,
+            hint="Recheck every arithmetic step and return the corrected final number.",
+        )
+
+    evidence["correct"] = True if expected is not None else None
+    return _success(str(normalized), evidence=evidence)
+
+
+def validate_reasoning_retry(
+    goal: Any,
+    raw: Any,
+    *,
+    expected: Optional[int | float] = None,
+    is_last_attempt: bool,
+) -> Optional[ValidationResult]:
+    """Return a reasoning failure only when a non-final numeric retry is warranted."""
+    if is_last_attempt or not any(marker in str(goal or "").lower() for marker in ("reasoning", "gsm8k", "risolvi il problema matematico")):
+        return None
+    result = validate_reasoning_output(raw, expected=expected)
+    return result if result.failure_code in _REASONING_FAILURES else None
+
+
+_CODING_RETRY_FAILURES = frozenset({
+    "code_missing",
+    "code_wrong_language",
+    "code_empty",
+    "code_placeholder",
+    "required_symbol_missing",
+    "code_syntax_suspect",
+})
+
+
+def is_typescript_goal(goal: Any) -> bool:
+    lowered = str(goal or "").lower()
+    return any(marker in lowered for marker in ("code_correct", "typescript", "```ts", "```typescript"))
+
+
+def validate_coding_retry(goal: Any, raw: Any, *, is_last_attempt: bool) -> Optional[ValidationResult]:
+    """Return the failed result only when a non-final TypeScript retry is warranted."""
+
+    if is_last_attempt or not is_typescript_goal(goal):
+        return None
+    result = validate_coding_output(raw)
+    return result if result.failure_code in _CODING_RETRY_FAILURES else None
+
+
+def validate_coding_output(
+    raw: Any,
+    *,
+    required_symbols: Iterable[str] = (),
+    min_significant_lines: int = 1,
+    reject_placeholders: bool = True,
+) -> ValidationResult:
+    """Validate extraction and minimum structural quality of TypeScript output.
+
+    This is intentionally a contract validator, not a compiler. Syntax checks are
+    conservative and deterministic; full compilation remains a separate isolated
+    integration test because it depends on the repository's TypeScript toolchain.
+    """
+
+    code, source, extraction_evidence = _extract_code(raw)
+    if code is None:
+        failure = "code_wrong_language" if source == "fence:wrong-language" else "code_missing"
+        return _failure(
+            failure,
+            evidence=extraction_evidence | {"extraction": source},
+            hint="Return exactly one non-empty ```typescript code block.",
+        )
+
+    significant_lines = [line for line in code.splitlines() if line.strip() and not line.strip().startswith("//")]
+    evidence: dict[str, Any] = extraction_evidence | {
+        "extraction": source,
+        "significant_lines": len(significant_lines),
+        "has_import_or_export": bool(_TS_IMPORT_EXPORT.search(code)),
+        "has_syntax_tokens": bool(_TS_SYNTAX_TOKENS.search(code)),
+    }
+
+    if not significant_lines or len(significant_lines) < max(1, min_significant_lines):
+        return _failure(
+            "code_empty",
+            evidence=evidence,
+            hint="Provide a complete non-empty TypeScript implementation.",
+        )
+    if reject_placeholders and _PLACEHOLDER.search(code):
+        return _failure(
+            "code_placeholder",
+            evidence=evidence,
+            hint="Replace TODO/TBD placeholders with executable TypeScript.",
+        )
+
+    declarations = {match.group(1) for match in _TS_DECLARATION.finditer(code)}
+    required = _normalize_symbols(required_symbols)
+    missing = [symbol for symbol in required if symbol not in declarations and not re.search(rf"\b{re.escape(symbol)}\b", code)]
+    evidence["declarations"] = sorted(declarations)
+    evidence["required_symbols"] = required
+    evidence["missing_symbols"] = missing
+    if missing:
+        return _failure(
+            "required_symbol_missing",
+            evidence=evidence,
+            hint=f"Implement and expose the required symbols: {', '.join(missing)}.",
+        )
+
+    if not _TS_SYNTAX_TOKENS.search(code):
+        return _failure(
+            "code_syntax_suspect",
+            evidence=evidence,
+            hint="Return syntactically structured TypeScript with declarations and delimiters.",
+        )
+
+    return _success(code, evidence=evidence)

models/ai_client.py

@@ -16,6 +16,7 @@ from __future__ import annotations
 import asyncio
 import json
 import os
+import re
 import time as _time_mod
 from dataclasses import dataclass
 from typing import AsyncIterator, Optional, List, Tuple
@@ -62,7 +63,7 @@ _PROVIDER_DEFS = [
     # tier 0 — free tier veloce e affidabile
     {"name": "groq",       "env_key": "GROQ_API_KEY",       "base_url": "https://api.groq.com/openai/v1",                    "model_env": "GROQ_MODEL",      "default_model": "openai/gpt-oss-120b",     "tier": 0, "purpose": "reasoning"},
     {"name": "cerebras",   "env_key": "CEREBRAS_API_KEY",   "base_url": "https://api.cerebras.ai/v1",                        "model_env": "CEREBRAS_MODEL",  "default_model": "gpt-oss-120b",                "tier": 0, "purpose": "reasoning"},
-    {"name": "sambanova",  "env_key": "SAMBANOVA_API_KEY",  "base_url": "https://api.sambanova.ai/v1",                       "model_env": "SAMBANOVA_MODEL", "default_model": "DeepSeek-V3.2",               "tier": 0, "purpose": "reasoning"},
+    {"name": "sambanova",  "env_key": "SAMBANOVA_API_KEY",  "base_url": "https://api.sambanova.ai/v1",                       "model_env": "SAMBANOVA_MODEL", "default_model": "DeepSeek-V3.1",               "tier": 0, "purpose": "reasoning"},
     # tier 1 — free tier con rate limit più stretti
     {"name": "openrouter", "env_key": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1",                      "model_env": "OPENROUTER_MODEL","default_model": "openai/gpt-oss-20b:free", "tier": 1, "purpose": "coding"},
     {"name": "hf_router",  "env_key": "HF_TOKEN",           "base_url": "https://router.huggingface.co/v1",                  "model_env": "HF_MODEL",        "default_model": "Qwen/Qwen2.5-Coder-32B-Instruct", "tier": 1, "purpose": "coding"},
@@ -261,12 +262,26 @@ class AIClient:
         state = self._breaker.setdefault(provider.identity, {"failures": 0, "open_until": 0.0})
         failures = int(state.get("failures", 0)) + 1
         severe = any(token in message for token in ("401", "403"))
-        threshold = 1 if severe else self._breaker_threshold
+        quota_limited = any(token in message for token in ("429", "rate limit", "quota"))
+        # A quota/rate-limit response is deterministic: retrying the same
+        # profile immediately only creates a storm. Open that profile on the
+        # first signal and let the provider pool move to another provider.
+        threshold = 1 if severe or quota_limited else self._breaker_threshold
         if failures >= threshold:
-            cooldown = 900.0 if severe else (120.0 if any(token in message for token in ("429", "rate limit", "quota")) else self._breaker_cooldown_s)
+            cooldown = 900.0 if severe else self._rate_limit_cooldown_seconds(message) if quota_limited else self._breaker_cooldown_s
             state["open_until"] = _time_mod.monotonic() + cooldown
         state["failures"] = failures
 
+    @staticmethod
+    def _rate_limit_cooldown_seconds(message: str) -> float:
+        """Return a provider reset-aware cooldown, never shorter than 15 min."""
+        reset_match = re.search(r"x-ratelimit-reset[^0-9]*(\d{10,13})", message, re.IGNORECASE)
+        if reset_match:
+            reset_value = float(reset_match.group(1))
+            reset_epoch = reset_value / 1000.0 if reset_value > 10_000_000_000 else reset_value
+            return max(900.0, reset_epoch - _time_mod.time())
+        return 900.0
+
     def _execution_pool(self, providers: list[ProviderConfig], purpose: str) -> list[ProviderConfig]:
         """Return one rotated, healthy profile per provider endpoint group."""
         groups: dict[tuple[str, str], list[ProviderConfig]] = {}
@@ -286,6 +301,7 @@ class AIClient:
         self,
         purpose: str,
         excluded: set[str] | None = None,
+        providers: list[ProviderConfig] | None = None,
     ) -> list[ProviderConfig]:
         """Select one healthy profile per provider, prioritizing the target purpose.
 
@@ -294,8 +310,9 @@ class AIClient:
         fallback. This prevents retry storms against an exhausted pool.
         """
         excluded = excluded or set()
+        source = self.providers if providers is None else providers
         candidates = [
-            provider for provider in self.providers
+            provider for provider in source
             if provider.name not in excluded and self._is_available(provider)
         ]
         candidates.sort(key=lambda provider: (
@@ -454,7 +471,7 @@ class AIClient:
 
         # Un profilo sano per provider: se l’intero pool primario è in rate
         # limit, il fallback passa automaticamente al provider successivo.
-        providers = self._inter_provider_fallback_pool("stream")
+        providers = self._inter_provider_fallback_pool("stream", providers=providers)
         attempted: list[str] = []
         for provider in providers:
             attempted.append(provider.name)

tests/test_ai_client_provider_unavailability.py

@@ -26,6 +26,12 @@ class _ClientWithFailingProviders(AIClient):
         ]
         self._client_cache = {}
         self._rr_indices = {}
+        # Stato minimo richiesto dai percorsi chat/stream dopo l’introduzione
+        # del circuit breaker per profilo. Non chiama AIClient.__init__ e non
+        # carica provider o segreti dall’ambiente.
+        self._breaker = {}
+        self._breaker_threshold = 2
+        self._breaker_cooldown_s = 60.0
 
     def _client_for(self, _provider):
         return _FailingClient()
@@ -38,7 +44,7 @@ class ProviderUnavailableTests(unittest.IsolatedAsyncioTestCase):
         with self.assertRaises(ProviderUnavailableError) as raised:
             await client.chat([{"role": "user", "content": "hello"}], max_tokens=8)
 
-        self.assertEqual(raised.exception.providers, ("primary", "fallback"))
+        self.assertCountEqual(raised.exception.providers, ("primary", "fallback"))
         self.assertNotIn("api_key", str(raised.exception).lower())
 
     async def test_stream_chat_raises_structured_error_when_every_provider_fails(self):
@@ -48,7 +54,7 @@ class ProviderUnavailableTests(unittest.IsolatedAsyncioTestCase):
             async for _ in client.stream_chat([{"role": "user", "content": "hello"}], max_tokens=8):
                 pass
 
-        self.assertEqual(raised.exception.providers, ("primary", "fallback"))
+        self.assertCountEqual(raised.exception.providers, ("primary", "fallback"))
         self.assertNotIn("api_key", str(raised.exception).lower())
 
     async def test_stream_chat_expands_a_role_specific_provider_pool(self):
@@ -65,7 +71,7 @@ class ProviderUnavailableTests(unittest.IsolatedAsyncioTestCase):
                 async for _ in client.stream_chat([{"role": "user", "content": "hello"}], max_tokens=8):
                     pass
 
-        self.assertEqual(raised.exception.providers, ("gemini-role", "nvidia"))
+        self.assertCountEqual(raised.exception.providers, ("gemini-role", "nvidia"))
 
 
 class RuntimeModelOverrideTests(unittest.TestCase):

tests/test_benchmark_validators.py

@@ -0,0 +1,232 @@
+import unittest
+
+from benchmarks.validators import (
+    validate_coding_output,
+    validate_coding_retry,
+    validate_mmlu_output,
+    validate_reasoning_output,
+    validate_reasoning_retry,
+)
+
+
+class MMLUValidatorTests(unittest.TestCase):
+    def test_accepts_explicit_answer_with_explanation(self):
+        result = validate_mmlu_output(
+            "ANSWER: C\nPerché la complessità nel caso peggiore è quadratica.",
+            expected="C",
+        )
+        self.assertTrue(result.valid)
+        self.assertEqual(result.normalized, "C")
+        self.assertTrue(result.evidence["correct"])
+
+    def test_accepts_marked_choice(self):
+        result = validate_mmlu_output("La scelta corretta è (B). La stack segue LIFO.")
+        self.assertTrue(result.valid)
+        self.assertEqual(result.normalized, "B")
+
+    def test_accepts_single_isolated_letter(self):
+        result = validate_mmlu_output("D")
+        self.assertTrue(result.valid)
+        self.assertEqual(result.normalized, "D")
+
+    def test_accepts_final_answer_contract_used_by_retry(self):
+        result = validate_mmlu_output("Final answer: D\nThe two values overflow because both are negative.")
+        self.assertTrue(result.valid)
+        self.assertEqual(result.normalized, "D")
+
+    def test_accepts_runner_bold_contract(self):
+        result = validate_mmlu_output("**(B)** — risposta scelta")
+        self.assertTrue(result.valid)
+        self.assertEqual(result.normalized, "B")
+
+    def test_explanation_letters_do_not_override_explicit_answer(self):
+        result = validate_mmlu_output("ANSWER: A. Le opzioni B, C e D sono errate.")
+        self.assertTrue(result.valid)
+        self.assertEqual(result.normalized, "A")
+
+    def test_rejects_missing_answer(self):
+        result = validate_mmlu_output("La spiegazione descrive il concetto ma non seleziona un'opzione.")
+        self.assertFalse(result.valid)
+        self.assertEqual(result.failure_code, "answer_missing")
+
+    def test_rejects_conflicting_explicit_answers(self):
+        result = validate_mmlu_output("ANSWER: A\nFinal answer: C")
+        self.assertFalse(result.valid)
+        self.assertEqual(result.failure_code, "answer_ambiguous")
+        self.assertEqual(result.evidence["distinct_candidates"], ["A", "C"])
+
+    def test_rejects_empty_output(self):
+        result = validate_mmlu_output(None)
+        self.assertFalse(result.valid)
+        self.assertEqual(result.failure_code, "answer_missing")
+
+    def test_expected_answer_only_affects_evidence(self):
+        result = validate_mmlu_output("ANSWER: B", expected="C")
+        self.assertTrue(result.valid)
+        self.assertFalse(result.evidence["correct"])
+        self.assertEqual(result.normalized, "B")
+
+
+class CodingValidatorTests(unittest.TestCase):
+    def test_accepts_typescript_fence_and_required_symbol(self):
+        output = """Ecco l'implementazione:
+```typescript
+export function reverseWords(value: string): string {
+  return value.trim().split(/\\s+/).reverse().join(' ');
+}
+```
+"""
+        result = validate_coding_output(output, required_symbols=["reverseWords"], min_significant_lines=3)
+        self.assertTrue(result.valid)
+        self.assertIn("reverseWords", result.normalized)
+        self.assertEqual(result.evidence["missing_symbols"], [])
+
+    def test_accepts_json_envelope(self):
+        output = '{"language":"typescript","code":"export const add = (a: number, b: number): number => a + b;"}'
+        result = validate_coding_output(output, required_symbols=["add"])
+        self.assertTrue(result.valid)
+        self.assertEqual(result.evidence["extraction"], "json:code")
+
+    def test_rejects_empty_fence(self):
+        result = validate_coding_output("```typescript\n\n```")
+        self.assertFalse(result.valid)
+        self.assertEqual(result.failure_code, "code_empty")
+
+    def test_rejects_missing_code_block(self):
+        result = validate_coding_output("La soluzione è implementata nel testo seguente, ma il codice non è incluso.")
+        self.assertFalse(result.valid)
+        self.assertEqual(result.failure_code, "code_missing")
+
+    def test_rejects_wrong_language_fence(self):
+        result = validate_coding_output("```python\ndef add(a, b): return a + b\n```")
+        self.assertFalse(result.valid)
+        self.assertEqual(result.failure_code, "code_wrong_language")
+
+    def test_rejects_required_symbol_missing(self):
+        result = validate_coding_output(
+            "```ts\nexport function subtract(a: number, b: number): number { return a - b; }\n```",
+            required_symbols=["add"],
+        )
+        self.assertFalse(result.valid)
+        self.assertEqual(result.failure_code, "required_symbol_missing")
+        self.assertEqual(result.evidence["missing_symbols"], ["add"])
+
+    def test_repaired_typescript_output_passes_contract(self):
+        result = validate_coding_output(
+            "```typescript\nexport function add(a: number, b: number): number {\n  return a + b;\n}\n```",
+            required_symbols=["add"],
+            min_significant_lines=3,
+        )
+        self.assertTrue(result.valid)
+        self.assertIsNone(result.failure_code)
+
+    def test_retry_is_requested_for_missing_typescript_before_last_attempt(self):
+        result = validate_coding_retry(
+            "code_correct: implementa TypeScript",
+            "La spiegazione non contiene codice.",
+            is_last_attempt=False,
+        )
+        self.assertIsNotNone(result)
+        self.assertEqual(result.failure_code, "code_missing")
+
+    def test_retry_is_not_requested_on_last_attempt(self):
+        result = validate_coding_retry(
+            "code_correct: implementa TypeScript",
+            "La spiegazione non contiene codice.",
+            is_last_attempt=True,
+        )
+        self.assertIsNone(result)
+
+    def test_retry_is_not_requested_for_non_coding_goal(self):
+        result = validate_coding_retry(
+            "Scrivi una spiegazione concettuale",
+            "La spiegazione non contiene codice.",
+            is_last_attempt=False,
+        )
+        self.assertIsNone(result)
+
+    def test_retry_is_not_requested_for_valid_typescript(self):
+        result = validate_coding_retry(
+            "code_correct: implementa TypeScript",
+            "```typescript\nexport const add = (a: number, b: number): number => a + b;\n```",
+            is_last_attempt=False,
+        )
+        self.assertIsNone(result)
+
+    def test_rejects_placeholder_implementation(self):
+        result = validate_coding_output(
+            "```typescript\nexport function add(a: number, b: number): number {\n  // TODO implement here\n  return 0;\n}\n```",
+            required_symbols=["add"],
+        )
+        self.assertFalse(result.valid)
+        self.assertEqual(result.failure_code, "code_placeholder")
+
+    def test_rejects_non_typescript_prose_inside_fence(self):
+        result = validate_coding_output("```typescript\nThis is only explanatory prose.\n```")
+        self.assertFalse(result.valid)
+        self.assertEqual(result.failure_code, "code_syntax_suspect")
+
+
+class ReasoningValidatorTests(unittest.TestCase):
+    def test_accepts_gsm8k_contract_with_thousands_separator(self):
+        result = validate_reasoning_output(
+            "Somma i valori: 100 + 125 = 225.\n#### 225",
+            expected=225,
+        )
+        self.assertTrue(result.valid)
+        self.assertEqual(result.normalized, "225")
+        self.assertTrue(result.evidence["correct"])
+
+    def test_accepts_labeled_final_answer(self):
+        result = validate_reasoning_output("I passaggi portano al totale. Final answer: 2,250", expected=2250)
+        self.assertTrue(result.valid)
+        self.assertEqual(result.normalized, "2250")
+
+    def test_classifies_wrong_numeric_answer(self):
+        result = validate_reasoning_output("Calcolo completo. #### 250", expected=225)
+        self.assertFalse(result.valid)
+        self.assertEqual(result.failure_code, "wrong_numeric_answer")
+        self.assertFalse(result.evidence["correct"])
+
+    def test_classifies_missing_numeric_answer(self):
+        result = validate_reasoning_output("La spiegazione termina senza un numero finale.", expected=225)
+        self.assertFalse(result.valid)
+        self.assertEqual(result.failure_code, "answer_missing")
+
+    def test_classifies_conflicting_explicit_answers(self):
+        result = validate_reasoning_output("#### 250\nFinal answer: 225", expected=225)
+        self.assertFalse(result.valid)
+        self.assertEqual(result.failure_code, "calculation_conflict")
+        self.assertEqual(result.evidence["distinct_candidates"], [250, 225])
+
+    def test_reasoning_retry_is_requested_for_wrong_answer_before_last_attempt(self):
+        result = validate_reasoning_retry(
+            "reasoning GSM8K: risolvi il problema",
+            "#### 250",
+            expected=225,
+            is_last_attempt=False,
+        )
+        self.assertIsNotNone(result)
+        self.assertEqual(result.failure_code, "wrong_numeric_answer")
+
+    def test_reasoning_retry_is_not_requested_on_last_attempt(self):
+        result = validate_reasoning_retry(
+            "reasoning GSM8K: risolvi il problema",
+            "#### 250",
+            expected=225,
+            is_last_attempt=True,
+        )
+        self.assertIsNone(result)
+
+    def test_reasoning_retry_is_not_requested_for_non_reasoning_goal(self):
+        result = validate_reasoning_retry(
+            "Implementa un componente TypeScript",
+            "#### 250",
+            expected=225,
+            is_last_attempt=False,
+        )
+        self.assertIsNone(result)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_model_watch_adapter.py

@@ -0,0 +1,192 @@
+import unittest
+
+import httpx
+
+from benchmarks.model_watch_adapter import (
+    CatalogStatus,
+    GeminiModelsAdapter,
+    ObserveOnlyModelsAdapter,
+    ProviderProfile,
+    models_url,
+)
+
+
+class ModelWatchAdapterTests(unittest.IsolatedAsyncioTestCase):
+    def profile(self, **overrides):
+        values = {
+            "provider": "groq",
+            "profile": "A",
+            "base_url": "https://api.example.test/openai/v1",
+            "api_key": "secret-not-logged",
+            "default_model": "openai/gpt-oss-120b",
+        }
+        values.update(overrides)
+        return ProviderProfile(**values)
+
+    def adapter(self, handler):
+        transport = httpx.MockTransport(handler)
+        return ObserveOnlyModelsAdapter(client=httpx.AsyncClient(transport=transport))
+
+    async def test_catalog_available_and_default_present(self):
+        async def handler(request):
+            self.assertEqual(request.url.path, "/openai/v1/models")
+            self.assertEqual(request.headers["Authorization"], "Bearer secret-not-logged")
+            return httpx.Response(200, json={"data": [{"id": "openai/gpt-oss-120b"}, {"id": "other"}]})
+
+        adapter = self.adapter(handler)
+        result = await adapter.list_models(self.profile())
+        await adapter._client.aclose()
+        self.assertEqual(result.status, CatalogStatus.AVAILABLE)
+        self.assertTrue(result.default_available)
+        self.assertFalse(result.should_auto_apply)
+        self.assertEqual(result.as_audit_record()["model_count"], 2)
+
+    async def test_unauthorized_never_suggests_apply(self):
+        async def handler(_request):
+            return httpx.Response(401, json={"error": "invalid key"})
+
+        adapter = self.adapter(handler)
+        result = await adapter.list_models(self.profile())
+        await adapter._client.aclose()
+        self.assertEqual(result.status, CatalogStatus.UNAUTHORIZED)
+        self.assertEqual(result.http_status, 401)
+        self.assertFalse(result.should_auto_apply)
+
+    async def test_forbidden_is_distinct_from_unauthorized(self):
+        async def handler(_request):
+            return httpx.Response(403, json={"error": "forbidden"})
+
+        adapter = self.adapter(handler)
+        result = await adapter.list_models(self.profile())
+        await adapter._client.aclose()
+        self.assertEqual(result.status, CatalogStatus.FORBIDDEN)
+
+    async def test_rate_limit_preserves_retry_after_without_secret(self):
+        async def handler(_request):
+            return httpx.Response(
+                429,
+                headers={"Retry-After": "37"},
+                json={"error": "quota exceeded", "key": "must-not-be-recorded"},
+            )
+
+        adapter = self.adapter(handler)
+        result = await adapter.list_models(self.profile())
+        await adapter._client.aclose()
+        self.assertEqual(result.status, CatalogStatus.RATE_LIMITED)
+        self.assertEqual(result.retry_after_seconds, 37)
+        self.assertNotIn("must-not-be-recorded", result.detail)
+
+    async def test_server_error_is_provider_error(self):
+        async def handler(_request):
+            return httpx.Response(503, text="temporarily unavailable")
+
+        adapter = self.adapter(handler)
+        result = await adapter.list_models(self.profile())
+        await adapter._client.aclose()
+        self.assertEqual(result.status, CatalogStatus.PROVIDER_ERROR)
+        self.assertEqual(result.http_status, 503)
+
+    async def test_malformed_catalog_is_not_empty_catalog(self):
+        async def handler(_request):
+            return httpx.Response(200, json={"models": [{"id": "x"}]})
+
+        adapter = self.adapter(handler)
+        result = await adapter.list_models(self.profile())
+        await adapter._client.aclose()
+        self.assertEqual(result.status, CatalogStatus.MALFORMED)
+        self.assertIsNone(result.default_available)
+
+    async def test_timeout_is_classified(self):
+        async def handler(_request):
+            raise httpx.ReadTimeout("provider timeout")
+
+        adapter = self.adapter(handler)
+        result = await adapter.list_models(self.profile())
+        await adapter._client.aclose()
+        self.assertEqual(result.status, CatalogStatus.TIMEOUT)
+
+    async def test_query_key_auth_does_not_use_bearer(self):
+        async def handler(request):
+            self.assertEqual(request.url.params.get("key"), "secret-not-logged")
+            self.assertNotIn("authorization", request.headers)
+            return httpx.Response(200, json=[{"id": "gemini-3.6-flash"}])
+
+        adapter = self.adapter(handler)
+        result = await adapter.list_models(self.profile(auth_mode="query_key", default_model="gemini-3.6-flash"))
+        await adapter._client.aclose()
+        self.assertEqual(result.status, CatalogStatus.AVAILABLE)
+        self.assertTrue(result.default_available)
+
+
+class ModelWatchUrlTests(unittest.TestCase):
+    def test_normalizes_completion_url(self):
+        self.assertEqual(models_url("https://x/v1/chat/completions"), "https://x/v1/models")
+
+    def test_keeps_existing_models_suffix(self):
+        self.assertEqual(models_url("https://x/v1/models"), "https://x/v1/models")
+
+
+if __name__ == "__main__":
+    unittest.main()
+
+
+class GeminiModelsAdapterTests(unittest.IsolatedAsyncioTestCase):
+    def profile(self, profile="A", default_model="gemini-3.6-flash"):
+        return ProviderProfile(
+            provider="gemini",
+            profile=profile,
+            base_url="https://generativelanguage.googleapis.com/v1beta",
+            api_key="gemini-secret-not-logged",
+            default_model=default_model,
+            auth_mode="query_key",
+        )
+
+    async def test_native_models_payload_is_parsed_and_prefix_removed(self):
+        async def handler(request):
+            self.assertEqual(request.url.params.get("key"), "gemini-secret-not-logged")
+            return httpx.Response(200, json={"models": [
+                {"name": "models/gemini-3.6-flash"},
+                {"name": "models/gemini-3.5-flash"},
+            ]})
+
+        client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+        result = await GeminiModelsAdapter(client=client).list_models(self.profile())
+        await client.aclose()
+        self.assertEqual(result.status, CatalogStatus.AVAILABLE)
+        self.assertEqual(result.models, ("gemini-3.6-flash", "gemini-3.5-flash"))
+        self.assertTrue(result.default_available)
+        self.assertEqual(result.metadata["catalog_format"], "gemini_native")
+
+    async def test_rate_limited_gemini_profile_is_classified(self):
+        async def handler(_request):
+            return httpx.Response(429, headers={"Retry-After": "60"}, json={"error": "quota"})
+
+        client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+        result = await GeminiModelsAdapter(client=client).list_models(self.profile())
+        await client.aclose()
+        self.assertEqual(result.status, CatalogStatus.RATE_LIMITED)
+        self.assertEqual(result.retry_after_seconds, 60)
+
+    async def test_scan_profiles_skips_429_but_keeps_healthy_profiles(self):
+        calls = []
+
+        async def handler(request):
+            profile = request.url.params.get("profile")
+            calls.append(request.url.path)
+            if len(calls) == 1:
+                return httpx.Response(429, json={"error": "quota"})
+            return httpx.Response(200, json={"models": [{"name": "models/gemini-3.6-flash"}]})
+
+        client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+        adapter = GeminiModelsAdapter(client=client)
+        from benchmarks.model_watch_adapter import scan_profiles
+        scan = await scan_profiles([self.profile("A"), self.profile("B")], adapter=adapter)
+        await client.aclose()
+        self.assertEqual(len(scan.skipped_rate_limited), 1)
+        self.assertEqual(scan.skipped_rate_limited[0].metadata["skip_reason"], "rate_limited")
+        self.assertEqual(len(scan.results), 1)
+        self.assertEqual(scan.results[0].status, CatalogStatus.AVAILABLE)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_provider_model_defaults.py

@@ -23,6 +23,9 @@ class ProviderModelDefaultsTests(unittest.TestCase):
         self.assertEqual(defaults["gemini"], "gemini-3.6-flash")
         self.assertEqual(defaults["cerebras"], "gpt-oss-120b")
         self.assertEqual(defaults["openrouter"], "openai/gpt-oss-20b:free")
+        self.assertEqual(defaults["sambanova"], "DeepSeek-V3.1")
+        self.assertEqual(defaults["hf_router"], "Qwen/Qwen2.5-Coder-32B-Instruct")
+        self.assertEqual(defaults["nvidia"], "nvidia/nemotron-3-ultra-550b-a55b")
 
     def test_retired_provider_fallbacks_are_not_reintroduced(self):
         defaults = set(self._provider_defaults().values())
@@ -30,6 +33,7 @@ class ProviderModelDefaultsTests(unittest.TestCase):
         self.assertNotIn("gemini-2.0-flash-exp", defaults)
         self.assertNotIn("llama-4-scout", defaults)
         self.assertNotIn("meta-llama/llama-4-scout:free", defaults)
+        self.assertNotIn("DeepSeek-V3.2", defaults)
 
 
 if __name__ == "__main__":

tests/test_provider_profile_pool.py

@@ -1,6 +1,7 @@
 import json
 import os
 import types
+import time
 import unittest
 from unittest.mock import AsyncMock, patch
 
@@ -79,11 +80,30 @@ class ProviderProfilePoolTests(unittest.TestCase):
         self.assertEqual(client._execution_pool(profiles, "coding")[0].profile, "a")
         self.assertEqual(client._execution_pool(profiles, "coding")[0].profile, "b")
         client._record_failure(profiles[1], RuntimeError("HTTP 429 rate limit"))
-        client._record_failure(profiles[1], RuntimeError("HTTP 429 rate limit"))
         self.assertFalse(client._is_available(profiles[1]))
         selected = client._execution_pool(profiles, "coding")
         self.assertNotEqual(selected[0].profile, "b")
 
+    def test_rate_limit_reset_opens_profile_on_first_error(self):
+        client = AIClient()
+        profile = self._profiles()[0]
+        client._record_failure(
+            profile,
+            RuntimeError("429 free-models-per-day X-RateLimit-Reset: 4102444800000"),
+        )
+        self.assertFalse(client._is_available(profile))
+        self.assertGreater(
+            client._breaker[profile.identity]["open_until"],
+            time.monotonic() + 900,
+        )
+
+    def test_all_openrouter_profiles_are_removed_from_execution_pool(self):
+        client = AIClient()
+        profiles = self._profiles()
+        for profile in profiles:
+            client._record_failure(profile, RuntimeError("HTTP 429 free-models-per-day"))
+        self.assertEqual(client._execution_pool(profiles, "coding"), [])
+
 
 class InterProviderFallbackChatTests(unittest.IsolatedAsyncioTestCase):
     async def test_chat_falls_back_when_primary_pool_returns_errors(self):

tests/test_shadow_telemetry.py

@@ -0,0 +1,64 @@
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from benchmarks.shadow_telemetry import validate_and_record_shadow
+
+
+class ShadowTelemetryTests(unittest.TestCase):
+    def test_disabled_mode_does_not_write_or_validate(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "shadow.jsonl"
+            with patch.dict(os.environ, {"BENCHMARK_SHADOW_MODE": "0", "BENCHMARK_SHADOW_LOG_PATH": str(path)}, clear=False):
+                result = validate_and_record_shadow(goal="MMLU domanda A/B/C/D", answer="ANSWER: C")
+            self.assertIsNone(result)
+            self.assertFalse(path.exists())
+
+    def test_mmlu_failure_event_is_recorded_without_raw_answer(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "shadow.jsonl"
+            raw = "RISPOSTA SEGRETA: il testo non deve essere salvato"
+            with patch.dict(os.environ, {"BENCHMARK_SHADOW_MODE": "1", "BENCHMARK_SHADOW_LOG_PATH": str(path)}, clear=False):
+                result = validate_and_record_shadow(
+                    goal="MMLU domanda di informatica A/B/C/D",
+                    answer=raw,
+                    metadata={"provider": "mock", "model": "test", "secret": "must-drop"},
+                )
+            self.assertIsNotNone(result)
+            self.assertEqual(result.failure_code, "answer_missing")
+            event = json.loads(path.read_text(encoding="utf-8"))
+            self.assertEqual(event["category"], "mmlu")
+            self.assertEqual(event["failure_code"], "answer_missing")
+            self.assertEqual(event["metadata"], {"provider": "mock", "model": "test"})
+            self.assertNotIn(raw, path.read_text(encoding="utf-8"))
+
+    def test_coding_failure_event_is_recorded(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "shadow.jsonl"
+            with patch.dict(os.environ, {"BENCHMARK_SHADOW_MODE": "true", "BENCHMARK_SHADOW_LOG_PATH": str(path)}, clear=False):
+                result = validate_and_record_shadow(
+                    goal="code_correct: implementa in TypeScript",
+                    answer="Non posso includere il codice.",
+                    metadata={"attempt": 1, "latency_ms": 123.4},
+                )
+            self.assertIsNotNone(result)
+            self.assertEqual(result.failure_code, "code_missing")
+            event = json.loads(path.read_text(encoding="utf-8"))
+            self.assertEqual(event["validator"], "coding_v1")
+            self.assertEqual(event["metadata"]["attempt"], 1)
+            self.assertEqual(event["metadata"]["latency_ms"], 123.4)
+
+    def test_unsupported_category_is_ignored(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "shadow.jsonl"
+            with patch.dict(os.environ, {"BENCHMARK_SHADOW_MODE": "1", "BENCHMARK_SHADOW_LOG_PATH": str(path)}, clear=False):
+                result = validate_and_record_shadow(goal="generic task", answer="output")
+            self.assertIsNone(result)
+            self.assertFalse(path.exists())
+
+
+if __name__ == "__main__":
+    unittest.main()