sync: 190 file da Baida98/AI@12bc67f1 (2026-08-26 12:40 UTC) [deploy-all]

benchmark-extended.mjs

@@ -77,12 +77,25 @@ const JUDGE_TELEMETRY = {
   model: process.env.GROQ_API_KEY ? "openai/gpt-oss-120b" : process.env.CEREBRAS_API_KEY ? "llama-3.3-70b" : null,
   lastFailure: null,
   failureReasons: {},
+  attempts: 0,
+  retries: 0,
+  recoveredRetries: 0,
 };
 const recordJudgeFailure = (reason) => {
   JUDGE_TELEMETRY.failed++;
   JUDGE_TELEMETRY.lastFailure = reason;
   JUDGE_TELEMETRY.failureReasons[reason] = (JUDGE_TELEMETRY.failureReasons[reason] ?? 0) + 1;
 };
+const classifyJudgeHttpFailure = (status, raw, isGroq) => {
+  const text = String(raw || "").toLowerCase();
+  if (status === 401 || status === 403) return "JUDGE_AUTH_REJECTED";
+  if (status === 429 || /rate.?limit|too many requests/.test(text)) return "JUDGE_RATE_LIMIT";
+  if (status >= 500) return "JUDGE_PROVIDER_5XX";
+  if (isGroq && status === 400 && /json.?schema|response_format|strict/.test(text)) return "GROQ_SCHEMA_REJECTED";
+  if (isGroq && status === 400 && /reasoning_effort|include_reasoning|max_completion_tokens|parameter/.test(text)) return "GROQ_PARAMETER_REJECTED";
+  if (status >= 400 && status < 500) return "JUDGE_REQUEST_REJECTED";
+  return `HTTP_${status}`;
+};
 const _categoriesArg = _A.find(a=>a.startsWith("--categories="));
 const TARGET_CATEGORIES = _categoriesArg
   ? new Set(_categoriesArg.slice("--categories=".length).split(",").map(c=>c.trim()).filter(Boolean))
@@ -95,6 +108,9 @@ const G="\x1b[32m",R="\x1b[31m",Y="\x1b[33m",B="\x1b[34m",
 // ── Constants ─────────────────────────────────────────────────────────────────
 const _baseUrlArg = _A.find(a=>a.startsWith("--base-url="));
 const BASE_URL  = (_baseUrlArg ? _baseUrlArg.slice("--base-url=".length) : (process.env.BENCHMARK_BASE_URL ?? process.env.BACKEND_URL ?? "https://baida07-terminal.hf.space")).replace(/\/+$/, "");
+const TEST_TRANSPORT_TIMEOUT_MS = process.env.BENCHMARK_TEST_TRANSPORT_TIMEOUT_MS && process.env.NODE_ENV === "test"
+  ? Math.max(50, Number(process.env.BENCHMARK_TEST_TRANSPORT_TIMEOUT_MS) || 0)
+  : null;
 const TASK_DIR  = "/tmp/bench-ext/tasks";
 const RUNNER_DIR = dirname(fileURLToPath(import.meta.url));
 let TSC_BIN = "";
@@ -836,25 +852,46 @@ async function judgeWithLLM(question, response, rubric) {
     };
   }
   JUDGE_TELEMETRY.requested++;
-  try {
-    const r = await fetch(endpoint, {
-      method: "POST",
-      headers: { "Authorization": `Bearer ${apiKey}`, "Content-Type": "application/json" },
-      body: JSON.stringify(request),
-      signal: AbortSignal.timeout(12_000),
-    });
-    if (!r.ok) {
-      recordJudgeFailure(`HTTP_${r.status}`);
+  const executeJudge = async (payload, isFallback = false) => {
+    JUDGE_TELEMETRY.attempts++;
+    let r;
+    try {
+      r = await fetch(endpoint, {
+        method: "POST",
+        headers: { "Authorization": `Bearer ${apiKey}`, "Content-Type": "application/json" },
+        body: JSON.stringify(payload),
+        signal: AbortSignal.timeout(12_000),
+      });
+    } catch (error) {
+      recordJudgeFailure(error?.name === "AbortError" ? "JUDGE_TIMEOUT" : "JUDGE_REQUEST_ERROR");
       return null;
     }
-    const data = await r.json();
+    if (!r.ok) {
+      const raw = await r.text().catch(() => "");
+      const reason = classifyJudgeHttpFailure(r.status, raw, isGroq);
+      recordJudgeFailure(reason);
+      const recoverable = isGroq && !isFallback && (reason === "GROQ_SCHEMA_REJECTED" || reason === "GROQ_PARAMETER_REJECTED");
+      if (!recoverable) return null;
+      JUDGE_TELEMETRY.retries++;
+      const fallback = { ...payload, response_format: { type: "json_object" } };
+      delete fallback.reasoning_effort;
+      delete fallback.include_reasoning;
+      const recovered = await executeJudge(fallback, true);
+      if (recovered) JUDGE_TELEMETRY.recoveredRetries++;
+      return recovered;
+    }
+    let data;
+    try { data = await r.json(); }
+    catch { recordJudgeFailure("JUDGE_INVALID_JSON_RESPONSE"); return null; }
     const content = data.choices?.[0]?.message?.content ?? "";
     const m = content.match(/\{[^}]+\}/);
     if (!m) {
       recordJudgeFailure("JUDGE_EMPTY_OR_NON_JSON");
       return null;
     }
-    const scores = JSON.parse(m[0]);
+    let scores;
+    try { scores = JSON.parse(m[0]); }
+    catch { recordJudgeFailure("JUDGE_INVALID_SCORE_JSON"); return null; }
     for (const d of dims) {
       if (typeof scores[d] !== "number") {
         recordJudgeFailure("JUDGE_SCHEMA_MISMATCH");
@@ -862,14 +899,14 @@ async function judgeWithLLM(question, response, rubric) {
       }
       scores[d] = Math.max(0, Math.min(1, +scores[d].toFixed(2)));
     }
-    JUDGE_TELEMETRY.succeeded++;
-    JUDGE_TELEMETRY.lastFailure = null;
-    _JUDGE_CACHE.set(cacheKey, scores);
     return scores;
-  } catch (error) {
-    recordJudgeFailure(error?.name === "AbortError" ? "JUDGE_TIMEOUT" : "JUDGE_REQUEST_ERROR");
-    return null;
-  }
+  };
+  const scores = await executeJudge(request);
+  if (!scores) return null;
+  JUDGE_TELEMETRY.succeeded++;
+  JUDGE_TELEMETRY.lastFailure = null;
+  _JUDGE_CACHE.set(cacheKey, scores);
+  return scores;
 }
 
 function tDir(id){
@@ -2485,7 +2522,7 @@ async function runOneSeed(seed,opts={}){
     const t0=Date.now();
     // Il target resta una metrica di punteggio; non è un hard-stop di trasporto.
     // I fallback gratuiti possono richiedere più tempo per il primo chunk su task coding.
-    const transportTimeout = (task.category === "feature" || task.category === "research_synthesis") ? 150000 : Math.max(task.targetMs||65000,180000);
+    const transportTimeout = TEST_TRANSPORT_TIMEOUT_MS ?? ((task.category === "feature" || task.category === "research_synthesis") ? 150000 : Math.max(task.targetMs||65000,180000));
     let agent=await callAgentWithRetry(task,transportTimeout);
     const repair=await repairSecurityIfNeeded(task,agent,Math.min(transportTimeout,120000));
     agent=repair.agent;
@@ -2628,7 +2665,7 @@ async function runOneSeed(seed,opts={}){
       runtime_input:{profile:"fixed-realistic-chat-v1",persona:BENCHMARK_PERSONA,negative_constraints:true,context_messages:BENCHMARK_CONTEXT.length},
       runner_prerequisites:{typescript_required:requiresTypeScript,typescript_bin:requiresTypeScript?TSC_BIN:null},
       sse_recovery:{resume:"Last-Event-ID",deduplicate_replayed_event_ids:true,max_reconnects:1,status_endpoint:"/api/agent/tasks/{taskId}/status",cancel_on_nonterminal_incomplete:true},
-      semantic_judge:{enabled:F_JUDGE,provider:JUDGE_TELEMETRY.provider,model:JUDGE_TELEMETRY.model,requested:JUDGE_TELEMETRY.requested,succeeded:JUDGE_TELEMETRY.succeeded,failed:JUDGE_TELEMETRY.failed,cache_hits:JUDGE_TELEMETRY.cacheHits,last_failure:JUDGE_TELEMETRY.lastFailure,failure_reasons:JUDGE_TELEMETRY.failureReasons,coverage_complete:JUDGE_TELEMETRY.requested===JUDGE_TELEMETRY.succeeded},
+      semantic_judge:{enabled:F_JUDGE,provider:JUDGE_TELEMETRY.provider,model:JUDGE_TELEMETRY.model,requested:JUDGE_TELEMETRY.requested,succeeded:JUDGE_TELEMETRY.succeeded,failed:JUDGE_TELEMETRY.failed,attempts:JUDGE_TELEMETRY.attempts,retries:JUDGE_TELEMETRY.retries,recovered_retries:JUDGE_TELEMETRY.recoveredRetries,cache_hits:JUDGE_TELEMETRY.cacheHits,last_failure:JUDGE_TELEMETRY.lastFailure,failure_reasons:JUDGE_TELEMETRY.failureReasons,coverage_complete:JUDGE_TELEMETRY.requested===JUDGE_TELEMETRY.succeeded},
       canonical_seed:"1337 (stesse domande per tutti gli agenti — usa --rotate per seed diverso)",
       coding:"enterprise: Acc(35%)+Stab(20%)+Auto(15%)+Perf(10%)+Spd(10%)+Cost(5%)+Tool(5%)",
       nonCoding:"content: Acc(40%)+Struct(20%)+Comp(15%)+Prec(10%)+Auto(5%)+Spd(5%)+Cost(5%)",