sync: 190 file da Baida98/AI@b589d1e9 (2026-08-26 10:21 UTC) [deploy-all]

benchmark-extended.mjs

@@ -68,6 +68,15 @@ const F_GAP       = _A.includes("--gap-analysis");
 const _multi  = _A.indexOf("--multi");
 const MULTI   = _multi !== -1 ? Math.max(2, Math.min(10, parseInt(_A[_multi+1])||3)) : 1;
 const F_JUDGE = !_A.includes("--no-judge"); // semantic judge abilitato di default
+const JUDGE_TELEMETRY = {
+  requested: 0,
+  succeeded: 0,
+  failed: 0,
+  cacheHits: 0,
+  provider: process.env.GROQ_API_KEY ? "groq" : process.env.CEREBRAS_API_KEY ? "cerebras" : null,
+  model: process.env.GROQ_API_KEY ? "openai/gpt-oss-120b" : process.env.CEREBRAS_API_KEY ? "llama-3.3-70b" : null,
+  lastFailure: null,
+};
 const _categoriesArg = _A.find(a=>a.startsWith("--categories="));
 const TARGET_CATEGORIES = _categoriesArg
   ? new Set(_categoriesArg.slice("--categories=".length).split(",").map(c=>c.trim()).filter(Boolean))
@@ -753,44 +762,84 @@ const _JUDGE_CACHE = new Map();
 async function judgeWithLLM(question, response, rubric) {
   if (!F_JUDGE) return null;
   const apiKey = process.env.GROQ_API_KEY || process.env.CEREBRAS_API_KEY;
-  if (!apiKey) return null;
+  if (!apiKey) {
+    JUDGE_TELEMETRY.lastFailure = "JUDGE_API_KEY_MISSING";
+    return null;
+  }
   const cacheKey = question.slice(0,40) + response.slice(0,40);
-  if (_JUDGE_CACHE.has(cacheKey)) return _JUDGE_CACHE.get(cacheKey);
+  if (_JUDGE_CACHE.has(cacheKey)) {
+    JUDGE_TELEMETRY.cacheHits++;
+    return _JUDGE_CACHE.get(cacheKey);
+  }
   const isGroq = !!process.env.GROQ_API_KEY;
   const endpoint = isGroq
-    ? "https://api.groq.com/openai/v1/chat/completions"
+    ? (process.env.BENCHMARK_GROQ_BASE_URL || "https://api.groq.com/openai/v1/chat/completions")
     : "https://api.cerebras.ai/v1/chat/completions";
   const model = isGroq ? "openai/gpt-oss-120b" : "llama-3.3-70b";
   const dims = Object.keys(rubric);
   const rubricText = dims.map(d => `- ${d} (0.0-1.0): ${rubric[d]}`).join("\n");
+  const scoreSchema = {
+    type: "object",
+    properties: Object.fromEntries(dims.map(d => [d, { type: "number", minimum: 0, maximum: 1 }])),
+    required: dims,
+    additionalProperties: false,
+  };
+  const request = {
+    model,
+    messages: [
+      { role: "system", content: "Strict benchmark evaluator. Return only the requested numeric JSON object." },
+      { role: "user", content: `TASK: ${question.slice(0,350)}\n\nRESPONSE: ${response.slice(0,1200)}\n\nScore each 0.0-1.0:\n${rubricText}` }
+    ],
+    temperature: 0,
+    max_completion_tokens: 256,
+  };
+  if (isGroq) {
+    request.reasoning_effort = "low";
+    request.include_reasoning = false;
+    request.response_format = {
+      type: "json_schema",
+      json_schema: { name: "benchmark_scores", strict: true, schema: scoreSchema },
+    };
+  }
+  JUDGE_TELEMETRY.requested++;
   try {
     const r = await fetch(endpoint, {
       method: "POST",
       headers: { "Authorization": `Bearer ${apiKey}`, "Content-Type": "application/json" },
-      body: JSON.stringify({
-        model,
-        messages: [
-          { role: "system", content: "Strict benchmark evaluator. Return ONLY valid JSON, no prose." },
-          { role: "user", content: `TASK: ${question.slice(0,350)}\n\nRESPONSE: ${response.slice(0,1200)}\n\nScore each 0.0-1.0:\n${rubricText}\n\nJSON only: {${dims.map(d=>`"${d}":0.0`).join(",")}}` }
-        ],
-        temperature: 0.05,
-        max_tokens: 120,
-      }),
-      signal: AbortSignal.timeout(9000),
+      body: JSON.stringify(request),
+      signal: AbortSignal.timeout(12_000),
     });
-    if (!r.ok) return null;
+    if (!r.ok) {
+      JUDGE_TELEMETRY.failed++;
+      JUDGE_TELEMETRY.lastFailure = `HTTP_${r.status}`;
+      return null;
+    }
     const data = await r.json();
     const content = data.choices?.[0]?.message?.content ?? "";
     const m = content.match(/\{[^}]+\}/);
-    if (!m) return null;
+    if (!m) {
+      JUDGE_TELEMETRY.failed++;
+      JUDGE_TELEMETRY.lastFailure = "JUDGE_EMPTY_OR_NON_JSON";
+      return null;
+    }
     const scores = JSON.parse(m[0]);
     for (const d of dims) {
-      if (typeof scores[d] !== "number") return null;
+      if (typeof scores[d] !== "number") {
+        JUDGE_TELEMETRY.failed++;
+        JUDGE_TELEMETRY.lastFailure = "JUDGE_SCHEMA_MISMATCH";
+        return null;
+      }
       scores[d] = Math.max(0, Math.min(1, +scores[d].toFixed(2)));
     }
+    JUDGE_TELEMETRY.succeeded++;
+    JUDGE_TELEMETRY.lastFailure = null;
     _JUDGE_CACHE.set(cacheKey, scores);
     return scores;
-  } catch { return null; }
+  } catch (error) {
+    JUDGE_TELEMETRY.failed++;
+    JUDGE_TELEMETRY.lastFailure = error?.name === "AbortError" ? "JUDGE_TIMEOUT" : "JUDGE_REQUEST_ERROR";
+    return null;
+  }
 }
 
 function tDir(id){
@@ -2532,6 +2581,7 @@ async function runOneSeed(seed,opts={}){
       runtime_input:{profile:"fixed-realistic-chat-v1",persona:BENCHMARK_PERSONA,negative_constraints:true,context_messages:BENCHMARK_CONTEXT.length},
       runner_prerequisites:{typescript_required:requiresTypeScript,typescript_bin:requiresTypeScript?TSC_BIN:null},
       sse_recovery:{resume:"Last-Event-ID",deduplicate_replayed_event_ids:true,max_reconnects:1,status_endpoint:"/api/agent/tasks/{taskId}/status",cancel_on_nonterminal_incomplete:true},
+      semantic_judge:{enabled:F_JUDGE,provider:JUDGE_TELEMETRY.provider,model:JUDGE_TELEMETRY.model,requested:JUDGE_TELEMETRY.requested,succeeded:JUDGE_TELEMETRY.succeeded,failed:JUDGE_TELEMETRY.failed,cache_hits:JUDGE_TELEMETRY.cacheHits,last_failure:JUDGE_TELEMETRY.lastFailure},
       canonical_seed:"1337 (stesse domande per tutti gli agenti — usa --rotate per seed diverso)",
       coding:"enterprise: Acc(35%)+Stab(20%)+Auto(15%)+Perf(10%)+Spd(10%)+Cost(5%)+Tool(5%)",
       nonCoding:"content: Acc(40%)+Struct(20%)+Comp(15%)+Prec(10%)+Auto(5%)+Spd(5%)+Cost(5%)",