sync: 190 file da Baida98/AI@977dd543 (2026-08-26 10:41 UTC) [deploy-all]

benchmark-extended.mjs

@@ -76,6 +76,12 @@ const JUDGE_TELEMETRY = {
   provider: process.env.GROQ_API_KEY ? "groq" : process.env.CEREBRAS_API_KEY ? "cerebras" : null,
   model: process.env.GROQ_API_KEY ? "openai/gpt-oss-120b" : process.env.CEREBRAS_API_KEY ? "llama-3.3-70b" : null,
   lastFailure: null,
+  failureReasons: {},
+};
+const recordJudgeFailure = (reason) => {
+  JUDGE_TELEMETRY.failed++;
+  JUDGE_TELEMETRY.lastFailure = reason;
+  JUDGE_TELEMETRY.failureReasons[reason] = (JUDGE_TELEMETRY.failureReasons[reason] ?? 0) + 1;
 };
 const _categoriesArg = _A.find(a=>a.startsWith("--categories="));
 const TARGET_CATEGORIES = _categoriesArg
@@ -810,23 +816,20 @@ async function judgeWithLLM(question, response, rubric) {
       signal: AbortSignal.timeout(12_000),
     });
     if (!r.ok) {
-      JUDGE_TELEMETRY.failed++;
-      JUDGE_TELEMETRY.lastFailure = `HTTP_${r.status}`;
+      recordJudgeFailure(`HTTP_${r.status}`);
       return null;
     }
     const data = await r.json();
     const content = data.choices?.[0]?.message?.content ?? "";
     const m = content.match(/\{[^}]+\}/);
     if (!m) {
-      JUDGE_TELEMETRY.failed++;
-      JUDGE_TELEMETRY.lastFailure = "JUDGE_EMPTY_OR_NON_JSON";
+      recordJudgeFailure("JUDGE_EMPTY_OR_NON_JSON");
       return null;
     }
     const scores = JSON.parse(m[0]);
     for (const d of dims) {
       if (typeof scores[d] !== "number") {
-        JUDGE_TELEMETRY.failed++;
-        JUDGE_TELEMETRY.lastFailure = "JUDGE_SCHEMA_MISMATCH";
+        recordJudgeFailure("JUDGE_SCHEMA_MISMATCH");
         return null;
       }
       scores[d] = Math.max(0, Math.min(1, +scores[d].toFixed(2)));
@@ -836,8 +839,7 @@ async function judgeWithLLM(question, response, rubric) {
     _JUDGE_CACHE.set(cacheKey, scores);
     return scores;
   } catch (error) {
-    JUDGE_TELEMETRY.failed++;
-    JUDGE_TELEMETRY.lastFailure = error?.name === "AbortError" ? "JUDGE_TIMEOUT" : "JUDGE_REQUEST_ERROR";
+    recordJudgeFailure(error?.name === "AbortError" ? "JUDGE_TIMEOUT" : "JUDGE_REQUEST_ERROR");
     return null;
   }
 }
@@ -969,6 +971,17 @@ async function callAgent(goal,timeoutMs=90000,options={}){
     terminalStatus=String(body.status||"UNKNOWN");
     return terminalStatus;
   };
+  const readStatusAfterAbort=async()=>{
+    if(!taskId)return null;
+    statusChecks++;
+    try{
+      const response=await fetch(`${BASE_URL}/api/agent/tasks/${encodeURIComponent(taskId)}/status`,{method:"GET",headers:taskHeaders,signal:AbortSignal.timeout(5000)});
+      if(!response.ok)return null;
+      const body=await response.json();
+      terminalStatus=String(body.status||"UNKNOWN");
+      return terminalStatus;
+    }catch{return null;}
+  };
   try{
     const created=await fetch(`${BASE_URL}/api/agent/tasks`,{
       method:"POST",headers,
@@ -1073,10 +1086,13 @@ async function callAgent(goal,timeoutMs=90000,options={}){
     if(done&&!failed&&!sawText&&!out)return{ok:false,output:"",engine,...telemetry(),toolCalls,durationMs:Date.now()-t0,failed:true,failureReason:"NO_OUTPUT"};
     if(done&&options.earlyComplete)await cancelTask();
   }catch(e){
-    await cancelTask();
+    const timedOut=e.name==="AbortError";
+    if(timedOut)await readStatusAfterAbort();
+    if(!["SUCCESS","ERROR","CANCELLED"].includes(String(terminalStatus)))await cancelTask();
     const partial=normalizeAgentOutput(out||"");
     const salvageFeature=options.earlyComplete === "typescript"&&partial.length>=120&&/(?:interface|type|class|function|const)\b/.test(partial)&&/(?:subscribe|getState|async|await|try|catch)/.test(partial);
-    return{ok:salvageFeature,output:partial,engine,...telemetry(),toolCalls,durationMs:Date.now()-t0,failed:!salvageFeature,failureReason:e.name==="AbortError"?(salvageFeature?"PARTIAL_TIMEOUT":"TIMEOUT"):String(e.message||e),partial:salvageFeature};
+    const timeoutReason=terminalStatus?`TIMEOUT_${terminalStatus}`:"TIMEOUT";
+    return{ok:salvageFeature,output:partial,engine,...telemetry(),toolCalls,durationMs:Date.now()-t0,failed:!salvageFeature,failureReason:timedOut?(salvageFeature?`PARTIAL_${timeoutReason}`:timeoutReason):String(e.message||e),partial:salvageFeature};
   }finally{clearTimeout(timer);}
   const finalOutput=normalizeAgentOutput(out);
   return{ok:finalOutput.length>30&&!failed,output:finalOutput,engine,...telemetry(),toolCalls,durationMs:Date.now()-t0,failed,failureReason};
@@ -2581,7 +2597,7 @@ async function runOneSeed(seed,opts={}){
       runtime_input:{profile:"fixed-realistic-chat-v1",persona:BENCHMARK_PERSONA,negative_constraints:true,context_messages:BENCHMARK_CONTEXT.length},
       runner_prerequisites:{typescript_required:requiresTypeScript,typescript_bin:requiresTypeScript?TSC_BIN:null},
       sse_recovery:{resume:"Last-Event-ID",deduplicate_replayed_event_ids:true,max_reconnects:1,status_endpoint:"/api/agent/tasks/{taskId}/status",cancel_on_nonterminal_incomplete:true},
-      semantic_judge:{enabled:F_JUDGE,provider:JUDGE_TELEMETRY.provider,model:JUDGE_TELEMETRY.model,requested:JUDGE_TELEMETRY.requested,succeeded:JUDGE_TELEMETRY.succeeded,failed:JUDGE_TELEMETRY.failed,cache_hits:JUDGE_TELEMETRY.cacheHits,last_failure:JUDGE_TELEMETRY.lastFailure},
+      semantic_judge:{enabled:F_JUDGE,provider:JUDGE_TELEMETRY.provider,model:JUDGE_TELEMETRY.model,requested:JUDGE_TELEMETRY.requested,succeeded:JUDGE_TELEMETRY.succeeded,failed:JUDGE_TELEMETRY.failed,cache_hits:JUDGE_TELEMETRY.cacheHits,last_failure:JUDGE_TELEMETRY.lastFailure,failure_reasons:JUDGE_TELEMETRY.failureReasons,coverage_complete:JUDGE_TELEMETRY.requested===JUDGE_TELEMETRY.succeeded},
       canonical_seed:"1337 (stesse domande per tutti gli agenti — usa --rotate per seed diverso)",
       coding:"enterprise: Acc(35%)+Stab(20%)+Auto(15%)+Perf(10%)+Spd(10%)+Cost(5%)+Tool(5%)",
       nonCoding:"content: Acc(40%)+Struct(20%)+Comp(15%)+Prec(10%)+Auto(5%)+Spd(5%)+Cost(5%)",