sync: 189 file da Baida98/AI@f527eaa5 (2026-08-26 09:08 UTC) [deploy-all]

benchmark-extended.mjs

@@ -46,7 +46,8 @@
 
 import { spawn }     from "child_process";
 import { writeFileSync, mkdirSync, rmSync, existsSync } from "fs";
-import { join }      from "path";
+import { join, dirname } from "path";
+import { fileURLToPath } from "url";
 
 // ── CLI args ──────────────────────────────────────────────────────────────────
 const _A = process.argv.slice(2);
@@ -80,8 +81,25 @@ const G="\x1b[32m",R="\x1b[31m",Y="\x1b[33m",B="\x1b[34m",
 const _baseUrlArg = _A.find(a=>a.startsWith("--base-url="));
 const BASE_URL  = (_baseUrlArg ? _baseUrlArg.slice("--base-url=".length) : (process.env.BENCHMARK_BASE_URL ?? process.env.BACKEND_URL ?? "https://baida07-terminal.hf.space")).replace(/\/+$/, "");
 const TASK_DIR  = "/tmp/bench-ext/tasks";
-const LOCAL_TSC = join(process.cwd(),"node_modules/.pnpm/typescript@5.9.3/node_modules/typescript/bin/tsc");
-const TSC_BIN   = process.env.TSC_BIN || (existsSync(LOCAL_TSC)?LOCAL_TSC:"tsc");
+const RUNNER_DIR = dirname(fileURLToPath(import.meta.url));
+let TSC_BIN = "";
+
+async function resolveTypeScriptCompiler(){
+  const configured = String(process.env.TSC_BIN ?? "").trim();
+  const candidates = configured
+    ? [configured]
+    : [
+      join(RUNNER_DIR, "node_modules", ".bin", "tsc"),
+      join(process.cwd(), "node_modules", ".bin", "tsc"),
+      join(process.cwd(), "node_modules", ".pnpm", "typescript@5.9.3", "node_modules", "typescript", "bin", "tsc"),
+    ];
+  for(const candidate of candidates){
+    if(candidate.includes("/") && !existsSync(candidate)) continue;
+    const probe = await runCmd(candidate, ["--version"], process.cwd(), 8_000);
+    if(probe.exitCode === 0) return candidate;
+  }
+  return "";
+}
 const HF_GSM8K  = 1319;   // openai/gsm8k test split size
 const HF_BBH    = 250;    // lukaemon/bbh logical_deduction size
 const HF_SCIQ   = 1000;   // allenai/sciq test split size
@@ -874,13 +892,34 @@ async function fetchSciQ(seed){
 
     // ── callAgent ─────────────────────────────────────────────────────────────────
 async function callAgent(goal,timeoutMs=90000,options={}){
-  const t0=Date.now(); let out="",engine="?",provider="?",model="?",ttfa=null,toolCalls=0,done=false,failed=false,failureReason="",taskId="",lastEvent="",lastEventAt=null;
-  const telemetry=()=>({provider:provider||"?",model:model||engine||"?",ttfaMs:ttfa??9999,lastEvent:lastEvent||null,lastEventAt});
+  const t0=Date.now();
+  let out="",engine="?",provider="?",model="?",ttfa=null,toolCalls=0,done=false,failed=false,failureReason="",taskId="",lastEvent="",lastEventAt=null;
+  let lastEventId=0,streamReconnects=0,heartbeatCount=0,statusChecks=0,replayedEventCount=0,terminalStatus=null,cancelAttempted=false,cancelHttpStatus=null;
+  const seenEventIds=new Set();
+  const telemetry=()=>({
+    taskId:taskId||null,provider:provider||"?",model:model||engine||"?",ttfaMs:ttfa??9999,
+    lastEvent:lastEvent||null,lastEventAt,lastEventId,streamReconnects,heartbeatCount,statusChecks,replayedEventCount,
+    terminalStatus,cancelAttempted,cancelHttpStatus,
+  });
   const internalToken=process.env.INTERNAL_TOKEN||"";
   if(!internalToken)return{ok:false,output:"",engine,...telemetry(),toolCalls,durationMs:0,failed:true,failureReason:"INTERNAL_TOKEN mancante"};
   const ctrl=new AbortController();
   const timer=setTimeout(()=>ctrl.abort(),timeoutMs);
   const headers={"Content-Type":"application/json","X-Internal-Token":internalToken};
+  const taskHeaders={"X-Internal-Token":internalToken};
+  const cancelTask=async()=>{
+    if(!taskId||cancelAttempted)return;
+    cancelAttempted=true;
+    try{const response=await fetch(`${BASE_URL}/api/agent/tasks/${encodeURIComponent(taskId)}`,{method:"DELETE",headers:taskHeaders});cancelHttpStatus=response.status;}catch{cancelHttpStatus=0;}
+  };
+  const readStatus=async()=>{
+    statusChecks++;
+    const response=await fetch(`${BASE_URL}/api/agent/tasks/${encodeURIComponent(taskId)}/status`,{method:"GET",headers:taskHeaders,signal:ctrl.signal});
+    if(!response.ok)throw new Error(`status HTTP ${response.status}`);
+    const body=await response.json();
+    terminalStatus=String(body.status||"UNKNOWN");
+    return terminalStatus;
+  };
   try{
     const created=await fetch(`${BASE_URL}/api/agent/tasks`,{
       method:"POST",headers,
@@ -894,13 +933,6 @@ async function callAgent(goal,timeoutMs=90000,options={}){
     taskId=String(createdBody.taskId||"");
     if(!taskId)return{ok:false,output:"",engine,...telemetry(),toolCalls,durationMs:Date.now()-t0,failed:true,failureReason:"taskId mancante"};
 
-    const res=await fetch(`${BASE_URL}/api/agent/tasks/${encodeURIComponent(taskId)}/stream`,{
-      method:"GET",headers:{"X-Internal-Token":internalToken},signal:ctrl.signal});
-    if(!res.ok)return{ok:false,output:"",engine,...telemetry(),toolCalls,durationMs:Date.now()-t0,failed:true,failureReason:`stream HTTP ${res.status}`};
-    if(!res.body)return{ok:false,output:"",engine,...telemetry(),toolCalls,durationMs:Date.now()-t0,failed:true,failureReason:"stream body mancante"};
-    const dec=new TextDecoder(); let buf="";
-    const rd=res.body.getReader();
-    let stopStream=false;
     let sawText=false;
     const rawPayloadLog=[];
     const processSSEData=(raw)=>{
@@ -923,11 +955,7 @@ async function callAgent(goal,timeoutMs=90000,options={}){
             sawText=true;
             if(!(trimmed.startsWith("{")&&trimmed.endsWith("}")))out+=chunk;
             ttfa=ttfa??now;
-            // Per feature il validator lavora sul blocco TS: non attendere task_done
-            // quando il modello ha già prodotto un blocco completo e sufficientemente lungo.
-            if(options.earlyComplete === "typescript" && /```(?:typescript|ts)\s*[\\s\\S]{80,}?```/i.test(out)){
-              done=true; return true;
-            }
+            if(options.earlyComplete === "typescript" && /```(?:typescript|ts)\s*[\\s\\S]{80,}?```/i.test(out)){done=true;return true;}
           }
         } else if(eventType==="task_error"){
           failed=true;failureReason=String(ev.error||"task_error");done=true;return true;
@@ -937,43 +965,68 @@ async function callAgent(goal,timeoutMs=90000,options={}){
           if(ev.success===false||result.startsWith("[LLM_UNAVAILABLE]")||result.includes("tutti i provider configurati sono falliti")){
             failed=true;failureReason=result||"provider_unavailable";engine=ev.engine??engine;done=true;return true;
           }
-          // Il backend può inviare un result stale o non coerente al termine dello stream.
-          // Se abbiamo ricevuto text_chunk, il buffer SSE è la fonte autorevole.
           if(result&&(!sawText||result.length>(out||"").length)&&!sawText)out=result;
-          engine=ev.engine??engine;done=true;return true;
+          engine=ev.engine??engine;terminalStatus="SUCCESS";done=true;return true;
         }
       }catch{}
       return false;
     };
-    while(!stopStream){
-      const{done:d,value}=await rd.read();
-      if(d){
-        buf+=dec.decode();
-        break;
-      }
-      buf+=dec.decode(value,{stream:true});
-      const lines=buf.split(/\r?\n/); buf=lines.pop()??"";
-      for(const ln of lines){
-        if(!ln.startsWith("data:"))continue;
-        if(processSSEData(ln.slice(5).trim())){stopStream=true;break;}
-      }
-    }
-    // A proxy may close after a final unterminated data line.
-    if(!stopStream&&buf.trim().startsWith("data:"))processSSEData(buf.trim().slice(5).trim());
-    rd.cancel?.();
-    if(options.debugRaw){try{writeFileSync("/tmp/code_correct_retry_sse.log",JSON.stringify({timestamp:new Date().toISOString(),sawText,outputLength:out.length,events:rawPayloadLog},null,2));}catch{}}
-    if(done && !sawText && !out){
-      return{ok:false,output:"",engine,...telemetry(),toolCalls,durationMs:Date.now()-t0,failed:true,failureReason:"NO_OUTPUT"};
+    const consumeStream=async()=>{
+      const streamHeaders={...taskHeaders};
+      if(lastEventId>0)streamHeaders["Last-Event-ID"]=String(lastEventId);
+      const response=await fetch(`${BASE_URL}/api/agent/tasks/${encodeURIComponent(taskId)}/stream`,{method:"GET",headers:streamHeaders,signal:ctrl.signal});
+      if(!response.ok)throw new Error(`stream HTTP ${response.status}`);
+      if(!response.body)throw new Error("stream body mancante");
+      const dec=new TextDecoder();let buf="";const rd=response.body.getReader();let stopStream=false,skipReplayData=false;
+      try{
+        while(!stopStream){
+          const{done:closed,value}=await rd.read();
+          if(closed){buf+=dec.decode();break;}
+          buf+=dec.decode(value,{stream:true});
+          const lines=buf.split(/\r?\n/);buf=lines.pop()??"";
+          for(const line of lines){
+            if(line.startsWith("id:")){
+              const id=Number(line.slice(3).trim());
+              if(Number.isInteger(id)&&id>0){
+                skipReplayData=seenEventIds.has(id);
+                if(skipReplayData)replayedEventCount++;
+                else seenEventIds.add(id);
+                if(id>lastEventId)lastEventId=id;
+              }
+              continue;
+            }
+            if(line.startsWith(":")){heartbeatCount++;continue;}
+            if(!line.startsWith("data:"))continue;
+            if(skipReplayData){skipReplayData=false;continue;}
+            if(processSSEData(line.slice(5).trim())){stopStream=true;break;}
+          }
+        }
+        if(!stopStream&&buf.trim().startsWith("data:"))processSSEData(buf.trim().slice(5).trim());
+      }finally{try{await rd.cancel();}catch{}}
+    };
+
+    // Il backend supporta resume con Last-Event-ID: un drop del trasporto viene
+    // riallacciato allo stesso task, mai rieseguito in silenzio.
+    while(!done&&!failed&&streamReconnects<=1){
+      await consumeStream();
+      if(done||failed)break;
+      await readStatus();
+      if(streamReconnects>=1)break;
+      streamReconnects++;
+      await new Promise(resolve=>setTimeout(resolve,250));
     }
-    if(done && options.earlyComplete && taskId){
-      try{await fetch(`${BASE_URL}/api/agent/tasks/${encodeURIComponent(taskId)}`,{method:"DELETE",headers:{"X-Internal-Token":internalToken}});}catch{}
+    if(!done&&!failed){
+      failureReason=`SSE_INCOMPLETE${terminalStatus?`_${terminalStatus}`:""}`;
+      failed=true;
+      if(!["SUCCESS","ERROR","CANCELLED"].includes(String(terminalStatus)))await cancelTask();
     }
+    if(options.debugRaw){try{writeFileSync("/tmp/code_correct_retry_sse.log",JSON.stringify({timestamp:new Date().toISOString(),sawText,outputLength:out.length,events:rawPayloadLog,telemetry:telemetry()},null,2));}catch{}}
+    if(done&&!failed&&!sawText&&!out)return{ok:false,output:"",engine,...telemetry(),toolCalls,durationMs:Date.now()-t0,failed:true,failureReason:"NO_OUTPUT"};
+    if(done&&options.earlyComplete)await cancelTask();
   }catch(e){
-    if(taskId){
-      try{await fetch(`${BASE_URL}/api/agent/tasks/${encodeURIComponent(taskId)}`,{method:"DELETE",headers:{"X-Internal-Token":internalToken}});}catch{}
-    }
-    const partial = normalizeAgentOutput(out || "");
-    const salvageFeature = options.earlyComplete === "typescript" && partial.length >= 120 && /(?:interface|type|class|function|const)\b/.test(partial) && /(?:subscribe|getState|async|await|try|catch)/.test(partial);
+    await cancelTask();
+    const partial=normalizeAgentOutput(out||"");
+    const salvageFeature=options.earlyComplete === "typescript"&&partial.length>=120&&/(?:interface|type|class|function|const)\b/.test(partial)&&/(?:subscribe|getState|async|await|try|catch)/.test(partial);
     return{ok:salvageFeature,output:partial,engine,...telemetry(),toolCalls,durationMs:Date.now()-t0,failed:!salvageFeature,failureReason:e.name==="AbortError"?(salvageFeature?"PARTIAL_TIMEOUT":"TIMEOUT"):String(e.message||e),partial:salvageFeature};
   }finally{clearTimeout(timer);}
   const finalOutput=normalizeAgentOutput(out);
@@ -2300,6 +2353,13 @@ async function runOneSeed(seed,opts={}){
   const n=selected.length;
   const hfN=selected.filter(t=>t.hfSource&&!t.hfSource.startsWith("local")).length;
   const ghN=[_ghEval,_ghSQL,_ghAdvisory].filter(Boolean).length;
+  const requiresTypeScript = selected.some(task => task.category === "code_correct");
+  if(requiresTypeScript){
+    TSC_BIN = await resolveTypeScriptCompiler();
+    if(!TSC_BIN){
+      throw new Error("BENCHMARK_PREREQUISITE_TYPESCRIPT: esegui pnpm install oppure imposta TSC_BIN su un compilatore TypeScript compatibile prima di misurare code_correct.");
+    }
+  }
 
   _log(`\n${B}${BOLD}╔══════════════════════════════════════════════════════════════════════╗${NC}`);
   _log(`${B}${BOLD}║  EXTENDED BENCHMARK v5  —  seed: ${String(seed).padEnd(12)}                    ║${NC}`);
@@ -2354,7 +2414,7 @@ async function runOneSeed(seed,opts={}){
         score:null,ref:task.ref,buildPassed:null,testsPassed:null,
         planScore:null,executionScore:null,recoveryScore:null,autonomyScore:null,
         agentMs,ttfa:agent.ttfa??9999,ttfaMs:agent.ttfaMs??agent.ttfa??9999,toolCalls:agent.toolCalls??0,tokEstimate:0,
-        engine:agent.engine,provider:agent.provider??"?",model:agent.model??agent.engine??"?",lastEvent:agent.lastEvent??null,lastEventAt:agent.lastEventAt??null,breakdown:null,detail:`non valutabile: ${reason}`,unavailable:true,failureReason:reason});
+        engine:agent.engine,provider:agent.provider??"?",model:agent.model??agent.engine??"?",lastEvent:agent.lastEvent??null,lastEventAt:agent.lastEventAt??null,lastEventId:agent.lastEventId??0,taskId:agent.taskId??null,streamReconnects:agent.streamReconnects??0,heartbeatCount:agent.heartbeatCount??0,statusChecks:agent.statusChecks??0,replayedEventCount:agent.replayedEventCount??0,terminalStatus:agent.terminalStatus??null,cancelAttempted:agent.cancelAttempted??false,cancelHttpStatus:agent.cancelHttpStatus??null,breakdown:null,detail:`non valutabile: ${reason}`,unavailable:true,failureReason:reason});
       continue;
     }
 
@@ -2393,7 +2453,7 @@ async function runOneSeed(seed,opts={}){
       score,ref,buildPassed:vr.buildPassed,testsPassed:vr.testsPassed,
       planScore:vr.planScore,executionScore:vr.executionScore,recoveryScore:vr.recoveryScore,
       agentMs,ttfa:agent.ttfa??9999,ttfaMs:agent.ttfaMs??agent.ttfa??9999,toolCalls:agent.toolCalls??0,tokEstimate:tokEst,
-      engine:agent.engine,provider:agent.provider??"?",model:agent.model??agent.engine??"?",lastEvent:agent.lastEvent??null,lastEventAt:agent.lastEventAt??null,breakdown,detail:(vr.detail||"").slice(0,200),failureReason:agent.failureReason||null});
+      engine:agent.engine,provider:agent.provider??"?",model:agent.model??agent.engine??"?",lastEvent:agent.lastEvent??null,lastEventAt:agent.lastEventAt??null,lastEventId:agent.lastEventId??0,taskId:agent.taskId??null,streamReconnects:agent.streamReconnects??0,heartbeatCount:agent.heartbeatCount??0,statusChecks:agent.statusChecks??0,replayedEventCount:agent.replayedEventCount??0,terminalStatus:agent.terminalStatus??null,cancelAttempted:agent.cancelAttempted??false,cancelHttpStatus:agent.cancelHttpStatus??null,breakdown,detail:(vr.detail||"").slice(0,200),failureReason:agent.failureReason||null});
   }
 
   // ── Summary ────────────────────────────────────────────────────────────────
@@ -2470,6 +2530,8 @@ async function runOneSeed(seed,opts={}){
       F_robustezza:"robustness",G_operatività:"coding+data_analysis"},
     methodology:{
       runtime_input:{profile:"fixed-realistic-chat-v1",persona:BENCHMARK_PERSONA,negative_constraints:true,context_messages:BENCHMARK_CONTEXT.length},
+      runner_prerequisites:{typescript_required:requiresTypeScript,typescript_bin:requiresTypeScript?TSC_BIN:null},
+      sse_recovery:{resume:"Last-Event-ID",deduplicate_replayed_event_ids:true,max_reconnects:1,status_endpoint:"/api/agent/tasks/{taskId}/status",cancel_on_nonterminal_incomplete:true},
       canonical_seed:"1337 (stesse domande per tutti gli agenti — usa --rotate per seed diverso)",
       coding:"enterprise: Acc(35%)+Stab(20%)+Auto(15%)+Perf(10%)+Spd(10%)+Cost(5%)+Tool(5%)",
       nonCoding:"content: Acc(40%)+Struct(20%)+Comp(15%)+Prec(10%)+Auto(5%)+Spd(5%)+Cost(5%)",