sync: 157 file da Baida98/AI@5bb711cb (2026-08-15 12:51 UTC) [deploy-all]

api/benchmark_handler.py

@@ -145,24 +145,41 @@ def _format_v7_report(report: dict[str, Any]) -> str:
     if canary:
         lines.append(f"⚠️ <b>Canary leak:</b> {canary} task\n")
 
-    # Score per categoria
+    # Score per categoria. Una categoria in timeout resta tentata ma non entra
+    # nella media: non va trasformata silenziosamente in uno score pari a zero.
     tasks = report.get("tasks", [])
     if tasks:
+        attempted_categories = {str(t.get("cat")) for t in tasks if t.get("cat")}
         by_cat: dict[str, list[float]] = {}
         for t in tasks:
             cat = t.get("cat", "?")
             sc  = t.get("score")
             if isinstance(sc, (int, float)):
                 by_cat.setdefault(cat, []).append(float(sc))
+        attempted = s.get("attemptedTaskCount", len(tasks))
+        scored = s.get("scoredTaskCount", sum(len(v) for v in by_cat.values()))
+        skipped = s.get("skippedTaskCount", max(0, attempted - scored))
+        lines.append(
+            f"🧪 <b>Copertura:</b> <code>{len(attempted_categories)}/20 categorie tentate · "
+            f"{scored} valutabili · {skipped} non valutabili</code>\n"
+        )
         if by_cat:
-            coverage = len(by_cat)
-            lines.append(f"🧪 <b>Copertura:</b> <code>{coverage}/20 categorie · {len(tasks)} task</code>\n")
             lines.append("\n📂 <b>Per categoria:</b>\n")
             for cat, scores in sorted(by_cat.items()):
                 avg_cat = sum(scores) / len(scores)
                 icon = "🟢" if avg_cat >= 70 else "🟡" if avg_cat >= 50 else "🔴"
                 lines.append(f"  {icon} <code>{avg_cat:5.1f}</code>  {cat}\n")
 
+    failures = report.get("taskFailures", [])
+    if failures:
+        lines.append("\n⚠️ <b>Categorie non valutabili:</b>\n")
+        for failure in failures[:3]:
+            cat = failure.get("cat", "?")
+            reason = str(failure.get("reason", "errore non specificato"))[:100]
+            lines.append(f"  • <code>{cat}</code> — {reason}\n")
+        if len(failures) > 3:
+            lines.append(f"  <i>...e altre {len(failures) - 3}.</i>\n")
+
     # Gap cards (prime 3)
     gap_cards = report.get("gapCards", [])
     if gap_cards:

benchmark-extended.mjs

@@ -2042,6 +2042,7 @@ async function runOneSeed(seed,opts={}){
   try{spaceV=(await fetch(`${BASE_URL}/api/state/health`).then(r=>r.json())).version??"";}catch{}
 
   const results=[];
+  const taskFailures=[];
   for(let i=0;i<n;i++){
     const task=selected[i];
     const isNC=!!task.isNonCoding,isAG=!!task.isAgentic;
@@ -2055,7 +2056,17 @@ async function runOneSeed(seed,opts={}){
     const agent=await callAgentWithRetry(task,Math.max(task.targetMs||65000,180000));
     const agentMs=Date.now()-t0;
     if(agent.failed){
-      throw new Error(`Benchmark non valido: backend non ha completato ${task.id}/${task.category} (${agent.failureReason||"errore sconosciuto"}).`);
+      const reason=String(agent.failureReason||"errore sconosciuto").slice(0,240);
+      taskFailures.push({id:task.id,cat:task.category,label:task.label.slice(0,60),reason,agentMs});
+      _log(`     ${Y}N/A ${NC} ${DIM}categoria non valutabile: ${reason.slice(0,80)} — continuo con le restanti${NC}`);
+      results.push({id:task.id,cat:task.category,label:task.label.slice(0,60),
+        isCoding:!!task.isCoding,isNonCoding:isNC,isAgentic:isAG,
+        hfSource:task.hfSource||"local",hfOffset:task.hfOffset??null,
+        score:null,ref:task.ref,buildPassed:null,testsPassed:null,
+        planScore:null,executionScore:null,recoveryScore:null,autonomyScore:null,
+        agentMs,ttfa:agent.ttfa??9999,toolCalls:agent.toolCalls??0,tokEstimate:0,
+        engine:agent.engine,breakdown:null,detail:`non valutabile: ${reason}`,unavailable:true,failureReason:reason});
+      continue;
     }
 
     let vr;
@@ -2098,30 +2109,35 @@ async function runOneSeed(seed,opts={}){
 
   // ── Summary ────────────────────────────────────────────────────────────────
   const avg  =r=>r.length?+(r.reduce((a,b)=>a+b,0)/r.length).toFixed(1):null;
-  const scores=results.map(r=>r.score);
+  const scoredResults=results.filter(r=>typeof r.score==="number"&&Number.isFinite(r.score));
+  const scores=scoredResults.map(r=>r.score);
   const avgS=avg(scores);
-  const avgR=avg(results.map(r=>r.ref.replit));
-  const avgC=avg(results.map(r=>r.ref.cursor));
-  const avgD=avg(results.map(r=>r.ref.devin));
-  const avgM=avg(results.map(r=>r.ref.manus));
-  const wins_r=results.filter(r=>r.score>=r.ref.replit).length;
-  const wins_d=results.filter(r=>r.score>=r.ref.devin).length;
-  const wins_m=results.filter(r=>r.score>=r.ref.manus).length;
-  const mft   =avg(results.map(r=>r.agentMs/1000));
-  const tools =avg(results.map(r=>r.toolCalls));
-  const hfUsed=results.filter(r=>r.hfSource&&!r.hfSource.startsWith("local")).length;
+  const avgR=avg(scoredResults.map(r=>r.ref.replit));
+  const avgC=avg(scoredResults.map(r=>r.ref.cursor));
+  const avgD=avg(scoredResults.map(r=>r.ref.devin));
+  const avgM=avg(scoredResults.map(r=>r.ref.manus));
+  const wins_r=scoredResults.filter(r=>r.score>=r.ref.replit).length;
+  const wins_d=scoredResults.filter(r=>r.score>=r.ref.devin).length;
+  const wins_m=scoredResults.filter(r=>r.score>=r.ref.manus).length;
+  const mft   =avg(scoredResults.map(r=>r.agentMs/1000));
+  const tools =avg(scoredResults.map(r=>r.toolCalls));
+  const hfUsed=scoredResults.filter(r=>r.hfSource&&!r.hfSource.startsWith("local")).length;
 
   _log(`\n${BOLD}${"─".repeat(78)}${NC}`);
   _log(`${DIM}  ${`Categoria`.padEnd(19)} ${`AI`.padEnd(6)} ${`Replit`.padEnd(7)} ${`Cursor`.padEnd(7)} ${`Devin`.padEnd(7)} ${`Manus`.padEnd(7)} Δ${NC}`);
   for(const r of results){
-    const dc=r.score>=r.ref.manus?G:r.score>=r.ref.devin?G:r.score>=r.ref.replit?Y:R;
     const tag=r.isAgentic?`${M}[AG]${NC}`:r.isNonCoding?`${C}[NC]${NC}`:`${Y}[C] ${NC}`;
     const hf=r.hfSource&&!r.hfSource.startsWith("local")?`${B}*${NC}`:" ";
+    if(typeof r.score!=="number"){
+      _log(`  ${tag}${hf}${r.cat.padEnd(17)} ${Y}N/A   ${NC}${DIM}non valutabile: ${(r.failureReason||"errore").slice(0,42)}${NC}`);
+      continue;
+    }
+    const dc=r.score>=r.ref.manus?G:r.score>=r.ref.devin?G:r.score>=r.ref.replit?Y:R;
     _log(`  ${tag}${hf}${r.cat.padEnd(17)} ${dc}${String(r.score).padEnd(6)}${NC}${String(r.ref.replit).padEnd(7)}${String(r.ref.cursor).padEnd(7)}${String(r.ref.devin).padEnd(7)}${String(r.ref.manus).padEnd(7)}${dc}${r.score-r.ref.replit>=0?"+":""}${r.score-r.ref.replit}${NC}`);
   }
 
   // Gap analysis
-  const gaps=results
+  const gaps=scoredResults
     .filter(r=>r.score<r.ref.replit)
     .map(r=>buildGapCard(selected.find(t=>t.id===r.id&&t.category===r.cat),r,r.score,r.ref))
     .sort((a,b)=>a.priorita-b.priorita);
@@ -2135,13 +2151,14 @@ async function runOneSeed(seed,opts={}){
     }
   }
 
-  const nodeMap=buildNodeMap(results);
-  const verdict=avgS>=avgM?"SUPERA_MANUS":avgS>=avgD?"SUPERA_DEVIN":avgS>=avgR?"PARI_REPLIT":"SOTTO_REPLIT";
-  const vc=avgS>=avgM?G:avgS>=avgD?G:avgS>=avgR?Y:R;
+  const nodeMap=buildNodeMap(scoredResults);
+  const verdict=avgS===null?"NON_VALUTABILE":avgS>=avgM?"SUPERA_MANUS":avgS>=avgD?"SUPERA_DEVIN":avgS>=avgR?"PARI_REPLIT":"SOTTO_REPLIT";
+  const vc=avgS===null?Y:avgS>=avgM?G:avgS>=avgD?G:avgS>=avgR?Y:R;
 
   _log(`\n${BOLD}${"═".repeat(78)}${NC}`);
-  _log(`  ${BOLD}AI Agent:  ${vc}${avgS}/100${NC}  vs Replit:${wins_r}/${n}  vs Devin:${wins_d}/${n}  vs Manus:${wins_m}/${n}`);
+  _log(`  ${BOLD}AI Agent:  ${vc}${avgS??"N/A"}/100${NC}  vs Replit:${wins_r}/${scoredResults.length}  vs Devin:${wins_d}/${scoredResults.length}  vs Manus:${wins_m}/${scoredResults.length}`);
   _log(`  ${DIM}Refs — Replit:${avgR}  Cursor:${avgC}  Devin:${avgD}  Manus:${avgM}${NC}`);
+  if(taskFailures.length)_log(`  ${Y}Categorie non valutabili: ${taskFailures.length}/${n}${NC}`);
   _log(`  ${BOLD}HF tasks:  ${hfUsed}/${n} reali  mft:${mft}s  tools:${tools}${NC}`);
   _log(`  ${BOLD}Gap cards: ${gaps.length} task sotto Replit${NC}`);
   _log(`  ${BOLD}Verdetto:  ${vc}${verdict.replace(/_/g," ")}${NC}`);
@@ -2169,13 +2186,14 @@ async function runOneSeed(seed,opts={}){
     },
     summary:{avgScore:avgS,avgReplit:avgR,avgCursor:avgC,avgDevin:avgD,avgManus:avgM,
       wins_replit:wins_r,wins_devin:wins_d,wins_manus:wins_m,
+      attemptedTaskCount:n,scoredTaskCount:scoredResults.length,skippedTaskCount:taskFailures.length,
       mft,hfTasksUsed:hfUsed,gapCount:gaps.length,verdict},
-    gapCards:gaps,orchestrationNodeMap:nodeMap,tasks:results,
+    taskFailures,gapCards:gaps,orchestrationNodeMap:nodeMap,tasks:results,
   };
   wsf(reportPath,JSON.stringify(report,null,2));
   _log(`${DIM}✓ Report: ${reportPath}${NC}`);
   if(F_JSON||OUTFILE){wsf(OUTFILE??"/tmp/agente-ai/benchmark-v5-latest.json",JSON.stringify(report,null,2));}
-  return{seed,avgScore:avgS,avgReplit:avgR,avgDevin:avgD,avgManus:avgM,wins_r,wins_d,wins_m,mft,gaps,tasks:results,selectedTasks:selected};
+  return{seed,avgScore:avgS,avgReplit:avgR,avgDevin:avgD,avgManus:avgM,wins_r,wins_d,wins_m,mft,gaps,taskFailures,tasks:results,selectedTasks:selected};
 }
 
 // ── Compare mode con CI95 per categoria ───────────────────────────────────────