sync: 157 file da Baida98/AI@eeb3e0e9 (2026-08-15 15:39 UTC) [deploy-all]

api/benchmark_handler.py

@@ -36,6 +36,11 @@ _BENCH_SCRIPT = next(
     os.path.join(_REPO_ROOT, "benchmark-extended.mjs"),
 )
 _REPORT_V7     = "/tmp/agente-ai/benchmark-v5-latest.json"
+_REPORT_V7_WEAK = "/tmp/agente-ai/benchmark-v5-weak-latest.json"
+_WEAK_CATEGORIES = (
+    "sql", "context_window", "reasoning", "data_analysis", "research_synthesis",
+    "mmlu", "technical_writing", "code_correct", "feature", "security",
+)
 # 20 task seriali possono richiedere più di 12 minuti con provider gratuiti.
 _BENCH_TIMEOUT = float(os.getenv("BENCH_TIMEOUT_SECS", "3600"))
 
@@ -50,12 +55,26 @@ async def run_benchmark_task(chat_id: int, send_reply_fn, mode: str = "full") ->
                             "Il deployment non ha incluso <code>benchmark-extended.mjs</code>.")
         return
 
-    flags = ["--full", "--json", f"--output={_REPORT_V7}", "--gap-analysis"]
-    await send_reply_fn(
-        chat_id,
-        "🚀 <b>Benchmark Extended v5 avviato</b>\n"
-        "<i>20/20 categorie · seed 1337 · task API moderna · durata variabile fino a ~60 min.</i>",
-    )
+    is_weak_run = mode == "weak"
+    if is_weak_run:
+        report_path = _REPORT_V7_WEAK
+        flags = [
+            f"--categories={','.join(_WEAK_CATEGORIES)}", "--json",
+            f"--output={report_path}", "--gap-analysis",
+        ]
+        await send_reply_fn(
+            chat_id,
+            "🎯 <b>Benchmark Extended v5 mirato avviato</b>\n"
+            "<i>10 categorie più deboli della baseline 39,1 · seed 1337 · task API moderna.</i>",
+        )
+    else:
+        report_path = _REPORT_V7
+        flags = ["--full", "--json", f"--output={report_path}", "--gap-analysis"]
+        await send_reply_fn(
+            chat_id,
+            "🚀 <b>Benchmark Extended v5 avviato</b>\n"
+            "<i>20/20 categorie · seed 1337 · task API moderna · durata variabile fino a ~60 min.</i>",
+        )
     env = {
         **os.environ,
         "INTERNAL_TOKEN": os.getenv("INTERNAL_TOKEN", ""),
@@ -96,25 +115,26 @@ async def run_benchmark_task(chat_id: int, send_reply_fn, mode: str = "full") ->
         await send_reply_fn(chat_id, f"💥 <b>Errore critico benchmark:</b> <code>{exc}</code>")
         return
 
-    report_exists = await asyncio.to_thread(os.path.exists, _REPORT_V7)
+    report_exists = await asyncio.to_thread(os.path.exists, report_path)
     if not report_exists:
         await send_reply_fn(chat_id, "⚠️ <b>Benchmark Extended terminato ma report non trovato.</b>")
         return
     try:
-        report: dict[str, Any] = await asyncio.to_thread(_read_json, _REPORT_V7)
+        report: dict[str, Any] = await asyncio.to_thread(_read_json, report_path)
     except Exception as exc:
         await send_reply_fn(chat_id, f"⚠️ <b>Report Extended non leggibile:</b> <code>{exc}</code>")
         return
 
     categories = {str(task.get("cat", "")) for task in report.get("tasks", []) if task.get("cat")}
-    if len(categories) != 20:
-        await send_reply_fn(chat_id, f"⚠️ <b>Run incompleta:</b> <code>{len(categories)}/20</code> categorie nel report."
+    expected_categories = len(_WEAK_CATEGORIES) if is_weak_run else 20
+    if len(categories) != expected_categories:
+        await send_reply_fn(chat_id, f"⚠️ <b>Run incompleta:</b> <code>{len(categories)}/{expected_categories}</code> categorie nel report."
                             " Nessun risultato incompleto viene presentato come benchmark completo.")
         return
-    await send_reply_fn(chat_id, _format_v7_report(report))
+    await send_reply_fn(chat_id, _format_v7_report(report, expected_categories=expected_categories, run_label="mirato · categorie deboli" if is_weak_run else None))
 
 
-def _format_v7_report(report: dict[str, Any]) -> str:
+def _format_v7_report(report: dict[str, Any], *, expected_categories: int = 20, run_label: str | None = None) -> str:
     """Formatta il report v7 per Telegram HTML."""
     s   = report.get("summary", {})
     ts  = (report.get("timestamp") or "")[:16].replace("T", " ")
@@ -132,7 +152,8 @@ def _format_v7_report(report: dict[str, Any]) -> str:
     lines: list[str] = [
         f"🏆 <b>Benchmark {ver} completato!</b>\n\n"
         f"📊 <b>Score agente:</b> <code>{avg}/100</code>\n"
-        f"📅 <b>Run:</b> <code>{ts}</code>\n\n"
+        f"📅 <b>Run:</b> <code>{ts}</code>\n"
+        + (f"🎯 <b>Modalità:</b> <code>{run_label}</code>\n" if run_label else "") + "\n"
         "📈 <b>Confronto vs riferimenti:</b>\n"
         f"  • Replit:  <code>{repl}/100</code>\n"
         f"  • Cursor:  <code>{curs}/100</code>\n"
@@ -160,7 +181,7 @@ def _format_v7_report(report: dict[str, Any]) -> str:
         scored = s.get("scoredTaskCount", sum(len(v) for v in by_cat.values()))
         skipped = s.get("skippedTaskCount", max(0, attempted - scored))
         lines.append(
-            f"🧪 <b>Copertura:</b> <code>{len(attempted_categories)}/20 categorie tentate · "
+            f"🧪 <b>Copertura:</b> <code>{len(attempted_categories)}/{expected_categories} categorie tentate · "
             f"{scored} valutabili · {skipped} non valutabili</code>\n"
         )
         if by_cat:

api/telegram_webhook.py

@@ -1623,8 +1623,9 @@ async def _cmd_bench(chat_id: int, mode: str = "default") -> None:
     """Avvia il benchmark Extended su tutte le 20 categorie in background."""
     from .benchmark_handler import run_benchmark_task
 
-    _BENCH_CACHE[chat_id] = {"mode": "extended-20", "run_url": "", "started_at": time.time()}
-    task = asyncio.create_task(run_benchmark_task(chat_id, _tg_reply, mode="full"))
+    normalized_mode = "weak" if mode == "weak" else "full"
+    _BENCH_CACHE[chat_id] = {"mode": "extended-weak" if normalized_mode == "weak" else "extended-20", "run_url": "", "started_at": time.time()}
+    task = asyncio.create_task(run_benchmark_task(chat_id, _tg_reply, mode=normalized_mode))
     task.add_done_callback(lambda completed: _logger.error(
         "[bench-extended] background task failed: %s", completed.exception()
     ) if not completed.cancelled() and completed.exception() else None)
@@ -2058,7 +2059,7 @@ async def telegram_webhook(request: Request) -> dict:
             await _tg_reply(chat_id, "🧠 Uso: <code>/ask &lt;domanda&gt;</code>", keyboard=_MAIN_KB)
     elif cmd == "/bench":
         _mode = text[len(cmd):].strip() or "default"
-        if _mode not in ("default","full","coding-only","noncode-only","agentic-only"):
+        if _mode not in ("default","full","coding-only","noncode-only","agentic-only","weak"):
             _mode = "default"
         _t=asyncio.create_task(_cmd_bench(chat_id, _mode)); _t.add_done_callback(_log_tg_exc)
     elif cmd == "/score":

benchmark-extended.mjs

@@ -35,7 +35,7 @@
  * Usage:
  *   node benchmark-extended.mjs [--seed N] [--compare N]
  *                                [--coding-only] [--noncode-only] [--agentic-only]
- *                                [--full] [--json] [--output=PATH] [--no-hf]
+ *                                [--full] [--categories=a,b] [--json] [--output=PATH] [--no-hf]
  *                                [--gap-analysis]   # stampa gap cards per i fail
  *                                [--rotate]         # seed casuale (default: 1337 canonico)
  *
@@ -67,6 +67,10 @@ const F_GAP       = _A.includes("--gap-analysis");
 const _multi  = _A.indexOf("--multi");
 const MULTI   = _multi !== -1 ? Math.max(2, Math.min(10, parseInt(_A[_multi+1])||3)) : 1;
 const F_JUDGE = !_A.includes("--no-judge"); // semantic judge abilitato di default
+const _categoriesArg = _A.find(a=>a.startsWith("--categories="));
+const TARGET_CATEGORIES = _categoriesArg
+  ? new Set(_categoriesArg.slice("--categories=".length).split(",").map(c=>c.trim()).filter(Boolean))
+  : null;
 
 // ── Colors ────────────────────────────────────────────────────────────────────
 const G="\x1b[32m",R="\x1b[31m",Y="\x1b[33m",B="\x1b[34m",
@@ -2001,8 +2005,15 @@ async function runOneSeed(seed,opts={}){
     ccTask,
   ];
 
+  const allTasks = [...codingTasks,...[rsTask,daTask,twTask,ryTask,sqlTask,cwTask,mmluTask],...[orTask,mcTask,rcTask,rbTask,avTask]];
   let selected;
-  if(F_CODING){
+  if (TARGET_CATEGORIES) {
+    const known = new Set(allTasks.map(task => task.category));
+    const unknown = [...TARGET_CATEGORIES].filter(category => !known.has(category));
+    if (unknown.length) throw new Error(`Categorie benchmark non valide: ${unknown.join(", ")}`);
+    selected = allTasks.filter(task => TARGET_CATEGORIES.has(task.category));
+    if (!selected.length) throw new Error("Nessuna categoria benchmark selezionata");
+  } else if(F_CODING){
     selected=rng.shuffle(codingTasks).slice(0,7);
   } else if(F_NONCODE){
     selected=[rsTask,daTask,twTask,ryTask,sqlTask,cwTask,mmluTask];

tests/test_telegram_extended_benchmark.py

@@ -50,5 +50,37 @@ class ExtendedBenchmarkHandlerTests(unittest.IsolatedAsyncioTestCase):
         self.assertIn("20/20 categorie", final_text)
 
 
+    async def test_targeted_weak_runner_uses_only_low_score_categories_and_separate_report(self):
+        categories = list(handler._WEAK_CATEGORIES)
+        report = {
+            "timestamp": "2026-08-15T12:00:00Z",
+            "version": "extended-v5",
+            "summary": {"avgScore": 31, "avgReplit": 56, "avgCursor": 62, "avgDevin": 68, "avgManus": 71, "gapCount": 8, "verdict": "SOTTO_REPLIT"},
+            "tasks": [{"cat": category, "score": 31} for category in categories],
+            "gapCards": [],
+        }
+        replies = AsyncMock()
+        with tempfile.TemporaryDirectory() as directory:
+            script = Path(directory) / "benchmark-extended.mjs"
+            full_output = Path(directory) / "benchmark-v5-latest.json"
+            weak_output = Path(directory) / "benchmark-v5-weak-latest.json"
+            script.write_text("// runner")
+            full_output.write_text(json.dumps({"tasks": []}))
+            weak_output.write_text(json.dumps(report))
+            with patch.object(handler, "_BENCH_SCRIPT", str(script)), \
+                 patch.object(handler, "_REPORT_V7", str(full_output)), \
+                 patch.object(handler, "_REPORT_V7_WEAK", str(weak_output)), \
+                 patch("asyncio.create_subprocess_exec", AsyncMock(return_value=_Process())) as create_process:
+                await handler.run_benchmark_task(123, replies, mode="weak")
+
+        command = create_process.await_args.args
+        self.assertNotIn("--full", command)
+        self.assertIn("--categories=" + ",".join(handler._WEAK_CATEGORIES), command)
+        self.assertIn("--output=" + str(weak_output), command)
+        final_text = replies.await_args_list[-1].args[1]
+        self.assertIn("10/10 categorie", final_text)
+        self.assertIn("categorie deboli", final_text)
+
+
 if __name__ == "__main__":
     unittest.main()