api/benchmark.py

"""backend/api/benchmark.py — Self-test endpoint server-side (S-BENCH).

Endpoint: GET /api/debug/benchmark?token=<daily_hmac>

Auth: HMAC-SHA256(seed=_BENCH_SEED, msg=YYYY-MM-DD) — calcolabile da Replit
senza dover conoscere INTERNAL_TOKEN. Token cambia ogni giorno (replay protection).
Zero config aggiuntivo su HF Spaces.

Il benchmark esegue ~20 test interni e restituisce JSON con:
  { ok, score, pass, fail, warn, duration_ms, tests: [...], gaps: [...] }

Uso da Replit:
  python3 -c "
  import hmac, hashlib, datetime
  seed = 'agente-ai-bench-2026'
  day  = datetime.date.today().isoformat()
  tok  = hmac.new(seed.encode(), day.encode(), hashlib.sha256).hexdigest()
  print(tok)
  "
  curl 'https://arjanit98-terminal.hf.space/api/debug/benchmark?token=<tok>'
"""
import os, sys, asyncio, time, hmac, hashlib, datetime, importlib, tempfile, subprocess, re, uuid
from fastapi import APIRouter, BackgroundTasks, HTTPException, Query, Request
from fastapi.responses import JSONResponse

router = APIRouter()

# ── Seed per HMAC daily token — NON è un secret, è solo anti-scraping ─────────
_BENCH_SEED = "agente-ai-bench-2026"


def _daily_token() -> str:
    day = datetime.date.today().isoformat()
    return hmac.new(_BENCH_SEED.encode(), day.encode(), hashlib.sha256).hexdigest()


# ── Result helpers ──────────────────────────────────────────────────────────────
def _ok(id: str, desc: str, note: str = "") -> dict:
    return {"id": id, "desc": desc, "ok": True, "warn": False, "note": note}

def _ko(id: str, desc: str, note: str = "") -> dict:
    return {"id": id, "desc": desc, "ok": False, "warn": False, "note": note}

def _wn(id: str, desc: str, note: str = "") -> dict:
    return {"id": id, "desc": desc, "ok": False, "warn": True, "note": note}


async def _run_tests() -> list[dict]:
    results: list[dict] = []
    t_global = time.monotonic()

    # ── T01: Sprint / version ─────────────────────────────────────────────────
    try:
        from api.providers import api_version
        ver = await api_version() if asyncio.iscoroutinefunction(api_version) else api_version()
        sprint = ver.get("sprint", "?") if isinstance(ver, dict) else getattr(ver, "sprint", "?")
        body   = ver if isinstance(ver, dict) else ver.__dict__
        results.append(_ok("T01", f"Sprint: {sprint} — {body.get('version','?')}", f"build={body.get('build_date','?')}"))
    except Exception as e:
        results.append(_ko("T01", "Sprint/version import fallito", str(e)))

    # ── T02: INTERNAL_TOKEN configurato ──────────────────────────────────────
    itok = os.getenv("INTERNAL_TOKEN", "")
    if itok and len(itok) >= 16:
        results.append(_ok("T02", "INTERNAL_TOKEN configurato in env", f"len={len(itok)}"))
    elif itok:
        results.append(_wn("T02", "INTERNAL_TOKEN troppo corto (< 16 chars)", f"len={len(itok)}"))
    else:
        results.append(_wn("T02", "INTERNAL_TOKEN non configurato — token effimero generato a ogni boot"))

    # ── T03: UnifiedAgentLoop import ─────────────────────────────────────────
    t = time.monotonic()
    try:
        from agents.unified_loop import UnifiedAgentLoop
        ms = int((time.monotonic() - t) * 1000)
        results.append(_ok("T03", f"UnifiedAgentLoop import OK ({ms}ms)"))
    except Exception as e:
        ms = int((time.monotonic() - t) * 1000)
        results.append(_ko("T03", "UnifiedAgentLoop import FALLITO", str(e)[:120]))

    # ── T04: RoleRouter + Role.FAST ──────────────────────────────────────────
    t = time.monotonic()
    try:
        from models.role_router import RoleRouter, Role
        fast_role = Role.FAST
        client = RoleRouter.get_client(Role.FAST)
        ms = int((time.monotonic() - t) * 1000)
        provider = getattr(client, "provider_name", type(client).__name__)
        results.append(_ok("T04", f"Role.FAST client OK ({ms}ms) — provider={provider}"))
    except Exception as e:
        ms = int((time.monotonic() - t) * 1000)
        results.append(_ko("T04", f"Role.FAST client FALLITO ({ms}ms)", str(e)[:120]))

    # ── T05: Python exec via asyncio subprocess ───────────────────────────────
    t = time.monotonic()
    try:
        proc = await asyncio.wait_for(
            asyncio.create_subprocess_exec(
                sys.executable, "-c",
                "import sys; print(f'py{sys.version_info.major}.{sys.version_info.minor} ok')",
                stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
            ),
            timeout=10.0,
        )
        stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=10.0)
        ms = int((time.monotonic() - t) * 1000)
        out = stdout.decode().strip()
        if "ok" in out:
            results.append(_ok("T05", f"Python exec subprocess OK ({ms}ms)", out))
        else:
            results.append(_ko("T05", f"Python exec output inatteso ({ms}ms)", out[:80]))
    except Exception as e:
        ms = int((time.monotonic() - t) * 1000)
        results.append(_ko("T05", f"Python exec FALLITO ({ms}ms)", str(e)[:120]))

    # ── T06: Shell echo + date ────────────────────────────────────────────────
    t = time.monotonic()
    try:
        with tempfile.TemporaryDirectory() as tmpdir:
            proc = await asyncio.wait_for(
                asyncio.create_subprocess_shell(
                    "echo bench_ok && date +%s",
                    stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
                    cwd=tmpdir,
                ),
                timeout=8.0,
            )
            stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=8.0)
        ms = int((time.monotonic() - t) * 1000)
        out = stdout.decode().strip()
        if "bench_ok" in out:
            results.append(_ok("T06", f"Shell execute OK ({ms}ms)", out[:60]))
        else:
            results.append(_ko("T06", f"Shell output inatteso ({ms}ms)", out[:60]))
    except Exception as e:
        ms = int((time.monotonic() - t) * 1000)
        results.append(_ko("T06", f"Shell execute FALLITO ({ms}ms)", str(e)[:120]))

    # ── T07: Shell denylist — BLOCKED_CMDS + _EXEC_BLOCKED_RE ──────────────────
    # exec.py ha due meccanismi: BLOCKED_CMDS (pattern esatti per /api/execute-shell)
    # e _EXEC_BLOCKED_RE (regex per /api/exec sandbox Python).
    # Test: verifica che almeno uno dei due meccanismi esista e funzioni.
    try:
        from api.exec import BLOCKED_CMDS
        # BLOCKED_CMDS deve contenere pattern per i comandi più pericolosi
        # Realtà: {'rm -rf /', 'mkfs', ':(){:|:&};:', 'dd if=/dev/zero'}
        must_have = ["rm -rf /", "mkfs", "dd if=/dev/zero"]
        present = [p for p in must_have if any(p in b or b in p for b in BLOCKED_CMDS)]
        if len(present) >= 2:
            results.append(_ok("T07", f"Shell denylist OK — BLOCKED_CMDS: {len(BLOCKED_CMDS)} pattern",
                               f"include: {list(BLOCKED_CMDS)[:3]}"))
        else:
            results.append(_wn("T07", f"Shell denylist parziale — {len(present)}/{len(must_have)} pattern critici",
                               f"BLOCKED_CMDS={list(BLOCKED_CMDS)}"))
    except ImportError:
        try:
            from api.exec import _EXEC_BLOCKED_RE
            results.append(_ok("T07", "Shell denylist OK — _EXEC_BLOCKED_RE presente"))
        except ImportError:
            results.append(_wn("T07", "denylist non importabile da api.exec (modulo assente)"))

    # ── T08: asyncio.Semaphore S734 ──────────────────────────────────────────
    try:
        from agents.unified_loop_tools import DirectToolsMixin
        src_path = importlib.util.find_spec("agents.unified_loop_tools")
        if src_path:
            import inspect
            src = inspect.getsource(DirectToolsMixin)
            if "asyncio.Semaphore(4)" in src or "Semaphore" in src:
                results.append(_ok("T08", "asyncio.Semaphore S734 presente in DirectToolsMixin"))
            else:
                results.append(_wn("T08", "asyncio.Semaphore non trovato in DirectToolsMixin"))
        else:
            results.append(_wn("T08", "unified_loop_tools non trovato"))
    except Exception as e:
        results.append(_wn("T08", "S734 check non eseguibile", str(e)[:80]))

    # ── T09: Provider env keys — tutti i provider supportati ─────────────────
    # Aggiornato 2026-06-14: aggiunto SAMBANOVA_API_KEY (DeepSeek-V3.1 100% bench)
    providers_conf = {
        "GROQ_API_KEY":       "Groq",
        "OPENROUTER_API_KEY": "OpenRouter",
        "GEMINI_API_KEY":     "Gemini",
        "HF_TOKEN":           "HuggingFace",
        "CEREBRAS_API_KEY":   "Cerebras",
        "SAMBANOVA_API_KEY":  "SambaNova",
    }
    configured = [name for key, name in providers_conf.items() if os.getenv(key)]
    missing    = [name for key, name in providers_conf.items() if not os.getenv(key)]
    if len(configured) >= 2:
        results.append(_ok("T09", f"Provider keys: {len(configured)}/{len(providers_conf)} configurati", ", ".join(configured)))
    elif len(configured) == 1:
        results.append(_wn("T09", f"Provider keys: solo 1/{len(providers_conf)} ({configured[0]}) — fallback chain ridotta", f"mancanti: {', '.join(missing)}"))
    else:
        results.append(_ko("T09", "Nessuna provider API key configurata!", f"mancanti: {', '.join(missing)}"))

    # ── T10: SQLite DB write/read ─────────────────────────────────────────────
    t = time.monotonic()
    try:
        import sqlite3
        with tempfile.NamedTemporaryFile(suffix=".db", delete=True) as f:
            db_path = f.name
        conn = sqlite3.connect(db_path)
        conn.execute("CREATE TABLE bench (k TEXT, v TEXT)")
        conn.execute("INSERT INTO bench VALUES ('test', 'bench_ok')")
        conn.commit()
        row = conn.execute("SELECT v FROM bench WHERE k='test'").fetchone()
        conn.close()
        os.unlink(db_path)
        ms = int((time.monotonic() - t) * 1000)
        if row and row[0] == "bench_ok":
            results.append(_ok("T10", f"SQLite write/read OK ({ms}ms)"))
        else:
            results.append(_ko("T10", f"SQLite round-trip fallito ({ms}ms)", str(row)))
    except Exception as e:
        ms = int((time.monotonic() - t) * 1000)
        results.append(_ko("T10", f"SQLite FALLITO ({ms}ms)", str(e)[:120]))

    # ── T11: /tmp write + read ────────────────────────────────────────────────
    t = time.monotonic()
    try:
        with tempfile.NamedTemporaryFile(mode="w", suffix=".bench", delete=False) as f:
            f.write("bench_filesystem_ok")
            fname = f.name
        with open(fname) as _bf: content = _bf.read()
        os.unlink(fname)
        ms = int((time.monotonic() - t) * 1000)
        if content == "bench_filesystem_ok":
            results.append(_ok("T11", f"/tmp filesystem write/read OK ({ms}ms)"))
        else:
            results.append(_ko("T11", f"/tmp read mismatch ({ms}ms)", content[:40]))
    except Exception as e:
        ms = int((time.monotonic() - t) * 1000)
        results.append(_ko("T11", f"/tmp filesystem FALLITO ({ms}ms)", str(e)[:120]))

    # ── T12: asyncio concurrency — 5 task paralleli ───────────────────────────
    t = time.monotonic()
    try:
        async def _noop(i: int) -> int:
            await asyncio.sleep(0.01)
            return i * 2
        outcomes = await asyncio.gather(*[_noop(i) for i in range(5)])
        ms = int((time.monotonic() - t) * 1000)
        if outcomes == [0, 2, 4, 6, 8]:
            results.append(_ok("T12", f"asyncio concurrency 5x tasks OK ({ms}ms)"))
        else:
            results.append(_ko("T12", f"asyncio concurrency output inatteso ({ms}ms)", str(outcomes)))
    except Exception as e:
        ms = int((time.monotonic() - t) * 1000)
        results.append(_ko("T12", f"asyncio concurrency FALLITA ({ms}ms)", str(e)[:120]))

    # ── T13: Memory manager import ────────────────────────────────────────────
    t = time.monotonic()
    try:
        from memory.manager import MemoryManager
        ms = int((time.monotonic() - t) * 1000)
        results.append(_ok("T13", f"MemoryManager import OK ({ms}ms)"))
    except Exception as e:
        ms = int((time.monotonic() - t) * 1000)
        results.append(_wn("T13", f"MemoryManager import WARN ({ms}ms)", str(e)[:80]))

    # ── T14: Role.FAST _run_fast_path — verifica wiring nel codice sorgente ──
    try:
        import inspect
        from agents.unified_loop import UnifiedAgentLoop
        src = inspect.getsource(UnifiedAgentLoop)
        has_fast_llm    = "_fast_llm" in src
        has_get_fast    = "_get_fast_llm" in src
        has_fast_client = "_fast_client" in src
        if has_fast_llm and has_get_fast and has_fast_client:
            results.append(_ok("T14", "Role.FAST wiring completo in UnifiedAgentLoop",
                               "_fast_llm + _get_fast_llm() + _fast_client.chat()"))
        else:
            missing = [k for k, v in [("_fast_llm", has_fast_llm), ("_get_fast_llm", has_get_fast), ("_fast_client", has_fast_client)] if not v]
            results.append(_ko("T14", "Role.FAST wiring incompleto", f"mancanti: {missing}"))
    except Exception as e:
        results.append(_ko("T14", "Role.FAST wiring check FALLITO", str(e)[:120]))

    # ── T15: Python deps critici importabili ──────────────────────────────────
    critical_deps = ["fastapi", "pydantic", "httpx", "aiohttp", "groq", "google.generativeai"]
    dep_ok, dep_ko = [], []
    for dep in critical_deps:
        try:
            importlib.import_module(dep)
            dep_ok.append(dep)
        except ImportError:
            dep_ko.append(dep)
    if not dep_ko:
        results.append(_ok("T15", f"Deps critici: tutti {len(dep_ok)} importabili", ", ".join(dep_ok)))
    elif len(dep_ko) <= 2:
        results.append(_wn("T15", f"Deps critici: {len(dep_ko)} mancanti", f"ko={dep_ko}"))
    else:
        results.append(_ko("T15", f"Deps critici: {len(dep_ko)}/{len(critical_deps)} mancanti", f"ko={dep_ko}"))

    # ── T16: Groq FAST path — latenza client init ────────────────────────────
    t = time.monotonic()
    groq_key = os.getenv("GROQ_API_KEY", "")
    if groq_key:
        try:
            from models.role_router import RoleRouter, Role
            client = RoleRouter.get_client(Role.FAST)
            ms = int((time.monotonic() - t) * 1000)
            results.append(_ok("T16", f"Groq FAST client init ({ms}ms)",
                               getattr(client, "provider_name", type(client).__name__)))
        except Exception as e:
            ms = int((time.monotonic() - t) * 1000)
            results.append(_ko("T16", f"Groq FAST client FALLITO ({ms}ms)", str(e)[:120]))
    else:
        results.append(_wn("T16", "GROQ_API_KEY non configurata — Role.FAST userà self.llm come fallback"))

    # ── T17: Latenza totale benchmark ─────────────────────────────────────────
    total_ms = int((time.monotonic() - t_global) * 1000)
    results.append(_ok("T17", f"Benchmark completato in {total_ms}ms",
                        f"{'OK' if total_ms < 5000 else 'SLOW'} (target <5s)"))

    return results


@router.get("/api/debug/benchmark")
async def run_benchmark(
    token: str = Query(..., description="Daily HMAC token — vedi docstring modulo"),
    pretty: bool = Query(False, description="Output human-readable invece di JSON compatto"),
):
    """S-BENCH: Self-test server-side completo — zero dipendenza da Replit.

    Auth: HMAC-SHA256 daily token (seed fisso in codice, non è un secret).
    Calcola il token del giorno con:
        python3 -c "import hmac,hashlib,datetime; print(hmac.new(b'agente-ai-bench-2026', datetime.date.today().isoformat().encode(), hashlib.sha256).hexdigest())"
    """
    expected = _daily_token()
    if not hmac.compare_digest(token, expected):
        raise HTTPException(
            status_code=401,
            detail={
                "error": "Token non valido o scaduto (cambia ogni giorno).",
                "hint": "python3 -c \"import hmac,hashlib,datetime; "
                        "print(hmac.new(b'agente-ai-bench-2026', datetime.date.today().isoformat().encode(), hashlib.sha256).hexdigest())\"",
            },
        )

    t0 = time.monotonic()
    results = await _run_tests()
    elapsed_ms = int((time.monotonic() - t0) * 1000)

    pass_n  = sum(1 for r in results if r["ok"])
    fail_n  = sum(1 for r in results if not r["ok"] and not r["warn"])
    warn_n  = sum(1 for r in results if r["warn"])
    total_n = len(results)
    score   = round(pass_n / max(1, pass_n + fail_n) * 100)

    gaps = [r for r in results if not r["ok"] and not r["warn"]]

    payload = {
        "ok":          fail_n == 0,
        "score":       score,
        "pass":        pass_n,
        "fail":        fail_n,
        "warn":        warn_n,
        "total":       total_n,
        "duration_ms": elapsed_ms,
        "timestamp":   datetime.datetime.utcnow().isoformat() + "Z",
        "tests":       results,
        "gaps":        [{"id": r["id"], "desc": r["desc"], "note": r.get("note", "")} for r in gaps],
    }
    return JSONResponse(content=payload, status_code=200 if fail_n == 0 else 207)


# ═══════════════════════════════════════════════════════════════════════════════
# QUALITY BENCHMARK — misura miglioramenti LLM per sprint (S-BENCH-Q)
# POST /api/benchmark/quality/run          → avvia background task, ritorna task_id
# GET  /api/benchmark/quality/status/{id}  → polling risultati
#
# Per ogni categoria agente (DA / ORCH / MC / REC):
#   1. Inietta la context rule via UnifiedLoopPrompts._pick_context_rules()
#   2. Chiama il LLM (ARCHITECT = llama-3.3-70b-versatile) a temperatura 0.3
#   3. Valuta la risposta con checker regex (stessa logica di benchmark-extended.mjs)
#   4. Produce score 0-100 per categoria + media totale
#
# Auth: stesso token HMAC daily del /api/debug/benchmark.
# ═══════════════════════════════════════════════════════════════════════════════

_QUALITY_RUNS: dict[str, dict] = {}   # task_id → {status, results, …}

# ── Definizione task benchmark qualità ─────────────────────────────────────────
_QUALITY_TASKS = [
    {
        "id": "DA",
        "category": "data_analysis",
        "label": "Analisi Dati",
        "goal_for_rules": "analisi dati vendite statistiche media picco anomalia trend",
        "user_prompt": (
            "Analizza questi dati di vendite mensili:\n"
            "Gen=100, Feb=120, Mar=80, Apr=150, Mag=90, Giu=200.\n\n"
            "Fornisci un'analisi strutturata con Media, Picco, Anomalia e Trend."
        ),
        "criteria": [
            {"id": "media",    "label": "**Media: N**",       "weight": 25},
            {"id": "picco",    "label": "**Picco: MESE**",    "weight": 25},
            {"id": "anomalia", "label": "**Anomalia: MESE**", "weight": 25},
            {"id": "trend",    "label": "**Trend: ...**",     "weight": 25},
        ],
    },
    {
        "id": "ORCH",
        "category": "orchestration",
        "label": "Orchestrazione",
        "goal_for_rules": "implementa sistema backend typescript asincrono dipendenze sql async",
        "user_prompt": (
            "Implementa un sistema di notifiche email per e-commerce con:\n"
            "- Invio email alla conferma ordine\n"
            "- Retry automatico su failure (3 tentativi)\n"
            "- Tracking stato consegna\n\n"
            "TypeScript + Node.js. Mostra: piano → implementazione → dipendenze."
        ),
        "criteria": [
            {"id": "hasPlan",  "label": "Piano / sezione strutturata", "weight": 25},
            {"id": "hasCode",  "label": "Codice TypeScript",           "weight": 25},
            {"id": "hasAsync", "label": "async/await o Promise",       "weight": 25},
            {"id": "hasDeps",  "label": "Dipendenze elencate",         "weight": 25},
        ],
    },
    {
        "id": "MC",
        "category": "memory_context",
        "label": "Memory Context",
        "goal_for_rules": "interface typescript endpoint apiresponse stack architettura libreria",
        "user_prompt": (
            "Implementa la route Express per GET /api/users/:id.\n"
            "Usa il pattern ApiResponse<T> con i campi: success, data, error, requestId.\n"
            "Rispetta le convenzioni dello stack del progetto (Drizzle, Zod, Express)."
        ),
        "criteria": [
            {"id": "hasApiResponse", "label": "ApiResponse<T> usato",        "weight": 35},
            {"id": "hasRequestId",   "label": "requestId nel response",       "weight": 35},
            {"id": "hasStack",       "label": "Stack reale (Drizzle/Zod/..)", "weight": 30},
        ],
    },
    {
        "id": "REC",
        "category": "recovery",
        "label": "Recovery",
        "goal_for_rules": "task ambiguo input mancante cosa fare rollback vincoli",
        "user_prompt": (
            "L'utente scrive solo: 'Fammi un'analisi'.\n"
            "Non specifica cosa analizzare, non ha fornito dati.\n\n"
            "Cosa fai? (non inventare dati, non procedere silenziosamente)"
        ),
        "criteria": [
            {"id": "asksDetails",     "label": "Chiede chiarimenti",               "weight": 40},
            {"id": "listAssumptions", "label": "Lista assunzioni / ipotesi",        "weight": 30},
            {"id": "noHallucinate",   "label": "Non inventa dati (anti-hallucination)", "weight": 30},
        ],
    },
    {
        "id": "ROB",
        "category": "robustness",
        "label": "Robustness",
        "goal_for_rules": "istruzioni diventano progressivamente meno specifiche gestisci l ambiguità rendila ancora più efficiente ordinamento typescript",
        "user_prompt": (
            "Implementa una funzione di ordinamento TypeScript efficiente.\n"
            "Poi rendila ancora più efficiente.\n"
            "Ottimizzala per il caso d'uso tipico.\n"
            "Assicurati che funzioni.\n\n"
            "Nota: le istruzioni diventano progressivamente meno specifiche. "
            "Gestisci l'ambiguità in modo esplicito."
        ),
        "criteria": [
            {"id": "hasCode",          "label": "Codice TypeScript (sort)",       "weight": 25},
            {"id": "handlesAmbiguity", "label": "Dichiara assunzioni esplicite",  "weight": 25},
            {"id": "hasSort",          "label": "Usa sort/algorithm",             "weight": 25},
            {"id": "hasRationale",     "label": "Motivazione / perché",           "weight": 25},
        ],
    },
]


def _score_quality_task(task: dict, response: str) -> dict:
    """Valuta risposta LLM su ogni criterio — ritorna score 0-100."""
    r   = response
    tid = task["id"]

    if tid == "DA":
        checks = {
            "media":    bool(re.search(r'\*\*Media',   r, re.IGNORECASE)),
            "picco":    bool(re.search(r'\*\*Picco',   r, re.IGNORECASE)),
            "anomalia": bool(re.search(r'\*\*Anomali', r, re.IGNORECASE)),
            "trend":    bool(re.search(r'\*\*(Trend|Andamento|Tendenz)',   r, re.IGNORECASE)),
        }
    elif tid == "ORCH":
        checks = {
            "hasPlan":  bool(re.search(r'(piano|step\s*\d|fase\s*\d|\d+\.\s+[A-Z])', r, re.IGNORECASE)),
            "hasCode":  bool(re.search(r'```(ts|typescript|javascript|js)', r, re.IGNORECASE)),
            "hasAsync": bool(re.search(r'(async|await|Promise\.all)', r)),
            "hasDeps":  bool(re.search(r'(npm|pnpm|yarn|install|dependen|dipendenz|package\.json)', r, re.IGNORECASE)),
        }
    elif tid == "MC":
        checks = {
            "hasApiResponse": bool(re.search(r'ApiResponse', r)),
            "hasRequestId":   bool(re.search(r'requestId', r)),
            "hasStack":       bool(re.search(r'(Drizzle|Zod|Express|Prisma|knex)', r, re.IGNORECASE)),
        }
    elif tid == "REC":
        checks = {
            "asksDetails":     bool(re.search(
                r'(\?|qual[ei]|cosa intendi|chiar|specificar|dettagl|di\s+pi)', r, re.IGNORECASE)),
            "listAssumptions": bool(re.search(
                r'(assumo|ipotesi|assunzion|potrebbe essere|se intendi|per esempio|ad esempio'
                r'|se si tratta|che tipo|quale tipo|se vuole|potrei fare|opzione [ab]'
                r'|se intende|potrebbe trattarsi|quale delle|in base a cosa)', r, re.IGNORECASE)),
            # pass se NON inventa dati concreti senza chiedere
            "noHallucinate":   not bool(re.search(
                r'(ecco l.analisi|ecco i dati|i dati mostrano|risultati:|media:\s*\d)', r, re.IGNORECASE)),
        }
    elif tid == "ROB":
        import re as _re
        code_blocks = _re.findall(r"```(?:typescript|ts)[\s\S]*?```", r, _re.IGNORECASE)
        code_txt = " ".join(code_blocks)
        checks = {
            "hasCode":          bool(code_blocks) and len(code_txt) > 50,
            "handlesAmbiguity": bool(_re.search(r"assumo|ipotizzo|ambiguo|caso tipico|interpretto", r, _re.IGNORECASE)),
            "hasSort":          bool(_re.search(r"sort|quicksort|mergesort|compareFn|algorithm", r, _re.IGNORECASE)),
            "hasRationale":     bool(_re.search(r"perché|motivazione|scelta|rationale|perche", r, _re.IGNORECASE)),
        }
    else:
        checks = {}

    scored = [{**c, "pass": checks.get(c["id"], False)} for c in task["criteria"]]
    score  = sum(c["weight"] for c in scored if c["pass"])
    return {
        "id":               task["id"],
        "category":         task["category"],
        "label":            task["label"],
        "score":            score,
        "criteria":         scored,
        "response_preview": r[:400] + ("\u2026" if len(r) > 400 else ""),
    }


async def _run_quality_benchmark(task_id: str) -> None:
    """Background task: 4 chiamate LLM in parallelo → scorecard qualità."""
    _QUALITY_RUNS[task_id]["status"] = "running"
    results: list[dict] = []
    errors:  list[dict] = []

    try:
        from models.role_router import RoleRouter, Role            # noqa: PLC0415
        from agents.unified_loop_prompts import PromptBuilderMixin  # noqa: PLC0415

        # System prompt leggero per benchmark: NO tool definitions (evita tool-call da FAST)
        # Le context rules iniettano le istruzioni specifiche per categoria
        prompts_obj = PromptBuilderMixin()
        _BENCH_SYS = (
            "Sei un assistente AI specializzato in sviluppo software e analisi dati. "
            "Rispondi in italiano usando markdown. "
            "Usa blocchi di codice ```typescript``` / ```javascript``` per il codice. "
            "NON usare tool calls o function calls — rispondi sempre con testo e codice."
        )

        # Provider chain: ARCHITECT prima (qualità), poi fallback per 402/depleted
        _PROVIDER_CHAIN = ["ARCHITECT", "REASONER", "CODER", "FAST"]

        async def _run_one_task(msgs: list[dict]) -> str:
            """Retry indipendente per task — ARCHITECT first, FAST come ultimo resort."""
            last_exc: Exception | None = None
            for _rname in _PROVIDER_CHAIN:
                _r = getattr(Role, _rname, None)
                if _r is None:
                    continue
                try:
                    _c = RoleRouter.get_client(_r)
                    result = await asyncio.wait_for(
                        _c.chat(msgs, temperature=0.3, max_tokens=1200),
                        timeout=35.0,
                    )
                    return str(result)
                except Exception as _exc:
                    last_exc = _exc
                    _exc_s = str(_exc).lower()
                    # 402 / depleted / rate-limit → prova il prossimo provider
                    if any(k in _exc_s for k in ["402", "depleted", "rate limit", "too many", "429"]):
                        continue
                    raise  # errore non-recuperabile → propaga subito
            raise last_exc or Exception("Nessun provider disponibile")

        # Esecuzione sequenziale — evita rate-limit da 5 richieste simultanee allo stesso provider
        # Latenza: ~30-50s (5 × ~7-10s) vs 402 su tutto con parallelo
        results_map: list[str | Exception] = []
        for task in _QUALITY_TASKS:
            ctx_rules = prompts_obj._pick_context_rules(task["goal_for_rules"])
            system    = _BENCH_SYS + (("\n\n" + ctx_rules) if ctx_rules else "")
            msgs = [
                {"role": "system", "content": system},
                {"role": "user",   "content": task["user_prompt"]},
            ]
            try:
                resp = await _run_one_task(msgs)
                results_map.append(resp)
            except Exception as _exc:
                results_map.append(_exc)

        responses = results_map

        for task, response in zip(_QUALITY_TASKS, responses):
            if isinstance(response, Exception):
                err_msg = str(response)[:150]
                errors.append({"id": task["id"], "error": err_msg})
                results.append({
                    "id": task["id"], "category": task["category"],
                    "label": task["label"], "score": 0, "criteria": [], "error": err_msg,
                })
            else:
                results.append(_score_quality_task(task, str(response)))

    except Exception as exc:
        errors.append({"global": str(exc)[:250]})

    passed    = [r for r in results if not r.get("error")]
    avg_score = round(sum(r["score"] for r in passed) / max(1, len(passed)))

    _QUALITY_RUNS[task_id].update({
        "status":         "done",
        "results":        results,
        "errors":         errors,
        "total_score":    avg_score,
        "categories_run": len(results),
        "finished_at":    datetime.datetime.utcnow().isoformat() + "Z",
    })


@router.post("/api/benchmark/quality/run")
async def start_quality_benchmark(
    background_tasks: BackgroundTasks,
    token: str = Query(..., description="Daily HMAC token — stesso del /api/debug/benchmark"),
):
    """Avvia il benchmark qualità LLM in background (S-BENCH-Q).

    Testa 4 categorie agentiche iniettando le context rules del sprint corrente,
    chiama il LLM e valuta la risposta con checker regex.

    Ritorna task_id per polling: GET /api/benchmark/quality/status/{task_id}

    Calcola il token del giorno con:
        python3 -c "import hmac,hashlib,datetime; \\
            print(hmac.new(b'agente-ai-bench-2026', \\
            datetime.date.today().isoformat().encode(), \\
            hashlib.sha256).hexdigest())"
    """
    expected = _daily_token()
    if not hmac.compare_digest(token, expected):
        raise HTTPException(
            status_code=401,
            detail={
                "error": "Token non valido o scaduto (cambia ogni giorno).",
                "hint":  ("python3 -c \"import hmac,hashlib,datetime; "
                          "print(hmac.new(b'agente-ai-bench-2026',"
                          "datetime.date.today().isoformat().encode(),"
                          "hashlib.sha256).hexdigest())\""),
            },
        )

    task_id = uuid.uuid4().hex[:12]
    now_iso = datetime.datetime.utcnow().isoformat() + "Z"
    _QUALITY_RUNS[task_id] = {
        "status":         "queued",
        "started_at":     now_iso,
        "results":        [],
        "errors":         [],
        "total_score":    None,
        "categories_run": 0,
    }
    background_tasks.add_task(_run_quality_benchmark, task_id)

    return JSONResponse({
        "task_id":    task_id,
        "status":     "queued",
        "poll_url":   f"/api/benchmark/quality/status/{task_id}",
        "categories": [f"{t['id']} ({t['label']})" for t in _QUALITY_TASKS],
        "started_at": now_iso,
    }, status_code=202)



# ═══════════════════════════════════════════════════════════════════════════════
# RUN-SELF — il bot esegue il proprio benchmark in autonomia (S-BENCH-SELF)
# POST /api/benchmark/run-self  →  auth: X-Internal-Token header
#
# Esegue il quality benchmark su tutte le categorie + ROB e ritorna i risultati
# direttamente (sincrono, ~20-30s). Il bot può chiamare questo endpoint
# autonomamente senza Replit, senza token HMAC, senza configurazione esterna.
# ═══════════════════════════════════════════════════════════════════════════════

@router.post("/api/benchmark/run-self")
async def run_self_benchmark(request: "Request"):
    """Self-benchmark: il bot misura le proprie performance in autonomia.

    Auth: X-Internal-Token header (stesso usato per /api/agent/run-stream).
    Nessun token HMAC, nessuna dipendenza da Replit.

    Esegue il quality benchmark su tutte le categorie (DA, ORCH, MC, REC, ROB)
    e ritorna lo scorecard completo.

    Esempio:
        curl -X POST https://arjanit98-terminal.hf.space/api/benchmark/run-self \
             -H "X-Internal-Token: <token>"
    """
    import os as _os
    from fastapi import Request as _Req
    itok_conf = _os.getenv("INTERNAL_TOKEN", "")
    itok_recv = request.headers.get("X-Internal-Token", "")
    if not itok_conf or not itok_recv or itok_recv != itok_conf:
        from fastapi import HTTPException as _HTTP
        raise _HTTP(status_code=401, detail="X-Internal-Token non valido o mancante.")

    t0 = __import__("time").monotonic()
    task_id = __import__("uuid").uuid4().hex[:12]
    _QUALITY_RUNS[task_id] = {
        "status": "running", "started_at": datetime.datetime.utcnow().isoformat() + "Z",
        "results": [], "errors": [], "total_score": None, "categories_run": 0,
    }
    await _run_quality_benchmark(task_id)
    elapsed_ms = int((__import__("time").monotonic() - t0) * 1000)

    run = _QUALITY_RUNS[task_id]
    results = run.get("results", [])
    passed  = [r for r in results if not r.get("error")]
    avg     = round(sum(r["score"] for r in passed) / max(1, len(passed)))

    return JSONResponse({
        "ok":          len(run.get("errors", [])) == 0,
        "total_score": avg,
        "categories_run": len(results),
        "duration_ms": elapsed_ms,
        "timestamp":   datetime.datetime.utcnow().isoformat() + "Z",
        "results":     results,
        "errors":      run.get("errors", []),
        "gaps": [
            {"id": r["id"], "label": r["label"], "score": r["score"],
             "failed_criteria": [c["label"] for c in r.get("criteria", []) if not c.get("pass")]}
            for r in results if r["score"] < 75
        ],
    })

@router.get("/api/benchmark/quality/status/{task_id}")
async def quality_benchmark_status(task_id: str):
    """Polling sullo stato del benchmark qualità.

    Lifecycle: queued → running → done

    Response fields:
      status:         queued | running | done
      total_score:    0-100 (media delle categorie senza errori)
      results:        per-task score + criteri + preview risposta
      errors:         errori LLM per task (score=0 se presente)
      categories_run: quante categorie hanno prodotto un risultato
    """
    run = _QUALITY_RUNS.get(task_id)
    if run is None:
        raise HTTPException(
            status_code=404,
            detail=f"Task '{task_id}' non trovato. Avvia prima POST /api/benchmark/quality/run",
        )
    return JSONResponse(run)