Spaces:
Running
Running
sync: 168 file da Baida98/AI@d7f424d0 (2026-08-22 12:59 UTC) [deploy-all]
Browse files- api/auth_guard.py +8 -4
- api/exec.py +4 -5
- api/providers.py +3 -3
- benchmark-extended.mjs +14 -6
- benchmarks/model_watch_adapter.py +2 -2
- memory/sync.py +7 -5
- models/ai_client.py +2 -2
- models/role_router.py +21 -21
- tests/test_model_catalog_scan.py +48 -0
- tests/test_provider_model_migrations.py +6 -6
api/auth_guard.py
CHANGED
|
@@ -201,6 +201,7 @@ def _get_token(env_var: str) -> str:
|
|
| 201 |
|
| 202 |
async def _resolve_role(
|
| 203 |
x_internal_token: Optional[str] = Header(None, alias="X-Internal-Token"),
|
|
|
|
| 204 |
x_operator_token: Optional[str] = Header(None, alias="X-Operator-Token"),
|
| 205 |
x_admin_token: Optional[str] = Header(None, alias="X-Admin-Token"),
|
| 206 |
) -> AuthRole:
|
|
@@ -218,9 +219,11 @@ async def _resolve_role(
|
|
| 218 |
logger.debug("auth: OPERATOR role granted")
|
| 219 |
return AuthRole.OPERATOR
|
| 220 |
|
| 221 |
-
# MACHINE
|
| 222 |
-
|
| 223 |
-
|
|
|
|
|
|
|
| 224 |
logger.debug("auth: MACHINE role granted")
|
| 225 |
return AuthRole.MACHINE
|
| 226 |
|
|
@@ -280,7 +283,8 @@ def require_role(min_role: AuthRole):
|
|
| 280 |
_token_hdr = (
|
| 281 |
request.headers.get('X-Admin-Token') or
|
| 282 |
request.headers.get('X-Operator-Token') or
|
| 283 |
-
request.headers.get('X-Internal-Token')
|
|
|
|
| 284 |
)
|
| 285 |
# GAP-AUTH-FIX: estrai IP reale (Railway/HF dietro proxy → X-Forwarded-For)
|
| 286 |
_client_ip: str | None = (
|
|
|
|
| 201 |
|
| 202 |
async def _resolve_role(
|
| 203 |
x_internal_token: Optional[str] = Header(None, alias="X-Internal-Token"),
|
| 204 |
+
x_machine_token: Optional[str] = Header(None, alias="X-Machine-Token"),
|
| 205 |
x_operator_token: Optional[str] = Header(None, alias="X-Operator-Token"),
|
| 206 |
x_admin_token: Optional[str] = Header(None, alias="X-Admin-Token"),
|
| 207 |
) -> AuthRole:
|
|
|
|
| 219 |
logger.debug("auth: OPERATOR role granted")
|
| 220 |
return AuthRole.OPERATOR
|
| 221 |
|
| 222 |
+
# MACHINE: supporta entrambi gli header per compatibilità tra runner e backend.
|
| 223 |
+
# Il valore resta confrontato esclusivamente con il secret server-side.
|
| 224 |
+
int_tok = _get_token("INTERNAL_TOKEN") or _get_token("MACHINE_TOKEN")
|
| 225 |
+
machine_header = x_internal_token or x_machine_token
|
| 226 |
+
if int_tok and machine_header and _sec_comp.compare_digest(machine_header, int_tok):
|
| 227 |
logger.debug("auth: MACHINE role granted")
|
| 228 |
return AuthRole.MACHINE
|
| 229 |
|
|
|
|
| 283 |
_token_hdr = (
|
| 284 |
request.headers.get('X-Admin-Token') or
|
| 285 |
request.headers.get('X-Operator-Token') or
|
| 286 |
+
request.headers.get('X-Internal-Token') or
|
| 287 |
+
request.headers.get('X-Machine-Token')
|
| 288 |
)
|
| 289 |
# GAP-AUTH-FIX: estrai IP reale (Railway/HF dietro proxy → X-Forwarded-For)
|
| 290 |
_client_ip: str | None = (
|
api/exec.py
CHANGED
|
@@ -568,15 +568,14 @@ async def llm_fix_code(
|
|
| 568 |
_FIX_CHAIN = []
|
| 569 |
groq_key = os.getenv('GROQ_API_KEY', '')
|
| 570 |
if groq_key:
|
| 571 |
-
_FIX_CHAIN.append(('https://api.groq.com/openai/v1', groq_key, '
|
| 572 |
-
_FIX_CHAIN.append(('https://api.groq.com/openai/v1', groq_key, 'openai/gpt-oss-20b'))
|
| 573 |
|
| 574 |
or_key = os.getenv('OPENROUTER_API_KEY', '')
|
| 575 |
if or_key:
|
| 576 |
for m in [
|
| 577 |
-
'
|
| 578 |
-
'
|
| 579 |
-
'
|
| 580 |
]:
|
| 581 |
_FIX_CHAIN.append(('https://openrouter.ai/api/v1', or_key, m))
|
| 582 |
|
|
|
|
| 568 |
_FIX_CHAIN = []
|
| 569 |
groq_key = os.getenv('GROQ_API_KEY', '')
|
| 570 |
if groq_key:
|
| 571 |
+
_FIX_CHAIN.append(('https://api.groq.com/openai/v1', groq_key, 'qwen/qwen3.6-27b'))
|
|
|
|
| 572 |
|
| 573 |
or_key = os.getenv('OPENROUTER_API_KEY', '')
|
| 574 |
if or_key:
|
| 575 |
for m in [
|
| 576 |
+
'openrouter/free',
|
| 577 |
+
'qwen/qwen3-coder:free',
|
| 578 |
+
'meta-llama/llama-3.3-70b-instruct:free',
|
| 579 |
]:
|
| 580 |
_FIX_CHAIN.append(('https://openrouter.ai/api/v1', or_key, m))
|
| 581 |
|
api/providers.py
CHANGED
|
@@ -930,14 +930,14 @@ async def update_provider_models(role: AuthRole = Depends(require_role(AuthRole.
|
|
| 930 |
# filtrare per `name` evita di applicare un formato incompatibile alla riga sbagliata.
|
| 931 |
# GPT-OSS 120B non compare come vecchio valore perché è già un modello supportato.
|
| 932 |
MODEL_FIXES = [
|
| 933 |
-
("groq", "llama-3.1-70b-versatile", "
|
| 934 |
("cerebras", "llama3.1-70b", "gpt-oss-120b"),
|
| 935 |
("nvidia", "llama-3.1-405b-instruct", "meta/llama-3.3-70b-instruct"),
|
| 936 |
-
("openrouter", "llama-3.1-405b", "
|
| 937 |
("sambanova", "llama3-70b", "DeepSeek-V3.2"),
|
| 938 |
("gemini", "gemini-1.5-flash", "gemini-3.5-flash-lite"),
|
| 939 |
("gemini", "gemini-1.5-pro", "gemini-3.6-flash"),
|
| 940 |
-
("openrouter", "claude-3.5-sonnet", "
|
| 941 |
]
|
| 942 |
|
| 943 |
import asyncio as _aio
|
|
|
|
| 930 |
# filtrare per `name` evita di applicare un formato incompatibile alla riga sbagliata.
|
| 931 |
# GPT-OSS 120B non compare come vecchio valore perché è già un modello supportato.
|
| 932 |
MODEL_FIXES = [
|
| 933 |
+
("groq", "llama-3.1-70b-versatile", "qwen/qwen3.6-27b"),
|
| 934 |
("cerebras", "llama3.1-70b", "gpt-oss-120b"),
|
| 935 |
("nvidia", "llama-3.1-405b-instruct", "meta/llama-3.3-70b-instruct"),
|
| 936 |
+
("openrouter", "llama-3.1-405b", "openrouter/free"),
|
| 937 |
("sambanova", "llama3-70b", "DeepSeek-V3.2"),
|
| 938 |
("gemini", "gemini-1.5-flash", "gemini-3.5-flash-lite"),
|
| 939 |
("gemini", "gemini-1.5-pro", "gemini-3.6-flash"),
|
| 940 |
+
("openrouter", "claude-3.5-sonnet", "openrouter/free"),
|
| 941 |
]
|
| 942 |
|
| 943 |
import asyncio as _aio
|
benchmark-extended.mjs
CHANGED
|
@@ -244,10 +244,10 @@ FSH.orchestration =
|
|
| 244 |
"ESECUZIONE: esegui ogni step nell'ordine con i tool dichiarati.";
|
| 245 |
|
| 246 |
FSH.bug_fix =
|
| 247 |
-
"
|
| 248 |
-
"
|
| 249 |
-
"
|
| 250 |
-
"
|
| 251 |
|
| 252 |
FSH.refactor =
|
| 253 |
"TECNICA OBBLIGATORIA: guard clauses + early return.\n" +
|
|
@@ -2443,6 +2443,10 @@ async function runOneSeed(seed,opts={}){
|
|
| 2443 |
const reportPath=`/tmp/agente-ai/benchmark-v5-${seed}.json`;
|
| 2444 |
const report={
|
| 2445 |
timestamp:new Date().toISOString(),version:"extended-v5",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2446 |
seed,replayCli:`node benchmark-extended.mjs --seed ${seed}`,
|
| 2447 |
spaceVersion:spaceV,
|
| 2448 |
specCoverage:{A_ragionamento:"reasoning",B_orchestrazione:"orchestration",
|
|
@@ -2589,9 +2593,13 @@ else if(MULTI>1){
|
|
| 2589 |
: null,
|
| 2590 |
avgDevin: _runResult.avgDevin,
|
| 2591 |
avgManus: _runResult.avgManus,
|
| 2592 |
-
verdict: _runResult.avgScore >= _runResult.avgReplit ? "PARI_REPLIT" : "SOTTO_REPLIT",
|
| 2593 |
};
|
| 2594 |
-
saveBenchmarkReport("benchmark-extended-single", {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2595 |
} catch (e) { console.log(`${R}⚠️ Report engine error: ${e.message}${NC}`); }
|
| 2596 |
|
| 2597 |
if(F_IMPROVE){ await runImprovementCycle(_runResult,_runResult.selectedTasks); }
|
|
|
|
| 244 |
"ESECUZIONE: esegui ogni step nell'ordine con i tool dichiarati.";
|
| 245 |
|
| 246 |
FSH.bug_fix =
|
| 247 |
+
"OUTPUT VINCOLATO: restituisci esclusivamente un singolo blocco ```typescript ... ``` completo e compilabile.\n" +
|
| 248 |
+
"Non aggiungere Root cause, Test, spiegazioni o testo fuori dal blocco.\n" +
|
| 249 |
+
"Mantieni la struttura e le firme pubbliche; per effetti React includi guardia di smontaggio e cleanup/abort nel return di useEffect.\n" +
|
| 250 |
+
"Usa export named quando il codice definisce una funzione o una classe pubblica.";
|
| 251 |
|
| 252 |
FSH.refactor =
|
| 253 |
"TECNICA OBBLIGATORIA: guard clauses + early return.\n" +
|
|
|
|
| 2443 |
const reportPath=`/tmp/agente-ai/benchmark-v5-${seed}.json`;
|
| 2444 |
const report={
|
| 2445 |
timestamp:new Date().toISOString(),version:"extended-v5",
|
| 2446 |
+
runner:"benchmark-extended.mjs",
|
| 2447 |
+
apiContract:"POST /api/agent/tasks + GET /api/agent/tasks/{taskId}/stream",
|
| 2448 |
+
sseParser:"normalizeSSEEvent",
|
| 2449 |
+
codeExtractor:"extractCode-v2",
|
| 2450 |
seed,replayCli:`node benchmark-extended.mjs --seed ${seed}`,
|
| 2451 |
spaceVersion:spaceV,
|
| 2452 |
specCoverage:{A_ragionamento:"reasoning",B_orchestrazione:"orchestration",
|
|
|
|
| 2593 |
: null,
|
| 2594 |
avgDevin: _runResult.avgDevin,
|
| 2595 |
avgManus: _runResult.avgManus,
|
| 2596 |
+
verdict: _runResult.avgScore == null ? "NON_VALUTABILE" : _runResult.avgScore >= _runResult.avgReplit ? "PARI_REPLIT" : "SOTTO_REPLIT",
|
| 2597 |
};
|
| 2598 |
+
saveBenchmarkReport("benchmark-extended-single", {
|
| 2599 |
+
summary,
|
| 2600 |
+
tasks: _runResult.tasks ?? [],
|
| 2601 |
+
taskFailures: _runResult.taskFailures ?? [],
|
| 2602 |
+
});
|
| 2603 |
} catch (e) { console.log(`${R}⚠️ Report engine error: ${e.message}${NC}`); }
|
| 2604 |
|
| 2605 |
if(F_IMPROVE){ await runImprovementCycle(_runResult,_runResult.selectedTasks); }
|
benchmarks/model_watch_adapter.py
CHANGED
|
@@ -230,7 +230,7 @@ class ObserveOnlyModelsAdapter:
|
|
| 230 |
params: dict[str, str] = {}
|
| 231 |
if profile.auth_mode == "query_key":
|
| 232 |
params["key"] = profile.api_key
|
| 233 |
-
|
| 234 |
headers["Authorization"] = f"Bearer {profile.api_key}"
|
| 235 |
|
| 236 |
owns_client = self._client is None
|
|
@@ -292,7 +292,7 @@ class GeminiModelsAdapter(ObserveOnlyModelsAdapter):
|
|
| 292 |
async def list_models(self, profile: ProviderProfile) -> CatalogResult:
|
| 293 |
url = models_url(profile.base_url)
|
| 294 |
headers = {"Accept": "application/json"}
|
| 295 |
-
params = {"key": profile.api_key}
|
| 296 |
owns_client = self._client is None
|
| 297 |
client = self._client or httpx.AsyncClient(timeout=self.timeout_seconds)
|
| 298 |
try:
|
|
|
|
| 230 |
params: dict[str, str] = {}
|
| 231 |
if profile.auth_mode == "query_key":
|
| 232 |
params["key"] = profile.api_key
|
| 233 |
+
elif profile.auth_mode != "none":
|
| 234 |
headers["Authorization"] = f"Bearer {profile.api_key}"
|
| 235 |
|
| 236 |
owns_client = self._client is None
|
|
|
|
| 292 |
async def list_models(self, profile: ProviderProfile) -> CatalogResult:
|
| 293 |
url = models_url(profile.base_url)
|
| 294 |
headers = {"Accept": "application/json"}
|
| 295 |
+
params = {"key": profile.api_key} if profile.auth_mode != "none" else {}
|
| 296 |
owns_client = self._client is None
|
| 297 |
client = self._client or httpx.AsyncClient(timeout=self.timeout_seconds)
|
| 298 |
try:
|
memory/sync.py
CHANGED
|
@@ -58,6 +58,12 @@ class MemorySyncStatus(BaseModel):
|
|
| 58 |
stats: dict[str, Any]
|
| 59 |
|
| 60 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
# ── GAP-VAULT-AUTH: autenticazione Bearer ─────────────────────────────────────
|
| 62 |
_SYNC_ADMIN_TOKEN = os.getenv('VAULT_ADMIN_TOKEN', '') # stessa variabile del vault
|
| 63 |
|
|
@@ -218,13 +224,9 @@ def create_memory_sync_router(memory: Any) -> APIRouter:
|
|
| 218 |
"server_time": _now_ms(),
|
| 219 |
}
|
| 220 |
|
| 221 |
-
class _MemoryImportRequest(BaseModel):
|
| 222 |
-
records: list[dict[str, Any]] = Field(default_factory=list)
|
| 223 |
-
overwrite: bool = False
|
| 224 |
-
|
| 225 |
@router.post("/import")
|
| 226 |
async def memory_import(
|
| 227 |
-
req:
|
| 228 |
_auth: None = Depends(_require_sync_auth),
|
| 229 |
) -> dict[str, Any]:
|
| 230 |
"""Importa records nella semantic memory. Richiede Bearer VAULT_ADMIN_TOKEN."""
|
|
|
|
| 58 |
stats: dict[str, Any]
|
| 59 |
|
| 60 |
|
| 61 |
+
class MemoryImportRequest(BaseModel):
|
| 62 |
+
"""Payload di import definito a livello modulo per lo schema OpenAPI."""
|
| 63 |
+
records: list[dict[str, Any]] = Field(default_factory=list)
|
| 64 |
+
overwrite: bool = False
|
| 65 |
+
|
| 66 |
+
|
| 67 |
# ── GAP-VAULT-AUTH: autenticazione Bearer ─────────────────────────────────────
|
| 68 |
_SYNC_ADMIN_TOKEN = os.getenv('VAULT_ADMIN_TOKEN', '') # stessa variabile del vault
|
| 69 |
|
|
|
|
| 224 |
"server_time": _now_ms(),
|
| 225 |
}
|
| 226 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 227 |
@router.post("/import")
|
| 228 |
async def memory_import(
|
| 229 |
+
req: MemoryImportRequest,
|
| 230 |
_auth: None = Depends(_require_sync_auth),
|
| 231 |
) -> dict[str, Any]:
|
| 232 |
"""Importa records nella semantic memory. Richiede Bearer VAULT_ADMIN_TOKEN."""
|
models/ai_client.py
CHANGED
|
@@ -61,11 +61,11 @@ class ProviderConfig:
|
|
| 61 |
# (nessun proxy CF Worker qui: questo client gira lato backend Python, non browser).
|
| 62 |
_PROVIDER_DEFS = [
|
| 63 |
# tier 0 — free tier veloce e affidabile
|
| 64 |
-
{"name": "groq", "env_key": "GROQ_API_KEY", "base_url": "https://api.groq.com/openai/v1", "model_env": "GROQ_MODEL", "default_model": "
|
| 65 |
{"name": "cerebras", "env_key": "CEREBRAS_API_KEY", "base_url": "https://api.cerebras.ai/v1", "model_env": "CEREBRAS_MODEL", "default_model": "gpt-oss-120b", "tier": 0, "purpose": "reasoning"},
|
| 66 |
{"name": "sambanova", "env_key": "SAMBANOVA_API_KEY", "base_url": "https://api.sambanova.ai/v1", "model_env": "SAMBANOVA_MODEL", "default_model": "DeepSeek-V3.1", "tier": 0, "purpose": "reasoning"},
|
| 67 |
# tier 1 — free tier con rate limit più stretti
|
| 68 |
-
{"name": "openrouter", "env_key": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "model_env": "OPENROUTER_MODEL","default_model": "
|
| 69 |
{"name": "hf_router", "env_key": "HF_TOKEN", "base_url": "https://router.huggingface.co/v1", "model_env": "HF_MODEL", "default_model": "Qwen/Qwen2.5-Coder-32B-Instruct", "tier": 1, "purpose": "coding"},
|
| 70 |
{"name": "gemini", "env_key": "GEMINI_API_KEY", "base_url": "https://generativelanguage.googleapis.com/v1beta/openai/", "model_env": "GEMINI_MODEL", "default_model": "gemini-3.6-flash", "tier": 1, "purpose": "memory"},
|
| 71 |
# tier 2 — fallback opzionale (spesso a pagamento o quota limitata)
|
|
|
|
| 61 |
# (nessun proxy CF Worker qui: questo client gira lato backend Python, non browser).
|
| 62 |
_PROVIDER_DEFS = [
|
| 63 |
# tier 0 — free tier veloce e affidabile
|
| 64 |
+
{"name": "groq", "env_key": "GROQ_API_KEY", "base_url": "https://api.groq.com/openai/v1", "model_env": "GROQ_MODEL", "default_model": "qwen/qwen3.6-27b", "tier": 0, "purpose": "reasoning"},
|
| 65 |
{"name": "cerebras", "env_key": "CEREBRAS_API_KEY", "base_url": "https://api.cerebras.ai/v1", "model_env": "CEREBRAS_MODEL", "default_model": "gpt-oss-120b", "tier": 0, "purpose": "reasoning"},
|
| 66 |
{"name": "sambanova", "env_key": "SAMBANOVA_API_KEY", "base_url": "https://api.sambanova.ai/v1", "model_env": "SAMBANOVA_MODEL", "default_model": "DeepSeek-V3.1", "tier": 0, "purpose": "reasoning"},
|
| 67 |
# tier 1 — free tier con rate limit più stretti
|
| 68 |
+
{"name": "openrouter", "env_key": "OPENROUTER_API_KEY", "base_url": "https://openrouter.ai/api/v1", "model_env": "OPENROUTER_MODEL","default_model": "openrouter/free", "tier": 1, "purpose": "coding"},
|
| 69 |
{"name": "hf_router", "env_key": "HF_TOKEN", "base_url": "https://router.huggingface.co/v1", "model_env": "HF_MODEL", "default_model": "Qwen/Qwen2.5-Coder-32B-Instruct", "tier": 1, "purpose": "coding"},
|
| 70 |
{"name": "gemini", "env_key": "GEMINI_API_KEY", "base_url": "https://generativelanguage.googleapis.com/v1beta/openai/", "model_env": "GEMINI_MODEL", "default_model": "gemini-3.6-flash", "tier": 1, "purpose": "memory"},
|
| 71 |
# tier 2 — fallback opzionale (spesso a pagamento o quota limitata)
|
models/role_router.py
CHANGED
|
@@ -3,27 +3,27 @@ role_router.py — Multi-model role routing (S362, aggiornato 2026-06-14 benchma
|
|
| 3 |
|
| 4 |
BENCHMARK RESULTS 2026-06-14 FINALE (14 modelli × 3 test, max_tokens corretti):
|
| 5 |
100% qualità (ordinati per TTFT):
|
| 6 |
-
#1 Groq /
|
| 7 |
#2 Cerebras / gpt-oss-120b — 207ms 100% ← REASONING (max_tokens≥500)
|
| 8 |
-
#3 Groq /
|
| 9 |
#4 Cerebras / zai-glm-4.7 — 254ms 100%
|
| 10 |
#5 Groq / compound-mini — 341ms 100%
|
| 11 |
#6 SambaNova / DeepSeek-V3.1 — 482ms 100%
|
| 12 |
#7 SambaNova / gemma-4-31B — 2132ms 100%
|
| 13 |
-
#8 OpenRouter /
|
| 14 |
|
| 15 |
Role assignments 2026-06-14 FINALE:
|
| 16 |
-
FAST → Groq
|
| 17 |
ARCHITECT → Groq llama-4-scout-17b 10M ctx (244ms, 67% — best per contesto lungo)
|
| 18 |
-
CODER → Groq
|
| 19 |
-
TESTER → Groq
|
| 20 |
-
CONTEXT → Groq
|
| 21 |
RESEARCHER → Gemini 2.5-flash (599ms, 67% — math prompt-sensitive)
|
| 22 |
REASONER → Cerebras gpt-oss-120b (207ms, 100%, reasoning model → max_tokens≥500)
|
| 23 |
SAMBANOVA → SambaNova DeepSeek-V3.1 (482ms, 100%)
|
| 24 |
-
DEFAULT → AIClient() primary (
|
| 25 |
|
| 26 |
-
OpenRouter tenuto come fallback secondario (
|
| 27 |
"""
|
| 28 |
from __future__ import annotations
|
| 29 |
|
|
@@ -36,11 +36,11 @@ _logger = logging.getLogger("models.role_router")
|
|
| 36 |
|
| 37 |
|
| 38 |
class Role(str, Enum):
|
| 39 |
-
FAST = "fast" # greetings, math semplice, identity —
|
| 40 |
ARCHITECT = "architect" # planning, ragionamento complesso — GPT-OSS 120B
|
| 41 |
-
CODER = "coder" # coding, debug —
|
| 42 |
-
TESTER = "tester" # test gen, debug hints —
|
| 43 |
-
CONTEXT = "context" # summarization, context compression —
|
| 44 |
DEFAULT = "default" # AIClient() primary
|
| 45 |
RESEARCHER = "researcher" # web research + document synthesis — GPT-OSS 120B
|
| 46 |
REASONER = "reasoner" # throughput massimo — Cerebras GPT-OSS 120B
|
|
@@ -122,7 +122,7 @@ class RoleRouter:
|
|
| 122 |
name="groq-fast",
|
| 123 |
api_key=groq_key,
|
| 124 |
base_url="https://api.groq.com/openai/v1",
|
| 125 |
-
default_model=os.getenv("GROQ_FAST_MODEL", "
|
| 126 |
)
|
| 127 |
rest = [p for p in client.providers if p.name not in ("groq", "groq-fast", "groq-tester")]
|
| 128 |
client.providers = [fast, *rest]
|
|
@@ -163,7 +163,7 @@ class RoleRouter:
|
|
| 163 |
name="groq-architect",
|
| 164 |
api_key=groq_key,
|
| 165 |
base_url="https://api.groq.com/openai/v1",
|
| 166 |
-
default_model=os.getenv("ARCHITECT_MODEL", "
|
| 167 |
)
|
| 168 |
rest = [p for p in client.providers if p.name not in ("groq", "groq-architect")]
|
| 169 |
client.providers = [architect, *rest]
|
|
@@ -179,7 +179,7 @@ class RoleRouter:
|
|
| 179 |
profiles = [ProviderConfig(
|
| 180 |
name="openrouter", api_key=os.getenv("OPENROUTER_API_KEY", ""),
|
| 181 |
base_url="https://openrouter.ai/api/v1",
|
| 182 |
-
default_model=os.getenv("OPENROUTER_MODEL", "
|
| 183 |
profile="legacy",
|
| 184 |
)]
|
| 185 |
if profiles:
|
|
@@ -201,7 +201,7 @@ class RoleRouter:
|
|
| 201 |
return profiled
|
| 202 |
groq_key = os.getenv("GROQ_API_KEY")
|
| 203 |
nvidia_key = os.getenv("NVIDIA_API_KEY")
|
| 204 |
-
model = os.getenv("CODER_MODEL", "
|
| 205 |
if groq_key:
|
| 206 |
client = AIClient()
|
| 207 |
coder = ProviderConfig(
|
|
@@ -241,7 +241,7 @@ class RoleRouter:
|
|
| 241 |
profiles = [ProviderConfig(
|
| 242 |
name="openrouter", api_key=os.getenv("OPENROUTER_API_KEY", ""),
|
| 243 |
base_url="https://openrouter.ai/api/v1",
|
| 244 |
-
default_model=os.getenv("OPENROUTER_MODEL", "
|
| 245 |
profile="legacy",
|
| 246 |
)]
|
| 247 |
if profiles:
|
|
@@ -273,7 +273,7 @@ class RoleRouter:
|
|
| 273 |
base_url="https://api.groq.com/openai/v1",
|
| 274 |
default_model=os.getenv(
|
| 275 |
"GROQ_RESEARCH_MODEL",
|
| 276 |
-
os.getenv("GROQ_MODEL", "
|
| 277 |
),
|
| 278 |
)
|
| 279 |
rest = [
|
|
@@ -307,7 +307,7 @@ class RoleRouter:
|
|
| 307 |
base_url="https://api.groq.com/openai/v1",
|
| 308 |
default_model=os.getenv(
|
| 309 |
"GROQ_REASONER_MODEL",
|
| 310 |
-
os.getenv("GROQ_MODEL", "
|
| 311 |
),
|
| 312 |
)
|
| 313 |
rest = [
|
|
@@ -406,7 +406,7 @@ class RoleRouter:
|
|
| 406 |
name="groq-tester",
|
| 407 |
api_key=groq_key,
|
| 408 |
base_url="https://api.groq.com/openai/v1",
|
| 409 |
-
default_model=os.getenv("GROQ_FAST_MODEL", "
|
| 410 |
purpose="coding",
|
| 411 |
)
|
| 412 |
rest = [p for p in client.providers if p.name not in ("groq", "groq-tester")]
|
|
|
|
| 3 |
|
| 4 |
BENCHMARK RESULTS 2026-06-14 FINALE (14 modelli × 3 test, max_tokens corretti):
|
| 5 |
100% qualità (ordinati per TTFT):
|
| 6 |
+
#1 Groq / qwen/qwen3.6-27b — 170ms 100% ← FASTEST
|
| 7 |
#2 Cerebras / gpt-oss-120b — 207ms 100% ← REASONING (max_tokens≥500)
|
| 8 |
+
#3 Groq / qwen/qwen3.6-27b — 235ms 100%
|
| 9 |
#4 Cerebras / zai-glm-4.7 — 254ms 100%
|
| 10 |
#5 Groq / compound-mini — 341ms 100%
|
| 11 |
#6 SambaNova / DeepSeek-V3.1 — 482ms 100%
|
| 12 |
#7 SambaNova / gemma-4-31B — 2132ms 100%
|
| 13 |
+
#8 OpenRouter / openrouter/free — 2160ms 100%
|
| 14 |
|
| 15 |
Role assignments 2026-06-14 FINALE:
|
| 16 |
+
FAST → Groq qwen/qwen3.6-27b (170ms, 100%) ← #1 assoluto
|
| 17 |
ARCHITECT → Groq llama-4-scout-17b 10M ctx (244ms, 67% — best per contesto lungo)
|
| 18 |
+
CODER → Groq qwen/qwen3.6-27b (235ms, 100%) ← #3 qualità
|
| 19 |
+
TESTER → Groq qwen/qwen3.6-27b
|
| 20 |
+
CONTEXT → Groq qwen/qwen3.6-27b
|
| 21 |
RESEARCHER → Gemini 2.5-flash (599ms, 67% — math prompt-sensitive)
|
| 22 |
REASONER → Cerebras gpt-oss-120b (207ms, 100%, reasoning model → max_tokens≥500)
|
| 23 |
SAMBANOVA → SambaNova DeepSeek-V3.1 (482ms, 100%)
|
| 24 |
+
DEFAULT → AIClient() primary (qwen/qwen3.6-27b o primo disponibile)
|
| 25 |
|
| 26 |
+
OpenRouter tenuto come fallback secondario (openrouter/free = 1645ms ma 100% qualità).
|
| 27 |
"""
|
| 28 |
from __future__ import annotations
|
| 29 |
|
|
|
|
| 36 |
|
| 37 |
|
| 38 |
class Role(str, Enum):
|
| 39 |
+
FAST = "fast" # greetings, math semplice, identity — qwen/qwen3.6-27b
|
| 40 |
ARCHITECT = "architect" # planning, ragionamento complesso — GPT-OSS 120B
|
| 41 |
+
CODER = "coder" # coding, debug — qwen/qwen3.6-27b
|
| 42 |
+
TESTER = "tester" # test gen, debug hints — qwen/qwen3.6-27b
|
| 43 |
+
CONTEXT = "context" # summarization, context compression — qwen/qwen3.6-27b
|
| 44 |
DEFAULT = "default" # AIClient() primary
|
| 45 |
RESEARCHER = "researcher" # web research + document synthesis — GPT-OSS 120B
|
| 46 |
REASONER = "reasoner" # throughput massimo — Cerebras GPT-OSS 120B
|
|
|
|
| 122 |
name="groq-fast",
|
| 123 |
api_key=groq_key,
|
| 124 |
base_url="https://api.groq.com/openai/v1",
|
| 125 |
+
default_model=os.getenv("GROQ_FAST_MODEL", "qwen/qwen3.6-27b"),
|
| 126 |
)
|
| 127 |
rest = [p for p in client.providers if p.name not in ("groq", "groq-fast", "groq-tester")]
|
| 128 |
client.providers = [fast, *rest]
|
|
|
|
| 163 |
name="groq-architect",
|
| 164 |
api_key=groq_key,
|
| 165 |
base_url="https://api.groq.com/openai/v1",
|
| 166 |
+
default_model=os.getenv("ARCHITECT_MODEL", "qwen/qwen3.6-27b"),
|
| 167 |
)
|
| 168 |
rest = [p for p in client.providers if p.name not in ("groq", "groq-architect")]
|
| 169 |
client.providers = [architect, *rest]
|
|
|
|
| 179 |
profiles = [ProviderConfig(
|
| 180 |
name="openrouter", api_key=os.getenv("OPENROUTER_API_KEY", ""),
|
| 181 |
base_url="https://openrouter.ai/api/v1",
|
| 182 |
+
default_model=os.getenv("OPENROUTER_MODEL", "openrouter/free"),
|
| 183 |
profile="legacy",
|
| 184 |
)]
|
| 185 |
if profiles:
|
|
|
|
| 201 |
return profiled
|
| 202 |
groq_key = os.getenv("GROQ_API_KEY")
|
| 203 |
nvidia_key = os.getenv("NVIDIA_API_KEY")
|
| 204 |
+
model = os.getenv("CODER_MODEL", "qwen/qwen3.6-27b")
|
| 205 |
if groq_key:
|
| 206 |
client = AIClient()
|
| 207 |
coder = ProviderConfig(
|
|
|
|
| 241 |
profiles = [ProviderConfig(
|
| 242 |
name="openrouter", api_key=os.getenv("OPENROUTER_API_KEY", ""),
|
| 243 |
base_url="https://openrouter.ai/api/v1",
|
| 244 |
+
default_model=os.getenv("OPENROUTER_MODEL", "openrouter/free"),
|
| 245 |
profile="legacy",
|
| 246 |
)]
|
| 247 |
if profiles:
|
|
|
|
| 273 |
base_url="https://api.groq.com/openai/v1",
|
| 274 |
default_model=os.getenv(
|
| 275 |
"GROQ_RESEARCH_MODEL",
|
| 276 |
+
os.getenv("GROQ_MODEL", "qwen/qwen3.6-27b"),
|
| 277 |
),
|
| 278 |
)
|
| 279 |
rest = [
|
|
|
|
| 307 |
base_url="https://api.groq.com/openai/v1",
|
| 308 |
default_model=os.getenv(
|
| 309 |
"GROQ_REASONER_MODEL",
|
| 310 |
+
os.getenv("GROQ_MODEL", "qwen/qwen3.6-27b"),
|
| 311 |
),
|
| 312 |
)
|
| 313 |
rest = [
|
|
|
|
| 406 |
name="groq-tester",
|
| 407 |
api_key=groq_key,
|
| 408 |
base_url="https://api.groq.com/openai/v1",
|
| 409 |
+
default_model=os.getenv("GROQ_FAST_MODEL", "qwen/qwen3.6-27b"),
|
| 410 |
purpose="coding",
|
| 411 |
)
|
| 412 |
rest = [p for p in client.providers if p.name not in ("groq", "groq-tester")]
|
tests/test_model_catalog_scan.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import unittest
|
| 2 |
+
|
| 3 |
+
from scripts.model_catalog_scan import diff_catalogs
|
| 4 |
+
from scripts.model_catalog_notify import build_body
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class ModelCatalogDiffTests(unittest.TestCase):
|
| 8 |
+
def test_reports_exact_added_removed_and_unavailable_models(self):
|
| 9 |
+
previous = {"state": {"groq/A": {"provider": "groq", "profile": "A", "status": "available", "models": ["old-model", "stable-model"]}}}
|
| 10 |
+
current = {"state": {"groq/A": {"provider": "groq", "profile": "A", "status": "available", "models": ["new-model", "stable-model"], "default_model": "old-model", "default_available": False}}}
|
| 11 |
+
result = diff_catalogs(previous, current)
|
| 12 |
+
self.assertEqual(result["added_models"][0]["model"], "new-model")
|
| 13 |
+
self.assertEqual(result["removed_models"][0]["model"], "old-model")
|
| 14 |
+
self.assertEqual(result["unavailable_defaults"][0]["model"], "old-model")
|
| 15 |
+
self.assertTrue(result["has_changes"])
|
| 16 |
+
|
| 17 |
+
def test_reports_provider_errors_with_exact_status(self):
|
| 18 |
+
current = {"state": {"gemini/default": {"provider": "gemini", "profile": "default", "status": "rate_limited", "models": [], "detail": "retry after 60"}}}
|
| 19 |
+
result = diff_catalogs({}, current)
|
| 20 |
+
self.assertEqual(result["provider_errors"][0]["status"], "rate_limited")
|
| 21 |
+
self.assertIn("gemini", result["provider_errors"][0]["provider"])
|
| 22 |
+
|
| 23 |
+
def test_empty_profile_configuration_is_reported(self):
|
| 24 |
+
from scripts.model_catalog_scan import audit_payload
|
| 25 |
+
from benchmarks.model_watch_adapter import ModelWatchConfig, ObserveOnlyModelsAdapter
|
| 26 |
+
from types import SimpleNamespace
|
| 27 |
+
payload = audit_payload(SimpleNamespace(results=(), skipped_rate_limited=()), ObserveOnlyModelsAdapter(config=ModelWatchConfig()), [], {})
|
| 28 |
+
self.assertEqual(payload["profile_count"], 0)
|
| 29 |
+
self.assertTrue(payload["diff"]["has_changes"])
|
| 30 |
+
self.assertEqual(payload["diff"]["provider_errors"][0]["status"], "missing_profiles")
|
| 31 |
+
|
| 32 |
+
def test_notification_contains_exact_names_and_closed_gate(self):
|
| 33 |
+
body = build_body({
|
| 34 |
+
"auto_apply_gate_open": False,
|
| 35 |
+
"diff": {
|
| 36 |
+
"added_models": [{"provider": "groq", "profile": "A", "model": "new-model"}],
|
| 37 |
+
"removed_models": [{"provider": "groq", "profile": "A", "model": "old-model"}],
|
| 38 |
+
"unavailable_defaults": [],
|
| 39 |
+
"provider_errors": [],
|
| 40 |
+
},
|
| 41 |
+
})
|
| 42 |
+
self.assertIn("new-model", body)
|
| 43 |
+
self.assertIn("old-model", body)
|
| 44 |
+
self.assertIn("CHIUSA", body)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
if __name__ == "__main__":
|
| 48 |
+
unittest.main()
|
tests/test_provider_model_migrations.py
CHANGED
|
@@ -47,7 +47,7 @@ class RoleRouterModelMigrationTests(unittest.TestCase):
|
|
| 47 |
client = RoleRouter.get_client(Role.ARCHITECT)
|
| 48 |
|
| 49 |
self.assertEqual(client.provider_name, "groq-architect")
|
| 50 |
-
self.assertEqual(client.default_model, "
|
| 51 |
|
| 52 |
@patch.object(AIClient, "_load_providers", return_value=[])
|
| 53 |
@patch.dict(
|
|
@@ -59,8 +59,8 @@ class RoleRouterModelMigrationTests(unittest.TestCase):
|
|
| 59 |
architect = RoleRouter.get_client(Role.ARCHITECT)
|
| 60 |
coder = RoleRouter.get_client(Role.CODER)
|
| 61 |
|
| 62 |
-
self.assertEqual(architect.default_model, "
|
| 63 |
-
self.assertEqual(coder.default_model, "
|
| 64 |
|
| 65 |
@patch.object(AIClient, "_load_providers", return_value=[])
|
| 66 |
@patch.dict(
|
|
@@ -90,14 +90,14 @@ class ProviderTableMigrationTests(unittest.IsolatedAsyncioTestCase):
|
|
| 90 |
self.assertEqual(len(fake_supabase.table_ref.calls), 8)
|
| 91 |
|
| 92 |
expected = {
|
| 93 |
-
("groq", "llama-3.1-70b-versatile", "
|
| 94 |
("cerebras", "llama3.1-70b", "gpt-oss-120b"),
|
| 95 |
("nvidia", "llama-3.1-405b-instruct", "meta/llama-3.3-70b-instruct"),
|
| 96 |
-
("openrouter", "llama-3.1-405b", "
|
| 97 |
("sambanova", "llama3-70b", "DeepSeek-V3.2"),
|
| 98 |
("gemini", "gemini-1.5-flash", "gemini-3.5-flash-lite"),
|
| 99 |
("gemini", "gemini-1.5-pro", "gemini-3.6-flash"),
|
| 100 |
-
("openrouter", "claude-3.5-sonnet", "
|
| 101 |
}
|
| 102 |
actual = {
|
| 103 |
(
|
|
|
|
| 47 |
client = RoleRouter.get_client(Role.ARCHITECT)
|
| 48 |
|
| 49 |
self.assertEqual(client.provider_name, "groq-architect")
|
| 50 |
+
self.assertEqual(client.default_model, "qwen/qwen3.6-27b")
|
| 51 |
|
| 52 |
@patch.object(AIClient, "_load_providers", return_value=[])
|
| 53 |
@patch.dict(
|
|
|
|
| 59 |
architect = RoleRouter.get_client(Role.ARCHITECT)
|
| 60 |
coder = RoleRouter.get_client(Role.CODER)
|
| 61 |
|
| 62 |
+
self.assertEqual(architect.default_model, "openrouter/free")
|
| 63 |
+
self.assertEqual(coder.default_model, "openrouter/free")
|
| 64 |
|
| 65 |
@patch.object(AIClient, "_load_providers", return_value=[])
|
| 66 |
@patch.dict(
|
|
|
|
| 90 |
self.assertEqual(len(fake_supabase.table_ref.calls), 8)
|
| 91 |
|
| 92 |
expected = {
|
| 93 |
+
("groq", "llama-3.1-70b-versatile", "qwen/qwen3.6-27b"),
|
| 94 |
("cerebras", "llama3.1-70b", "gpt-oss-120b"),
|
| 95 |
("nvidia", "llama-3.1-405b-instruct", "meta/llama-3.3-70b-instruct"),
|
| 96 |
+
("openrouter", "llama-3.1-405b", "openrouter/free"),
|
| 97 |
("sambanova", "llama3-70b", "DeepSeek-V3.2"),
|
| 98 |
("gemini", "gemini-1.5-flash", "gemini-3.5-flash-lite"),
|
| 99 |
("gemini", "gemini-1.5-pro", "gemini-3.6-flash"),
|
| 100 |
+
("openrouter", "claude-3.5-sonnet", "openrouter/free"),
|
| 101 |
}
|
| 102 |
actual = {
|
| 103 |
(
|