Spaces:
Running
Running
sync: 165 file da Baida98/AI@2d40c46b (2026-08-21 15:45 UTC)
#48
by Baida07 - opened
- benchmarks/model_watch_adapter.py +75 -2
- main.py +12 -0
- tests/test_cognitive_gaps.py +8 -3
- tests/test_model_watch_adapter.py +35 -0
- tests/test_regression_doc2.py +15 -7
- tests/test_scaffold_project.py +2 -2
benchmarks/model_watch_adapter.py
CHANGED
|
@@ -10,8 +10,9 @@ from dataclasses import dataclass, field, replace
|
|
| 10 |
import asyncio
|
| 11 |
from enum import Enum
|
| 12 |
import json
|
|
|
|
| 13 |
import re
|
| 14 |
-
from typing import Any, Mapping, Optional
|
| 15 |
|
| 16 |
import httpx
|
| 17 |
|
|
@@ -27,6 +28,43 @@ class CatalogStatus(str, Enum):
|
|
| 27 |
MALFORMED = "malformed"
|
| 28 |
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
@dataclass(frozen=True)
|
| 31 |
class ProviderProfile:
|
| 32 |
provider: str
|
|
@@ -147,9 +185,44 @@ class ProfileScan:
|
|
| 147 |
class ObserveOnlyModelsAdapter:
|
| 148 |
"""Fetch a provider catalog and classify the result; never mutates state."""
|
| 149 |
|
| 150 |
-
def __init__(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
self.timeout_seconds = timeout_seconds
|
| 152 |
self._client = client
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
|
| 154 |
async def list_models(self, profile: ProviderProfile) -> CatalogResult:
|
| 155 |
url = models_url(profile.base_url)
|
|
|
|
| 10 |
import asyncio
|
| 11 |
from enum import Enum
|
| 12 |
import json
|
| 13 |
+
import os
|
| 14 |
import re
|
| 15 |
+
from typing import Any, Awaitable, Callable, Mapping, Optional
|
| 16 |
|
| 17 |
import httpx
|
| 18 |
|
|
|
|
| 28 |
MALFORMED = "malformed"
|
| 29 |
|
| 30 |
|
| 31 |
+
@dataclass(frozen=True)
|
| 32 |
+
class ModelWatchConfig:
|
| 33 |
+
"""Safety gate for optional model updates.
|
| 34 |
+
|
| 35 |
+
Discovery remains observe-only by default. Auto-apply is enabled only when
|
| 36 |
+
the explicit flag and approval marker are both present; callers must also
|
| 37 |
+
provide an allowlist of provider/old/new model triples.
|
| 38 |
+
"""
|
| 39 |
+
|
| 40 |
+
auto_apply_enabled: bool = False
|
| 41 |
+
approval_marker: str = ""
|
| 42 |
+
required_approval_marker: str = "I_UNDERSTAND_MODEL_UPDATES"
|
| 43 |
+
approved_updates: tuple[tuple[str, str, str], ...] = ()
|
| 44 |
+
|
| 45 |
+
@classmethod
|
| 46 |
+
def from_env(cls) -> "ModelWatchConfig":
|
| 47 |
+
raw_updates = os.getenv("MODEL_AUTO_APPLY_ALLOWLIST", "")
|
| 48 |
+
updates: list[tuple[str, str, str]] = []
|
| 49 |
+
for item in raw_updates.split(","):
|
| 50 |
+
parts = tuple(part.strip() for part in item.split("|"))
|
| 51 |
+
if len(parts) == 3 and all(parts):
|
| 52 |
+
updates.append(parts) # type: ignore[arg-type]
|
| 53 |
+
return cls(
|
| 54 |
+
auto_apply_enabled=os.getenv("MODEL_AUTO_APPLY_ENABLED", "0").lower() in {"1", "true", "yes"},
|
| 55 |
+
approval_marker=os.getenv("MODEL_AUTO_APPLY_APPROVAL", ""),
|
| 56 |
+
approved_updates=tuple(updates),
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
@property
|
| 60 |
+
def can_auto_apply(self) -> bool:
|
| 61 |
+
return (
|
| 62 |
+
self.auto_apply_enabled
|
| 63 |
+
and self.approval_marker == self.required_approval_marker
|
| 64 |
+
and bool(self.approved_updates)
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
@dataclass(frozen=True)
|
| 69 |
class ProviderProfile:
|
| 70 |
provider: str
|
|
|
|
| 185 |
class ObserveOnlyModelsAdapter:
|
| 186 |
"""Fetch a provider catalog and classify the result; never mutates state."""
|
| 187 |
|
| 188 |
+
def __init__(
|
| 189 |
+
self,
|
| 190 |
+
*,
|
| 191 |
+
timeout_seconds: float = 8.0,
|
| 192 |
+
client: httpx.AsyncClient | None = None,
|
| 193 |
+
config: ModelWatchConfig | None = None,
|
| 194 |
+
):
|
| 195 |
self.timeout_seconds = timeout_seconds
|
| 196 |
self._client = client
|
| 197 |
+
self.config = config or ModelWatchConfig.from_env()
|
| 198 |
+
|
| 199 |
+
@property
|
| 200 |
+
def can_auto_apply(self) -> bool:
|
| 201 |
+
"""True only when every explicit safety gate is satisfied."""
|
| 202 |
+
return self.config.can_auto_apply
|
| 203 |
+
|
| 204 |
+
async def apply_updates(
|
| 205 |
+
self,
|
| 206 |
+
updates: list[tuple[str, str, str]],
|
| 207 |
+
apply_callback: Callable[[str, str, str], Awaitable[None]],
|
| 208 |
+
) -> dict[str, Any]:
|
| 209 |
+
"""Apply only allowlisted updates through a caller-owned callback.
|
| 210 |
+
|
| 211 |
+
The adapter never receives a database client and cannot mutate state on
|
| 212 |
+
its own. With the default config this returns a dry-run result.
|
| 213 |
+
"""
|
| 214 |
+
if not self.can_auto_apply:
|
| 215 |
+
return {"applied": False, "dry_run": True, "reason": "auto_apply_disabled"}
|
| 216 |
+
approved = set(self.config.approved_updates)
|
| 217 |
+
applied = 0
|
| 218 |
+
skipped = 0
|
| 219 |
+
for provider, old_model, new_model in updates:
|
| 220 |
+
if (provider, old_model, new_model) not in approved:
|
| 221 |
+
skipped += 1
|
| 222 |
+
continue
|
| 223 |
+
await apply_callback(provider, old_model, new_model)
|
| 224 |
+
applied += 1
|
| 225 |
+
return {"applied": applied > 0, "dry_run": False, "applied_count": applied, "skipped_count": skipped}
|
| 226 |
|
| 227 |
async def list_models(self, profile: ProviderProfile) -> CatalogResult:
|
| 228 |
url = models_url(profile.base_url)
|
main.py
CHANGED
|
@@ -191,6 +191,18 @@ for prefix, module_name in _ROUTER_MAP.items():
|
|
| 191 |
except Exception as e:
|
| 192 |
_logger.error(f"β Errore montaggio rotta {prefix}: {e}")
|
| 193 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
# ββ CLI Task Execution ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 195 |
async def run_cli_task(task_description: str):
|
| 196 |
_logger.info(f"CLI: Avvio task richiesto: {task_description[:50]}...")
|
|
|
|
| 191 |
except Exception as e:
|
| 192 |
_logger.error(f"β Errore montaggio rotta {prefix}: {e}")
|
| 193 |
|
| 194 |
+
# ββ Memory sync protocol βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 195 |
+
# Γ una factory parametrica, quindi non puΓ² stare in _ROUTER_MAP. Riutilizza il
|
| 196 |
+
# singleton lazy di state.py per evitare una seconda istanza di MemoryManager.
|
| 197 |
+
try:
|
| 198 |
+
from memory.sync import create_memory_sync_router
|
| 199 |
+
from api.state import _get_mem_manager
|
| 200 |
+
_sync_router = create_memory_sync_router(_get_mem_manager())
|
| 201 |
+
app.include_router(_sync_router)
|
| 202 |
+
_logger.info("β
Route montata: /api/memory/sync (da memory.sync)")
|
| 203 |
+
except Exception as e:
|
| 204 |
+
_logger.error(f"β Errore montaggio memory sync router: {e}")
|
| 205 |
+
|
| 206 |
# ββ CLI Task Execution ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 207 |
async def run_cli_task(task_description: str):
|
| 208 |
_logger.info(f"CLI: Avvio task richiesto: {task_description[:50]}...")
|
tests/test_cognitive_gaps.py
CHANGED
|
@@ -24,7 +24,8 @@ if _BACKEND not in sys.path:
|
|
| 24 |
sys.path.insert(0, _BACKEND)
|
| 25 |
|
| 26 |
def _run(coro):
|
| 27 |
-
|
|
|
|
| 28 |
|
| 29 |
|
| 30 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -642,7 +643,9 @@ class TestCOG5WiringInUnifiedLoop(unittest.TestCase):
|
|
| 642 |
"""COG-5 wiring: in caso di drift, il messaggio viene aggiunto a exec_warn."""
|
| 643 |
idx = self.src.find("goal_drift_detector")
|
| 644 |
self.assertGreater(idx, 0)
|
| 645 |
-
|
|
|
|
|
|
|
| 646 |
self.assertIn("exec_warn.append", block)
|
| 647 |
|
| 648 |
def test_cog5_is_non_blocking(self):
|
|
@@ -650,7 +653,9 @@ class TestCOG5WiringInUnifiedLoop(unittest.TestCase):
|
|
| 650 |
idx = self.src.find("goal_drift_detector")
|
| 651 |
self.assertGreater(idx, 0)
|
| 652 |
# La try/except deve precedere l'import
|
| 653 |
-
|
|
|
|
|
|
|
| 654 |
self.assertIn("except Exception as _cog5_err", pre_block)
|
| 655 |
|
| 656 |
def test_cog5_marker_in_source(self):
|
|
|
|
| 24 |
sys.path.insert(0, _BACKEND)
|
| 25 |
|
| 26 |
def _run(coro):
|
| 27 |
+
"""Esegue una coroutine anche quando Python non ha un event loop corrente."""
|
| 28 |
+
return asyncio.run(coro)
|
| 29 |
|
| 30 |
|
| 31 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 643 |
"""COG-5 wiring: in caso di drift, il messaggio viene aggiunto a exec_warn."""
|
| 644 |
idx = self.src.find("goal_drift_detector")
|
| 645 |
self.assertGreater(idx, 0)
|
| 646 |
+
# Il blocco COG-5 puΓ² crescere con il logging diagnostico: non usare
|
| 647 |
+
# una finestra corta che tronca l'append effettivo.
|
| 648 |
+
block = self.src[idx: idx + 2200]
|
| 649 |
self.assertIn("exec_warn.append", block)
|
| 650 |
|
| 651 |
def test_cog5_is_non_blocking(self):
|
|
|
|
| 653 |
idx = self.src.find("goal_drift_detector")
|
| 654 |
self.assertGreater(idx, 0)
|
| 655 |
# La try/except deve precedere l'import
|
| 656 |
+
# L'import e il relativo guard devono restare nello stesso blocco COG-5;
|
| 657 |
+
# la finestra include anche il logging aggiunto dopo il fix originale.
|
| 658 |
+
pre_block = self.src[max(0, idx - 500): idx + 2200]
|
| 659 |
self.assertIn("except Exception as _cog5_err", pre_block)
|
| 660 |
|
| 661 |
def test_cog5_marker_in_source(self):
|
tests/test_model_watch_adapter.py
CHANGED
|
@@ -27,6 +27,41 @@ class ModelWatchAdapterTests(unittest.IsolatedAsyncioTestCase):
|
|
| 27 |
transport = httpx.MockTransport(handler)
|
| 28 |
return ObserveOnlyModelsAdapter(client=httpx.AsyncClient(transport=transport))
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
async def test_catalog_available_and_default_present(self):
|
| 31 |
async def handler(request):
|
| 32 |
self.assertEqual(request.url.path, "/openai/v1/models")
|
|
|
|
| 27 |
transport = httpx.MockTransport(handler)
|
| 28 |
return ObserveOnlyModelsAdapter(client=httpx.AsyncClient(transport=transport))
|
| 29 |
|
| 30 |
+
async def test_auto_apply_is_disabled_by_default(self):
|
| 31 |
+
adapter = ObserveOnlyModelsAdapter()
|
| 32 |
+
calls = []
|
| 33 |
+
|
| 34 |
+
async def callback(provider, old_model, new_model):
|
| 35 |
+
calls.append((provider, old_model, new_model))
|
| 36 |
+
|
| 37 |
+
result = await adapter.apply_updates([("groq", "old", "new")], callback)
|
| 38 |
+
self.assertFalse(adapter.can_auto_apply)
|
| 39 |
+
self.assertEqual(result["reason"], "auto_apply_disabled")
|
| 40 |
+
self.assertEqual(calls, [])
|
| 41 |
+
|
| 42 |
+
async def test_auto_apply_requires_marker_and_allowlist(self):
|
| 43 |
+
from benchmarks.model_watch_adapter import ModelWatchConfig
|
| 44 |
+
|
| 45 |
+
config = ModelWatchConfig(
|
| 46 |
+
auto_apply_enabled=True,
|
| 47 |
+
approval_marker="I_UNDERSTAND_MODEL_UPDATES",
|
| 48 |
+
approved_updates=(("groq", "old", "new"),),
|
| 49 |
+
)
|
| 50 |
+
adapter = ObserveOnlyModelsAdapter(config=config)
|
| 51 |
+
calls = []
|
| 52 |
+
|
| 53 |
+
async def callback(provider, old_model, new_model):
|
| 54 |
+
calls.append((provider, old_model, new_model))
|
| 55 |
+
|
| 56 |
+
result = await adapter.apply_updates(
|
| 57 |
+
[("groq", "old", "new"), ("gemini", "old", "new")],
|
| 58 |
+
callback,
|
| 59 |
+
)
|
| 60 |
+
self.assertTrue(adapter.can_auto_apply)
|
| 61 |
+
self.assertEqual(result["applied_count"], 1)
|
| 62 |
+
self.assertEqual(result["skipped_count"], 1)
|
| 63 |
+
self.assertEqual(calls, [("groq", "old", "new")])
|
| 64 |
+
|
| 65 |
async def test_catalog_available_and_default_present(self):
|
| 66 |
async def handler(request):
|
| 67 |
self.assertEqual(request.url.path, "/openai/v1/models")
|
tests/test_regression_doc2.py
CHANGED
|
@@ -189,7 +189,7 @@ class TestMemorySyncRouterMount(unittest.TestCase):
|
|
| 189 |
f"Prefix sbagliato: {router.prefix}")
|
| 190 |
|
| 191 |
def test_sync_router_has_required_endpoints(self):
|
| 192 |
-
"""Doc2-1b: router espone
|
| 193 |
try:
|
| 194 |
from memory.sync import create_memory_sync_router
|
| 195 |
except ImportError as e:
|
|
@@ -204,9 +204,9 @@ class TestMemorySyncRouterMount(unittest.TestCase):
|
|
| 204 |
|
| 205 |
router = create_memory_sync_router(_MemStub())
|
| 206 |
paths = {r.path for r in router.routes}
|
| 207 |
-
self.assertIn("/status", paths, "
|
| 208 |
-
self.assertIn("/push", paths, "
|
| 209 |
-
self.assertIn("/pull", paths, "
|
| 210 |
|
| 211 |
def test_main_py_mounts_sync_router(self):
|
| 212 |
"""
|
|
@@ -282,9 +282,17 @@ class TestTerminalRoutingNotFixedOnSpaceA(unittest.TestCase):
|
|
| 282 |
src = self._read(self._AGENT_SSE)
|
| 283 |
idx_chain = src.find("_getBackendChain")
|
| 284 |
self.assertNotEqual(idx_chain, -1, "_getBackendChain non trovato in agentSSE.ts")
|
| 285 |
-
chain_block = src[idx_chain: idx_chain +
|
| 286 |
-
|
| 287 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
self.assertNotIn("arjanit98-terminal.hf.space", chain_block,
|
| 289 |
"Space ritirato presente nella catena di fallback")
|
| 290 |
self.assertNotIn("baida00-ai-backend-collab.hf.space", chain_block,
|
|
|
|
| 189 |
f"Prefix sbagliato: {router.prefix}")
|
| 190 |
|
| 191 |
def test_sync_router_has_required_endpoints(self):
|
| 192 |
+
"""Doc2-1b: router espone status, push e pull sotto il prefisso API."""
|
| 193 |
try:
|
| 194 |
from memory.sync import create_memory_sync_router
|
| 195 |
except ImportError as e:
|
|
|
|
| 204 |
|
| 205 |
router = create_memory_sync_router(_MemStub())
|
| 206 |
paths = {r.path for r in router.routes}
|
| 207 |
+
self.assertIn("/api/memory/sync/status", paths, "status mancante dal sync router")
|
| 208 |
+
self.assertIn("/api/memory/sync/push", paths, "push mancante dal sync router")
|
| 209 |
+
self.assertIn("/api/memory/sync/pull", paths, "pull mancante dal sync router")
|
| 210 |
|
| 211 |
def test_main_py_mounts_sync_router(self):
|
| 212 |
"""
|
|
|
|
| 282 |
src = self._read(self._AGENT_SSE)
|
| 283 |
idx_chain = src.find("_getBackendChain")
|
| 284 |
self.assertNotEqual(idx_chain, -1, "_getBackendChain non trovato in agentSSE.ts")
|
| 285 |
+
chain_block = src[idx_chain: idx_chain + 1800]
|
| 286 |
+
# In produzione il contratto corrente Γ¨ il proxy same-origin CF Worker;
|
| 287 |
+
# in locale la catena Γ¨ interamente configurata tramite ENV.*.
|
| 288 |
+
self.assertIn('if (isProd) return ["/api"]', chain_block,
|
| 289 |
+
"Il routing production non usa il proxy same-origin /api")
|
| 290 |
+
for env_name in (
|
| 291 |
+
"ENV.BACKEND_URL", "ENV.BACKEND_URL_2", "ENV.BACKEND_URL_C",
|
| 292 |
+
"ENV.BACKEND_URL_D", "ENV.BACKEND_URL_E", "ENV.BACKEND_URL_HF_B",
|
| 293 |
+
):
|
| 294 |
+
self.assertIn(env_name, chain_block,
|
| 295 |
+
f"Fallback configurabile mancante: {env_name}")
|
| 296 |
self.assertNotIn("arjanit98-terminal.hf.space", chain_block,
|
| 297 |
"Space ritirato presente nella catena di fallback")
|
| 298 |
self.assertNotIn("baida00-ai-backend-collab.hf.space", chain_block,
|
tests/test_scaffold_project.py
CHANGED
|
@@ -37,8 +37,8 @@ if _BACKEND not in sys.path:
|
|
| 37 |
|
| 38 |
|
| 39 |
def _run(coro):
|
| 40 |
-
"""
|
| 41 |
-
return asyncio.
|
| 42 |
|
| 43 |
|
| 44 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 37 |
|
| 38 |
|
| 39 |
def _run(coro):
|
| 40 |
+
"""Esegue una coroutine anche quando Python non ha un event loop corrente."""
|
| 41 |
+
return asyncio.run(coro)
|
| 42 |
|
| 43 |
|
| 44 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|