Spaces:
Running
Running
fix: start provider heartbeat and guard initial state
Browse files- api/providers.py +10 -8
- api/state.py +4 -0
- main.py +6 -0
- tests/test_provider_heartbeat_state.py +43 -0
api/providers.py
CHANGED
|
@@ -428,15 +428,17 @@ async def debug_timing(role: AuthRole = Depends(require_role(AuthRole.MACHINE)))
|
|
| 428 |
@router.get("/api/providers/heartbeat")
|
| 429 |
async def providers_heartbeat(role: AuthRole = Depends(require_role(AuthRole.MACHINE))): # GAP-1-fix
|
| 430 |
now = int(time.time())
|
|
|
|
|
|
|
| 431 |
return {
|
| 432 |
-
"status": _heartbeat_state
|
| 433 |
-
"best_provider": _heartbeat_state
|
| 434 |
-
"best_latency_ms": _heartbeat_state
|
| 435 |
-
"providers": _heartbeat_state
|
| 436 |
-
"last_run_at": _heartbeat_state
|
| 437 |
-
"next_run_at": _heartbeat_state
|
| 438 |
-
"runs": _heartbeat_state
|
| 439 |
-
"error": _heartbeat_state
|
| 440 |
"interval_s": _HEARTBEAT_INTERVAL_S,
|
| 441 |
"server_time": now,
|
| 442 |
}
|
|
|
|
| 428 |
@router.get("/api/providers/heartbeat")
|
| 429 |
async def providers_heartbeat(role: AuthRole = Depends(require_role(AuthRole.MACHINE))): # GAP-1-fix
|
| 430 |
now = int(time.time())
|
| 431 |
+
# Il primo ciclo async potrebbe non essere ancora partito: la route di health
|
| 432 |
+
# deve restituire uno snapshot coerente, non propagare un KeyError come HTTP 500.
|
| 433 |
return {
|
| 434 |
+
"status": _heartbeat_state.get("status", "idle"),
|
| 435 |
+
"best_provider": _heartbeat_state.get("best_provider"),
|
| 436 |
+
"best_latency_ms": _heartbeat_state.get("best_latency_ms"),
|
| 437 |
+
"providers": _heartbeat_state.get("providers", []),
|
| 438 |
+
"last_run_at": _heartbeat_state.get("last_run_at"),
|
| 439 |
+
"next_run_at": _heartbeat_state.get("next_run_at"),
|
| 440 |
+
"runs": _heartbeat_state.get("runs", 0),
|
| 441 |
+
"error": _heartbeat_state.get("error"),
|
| 442 |
"interval_s": _HEARTBEAT_INTERVAL_S,
|
| 443 |
"server_time": now,
|
| 444 |
}
|
api/state.py
CHANGED
|
@@ -138,6 +138,10 @@ _AGENT_TASK_MAX = 200
|
|
| 138 |
_ai_health_cache: dict = {"data": None, "at": 0.0}
|
| 139 |
_AI_HEALTH_TTL = 60.0
|
| 140 |
_heartbeat_state: dict = {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
"last_run_at": None,
|
| 142 |
"next_run_at": None,
|
| 143 |
"best_provider": None,
|
|
|
|
| 138 |
_ai_health_cache: dict = {"data": None, "at": 0.0}
|
| 139 |
_AI_HEALTH_TTL = 60.0
|
| 140 |
_heartbeat_state: dict = {
|
| 141 |
+
# Stato completo disponibile già al boot: le route di osservabilità non
|
| 142 |
+
# devono dipendere dal primo ciclo async per avere le chiavi di risposta.
|
| 143 |
+
"status": "idle",
|
| 144 |
+
"error": None,
|
| 145 |
"last_run_at": None,
|
| 146 |
"next_run_at": None,
|
| 147 |
"best_provider": None,
|
main.py
CHANGED
|
@@ -215,6 +215,12 @@ async def startup_event():
|
|
| 215 |
except Exception as e:
|
| 216 |
_logger.warning(f"⚠️ BOOT: apply_rls_fix_sync() fallito (non bloccante): {e}")
|
| 217 |
asyncio.create_task(_run_auto_migration())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 218 |
if not any(arg in sys.argv for arg in ["--task", "-t"]):
|
| 219 |
try:
|
| 220 |
from api.job_queue import start_job_queue_consumer
|
|
|
|
| 215 |
except Exception as e:
|
| 216 |
_logger.warning(f"⚠️ BOOT: apply_rls_fix_sync() fallito (non bloccante): {e}")
|
| 217 |
asyncio.create_task(_run_auto_migration())
|
| 218 |
+
try:
|
| 219 |
+
from api.providers import start_heartbeat
|
| 220 |
+
start_heartbeat()
|
| 221 |
+
_logger.info("✅ BOOT: provider heartbeat avviato.")
|
| 222 |
+
except Exception as e:
|
| 223 |
+
_logger.warning(f"⚠️ BOOT: avvio provider heartbeat fallito (non bloccante): {e}")
|
| 224 |
if not any(arg in sys.argv for arg in ["--task", "-t"]):
|
| 225 |
try:
|
| 226 |
from api.job_queue import start_job_queue_consumer
|
tests/test_provider_heartbeat_state.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import unittest
|
| 3 |
+
|
| 4 |
+
from api import providers, state
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class ProviderHeartbeatStateTests(unittest.IsolatedAsyncioTestCase):
|
| 8 |
+
async def test_endpoint_returns_safe_snapshot_for_partial_legacy_state(self):
|
| 9 |
+
original = dict(state._heartbeat_state)
|
| 10 |
+
try:
|
| 11 |
+
# Simula uno stato parziale proveniente da un deploy precedente o
|
| 12 |
+
# dalla finestra di boot prima del primo ciclo heartbeat.
|
| 13 |
+
state._heartbeat_state.clear()
|
| 14 |
+
state._heartbeat_state.update({"providers": [], "runs": 0})
|
| 15 |
+
|
| 16 |
+
payload = await providers.providers_heartbeat(role=None)
|
| 17 |
+
|
| 18 |
+
self.assertEqual(payload["status"], "idle")
|
| 19 |
+
self.assertIsNone(payload["error"])
|
| 20 |
+
self.assertEqual(payload["providers"], [])
|
| 21 |
+
self.assertEqual(payload["runs"], 0)
|
| 22 |
+
self.assertIn("server_time", payload)
|
| 23 |
+
finally:
|
| 24 |
+
state._heartbeat_state.clear()
|
| 25 |
+
state._heartbeat_state.update(original)
|
| 26 |
+
|
| 27 |
+
async def test_start_heartbeat_creates_a_single_background_task(self):
|
| 28 |
+
original_task = providers._heartbeat_task
|
| 29 |
+
providers._heartbeat_task = None
|
| 30 |
+
try:
|
| 31 |
+
providers.start_heartbeat()
|
| 32 |
+
task = providers._heartbeat_task
|
| 33 |
+
self.assertIsNotNone(task)
|
| 34 |
+
self.assertFalse(task.done())
|
| 35 |
+
task.cancel()
|
| 36 |
+
with self.assertRaises(asyncio.CancelledError):
|
| 37 |
+
await task
|
| 38 |
+
finally:
|
| 39 |
+
providers._heartbeat_task = original_task
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
if __name__ == "__main__":
|
| 43 |
+
unittest.main()
|