Spaces:
Sleeping
Sleeping
Cyber Catalyst Team commited on
Commit ·
49c143f
1
Parent(s): 9621cee
Optimize health check frequency and models count to prevent NIM API rate limits
Browse files- backend.py +3 -10
backend.py
CHANGED
|
@@ -467,24 +467,17 @@ def auth(authorization: str = None):
|
|
| 467 |
|
| 468 |
async def check_models_health():
|
| 469 |
global RECOMMENDED_MODEL
|
| 470 |
-
#
|
| 471 |
models_to_test = [
|
| 472 |
"nvidia/nemotron-3-ultra-550b-a55b",
|
| 473 |
-
"z-ai/glm-5.1",
|
| 474 |
-
"moonshotai/kimi-k2.6",
|
| 475 |
-
"minimaxai/minimax-m3",
|
| 476 |
"stepfun-ai/step-3.7-flash",
|
| 477 |
-
"minimaxai/minimax-m2.7",
|
| 478 |
-
"nvidia/llama-3.1-nemotron-70b-instruct",
|
| 479 |
-
"meta/llama-3.1-405b-instruct",
|
| 480 |
"qwen/qwen2.5-coder-32b-instruct",
|
| 481 |
-
"meta/llama-3.3-70b-instruct",
|
| 482 |
]
|
| 483 |
|
| 484 |
best_model = None
|
| 485 |
best_latency = 999.0
|
| 486 |
|
| 487 |
-
log_activity("Periodic health check started: verifying
|
| 488 |
for model in models_to_test:
|
| 489 |
start_time = time.time()
|
| 490 |
try:
|
|
@@ -523,7 +516,7 @@ async def periodic_health_check_loop():
|
|
| 523 |
await check_models_health()
|
| 524 |
except Exception as e:
|
| 525 |
log_activity(f"Health check loop error: {e}")
|
| 526 |
-
await asyncio.sleep(
|
| 527 |
|
| 528 |
@app.on_event("startup")
|
| 529 |
async def startup():
|
|
|
|
| 467 |
|
| 468 |
async def check_models_health():
|
| 469 |
global RECOMMENDED_MODEL
|
| 470 |
+
# Limit test to top 3 models to prevent NVIDIA NIM quota exhaustion
|
| 471 |
models_to_test = [
|
| 472 |
"nvidia/nemotron-3-ultra-550b-a55b",
|
|
|
|
|
|
|
|
|
|
| 473 |
"stepfun-ai/step-3.7-flash",
|
|
|
|
|
|
|
|
|
|
| 474 |
"qwen/qwen2.5-coder-32b-instruct",
|
|
|
|
| 475 |
]
|
| 476 |
|
| 477 |
best_model = None
|
| 478 |
best_latency = 999.0
|
| 479 |
|
| 480 |
+
log_activity("Periodic health check started: verifying top 3 NIM models...")
|
| 481 |
for model in models_to_test:
|
| 482 |
start_time = time.time()
|
| 483 |
try:
|
|
|
|
| 516 |
await check_models_health()
|
| 517 |
except Exception as e:
|
| 518 |
log_activity(f"Health check loop error: {e}")
|
| 519 |
+
await asyncio.sleep(900) # every 15 minutes (reduce frequency to save quota)
|
| 520 |
|
| 521 |
@app.on_event("startup")
|
| 522 |
async def startup():
|