Cyber Catalyst Team commited on
Commit
49c143f
·
1 Parent(s): 9621cee

Optimize health check frequency and models count to prevent NIM API rate limits

Browse files
Files changed (1) hide show
  1. backend.py +3 -10
backend.py CHANGED
@@ -467,24 +467,17 @@ def auth(authorization: str = None):
467
 
468
  async def check_models_health():
469
  global RECOMMENDED_MODEL
470
- # Top models to test
471
  models_to_test = [
472
  "nvidia/nemotron-3-ultra-550b-a55b",
473
- "z-ai/glm-5.1",
474
- "moonshotai/kimi-k2.6",
475
- "minimaxai/minimax-m3",
476
  "stepfun-ai/step-3.7-flash",
477
- "minimaxai/minimax-m2.7",
478
- "nvidia/llama-3.1-nemotron-70b-instruct",
479
- "meta/llama-3.1-405b-instruct",
480
  "qwen/qwen2.5-coder-32b-instruct",
481
- "meta/llama-3.3-70b-instruct",
482
  ]
483
 
484
  best_model = None
485
  best_latency = 999.0
486
 
487
- log_activity("Periodic health check started: verifying 10 NIM models...")
488
  for model in models_to_test:
489
  start_time = time.time()
490
  try:
@@ -523,7 +516,7 @@ async def periodic_health_check_loop():
523
  await check_models_health()
524
  except Exception as e:
525
  log_activity(f"Health check loop error: {e}")
526
- await asyncio.sleep(300) # every 5 minutes
527
 
528
  @app.on_event("startup")
529
  async def startup():
 
467
 
468
  async def check_models_health():
469
  global RECOMMENDED_MODEL
470
+ # Limit test to top 3 models to prevent NVIDIA NIM quota exhaustion
471
  models_to_test = [
472
  "nvidia/nemotron-3-ultra-550b-a55b",
 
 
 
473
  "stepfun-ai/step-3.7-flash",
 
 
 
474
  "qwen/qwen2.5-coder-32b-instruct",
 
475
  ]
476
 
477
  best_model = None
478
  best_latency = 999.0
479
 
480
+ log_activity("Periodic health check started: verifying top 3 NIM models...")
481
  for model in models_to_test:
482
  start_time = time.time()
483
  try:
 
516
  await check_models_health()
517
  except Exception as e:
518
  log_activity(f"Health check loop error: {e}")
519
+ await asyncio.sleep(900) # every 15 minutes (reduce frequency to save quota)
520
 
521
  @app.on_event("startup")
522
  async def startup():