Cyber Catalyst Team commited on
Commit
0bc1706
·
1 Parent(s): 49c143f

Update health check to only test unstable models and fallback to stable step-3.7-flash

Browse files
Files changed (1) hide show
  1. backend.py +22 -9
backend.py CHANGED
@@ -467,17 +467,29 @@ def auth(authorization: str = None):
467
 
468
  async def check_models_health():
469
  global RECOMMENDED_MODEL
470
- # Limit test to top 3 models to prevent NVIDIA NIM quota exhaustion
 
471
  models_to_test = [
472
- "nvidia/nemotron-3-ultra-550b-a55b",
473
- "stepfun-ai/step-3.7-flash",
474
- "qwen/qwen2.5-coder-32b-instruct",
 
 
475
  ]
476
 
477
  best_model = None
478
  best_latency = 999.0
479
 
480
- log_activity("Periodic health check started: verifying top 3 NIM models...")
 
 
 
 
 
 
 
 
 
481
  for model in models_to_test:
482
  start_time = time.time()
483
  try:
@@ -492,8 +504,7 @@ async def check_models_health():
492
  MODEL_STATUSES[model] = {"status": "ONLINE", "latency": f"{latency:.2f}s", "raw_latency": latency}
493
  log_activity(f"Model checked: {model} is ONLINE ({latency:.2f}s)")
494
 
495
- # We want the model that is within 15 seconds
496
- # and is the fastest (lowest latency)
497
  if latency < best_latency:
498
  best_latency = latency
499
  best_model = model
@@ -504,9 +515,11 @@ async def check_models_health():
504
 
505
  if best_model:
506
  RECOMMENDED_MODEL = best_model
507
- log_activity(f"Best model selected: {RECOMMENDED_MODEL} ({best_latency:.2f}s)")
508
  else:
509
- log_activity("Warning: All checked models failed or timed out!")
 
 
510
 
511
  async def periodic_health_check_loop():
512
  # Wait 10 seconds after startup before the first check to let the space boot fully
 
467
 
468
  async def check_models_health():
469
  global RECOMMENDED_MODEL
470
+ # Test only the unstable frontier models (the rest).
471
+ # The stable ones (Step 3.7 Flash, Nemotron 3 Ultra, Qwen 2.5 Coder) are always free/working.
472
  models_to_test = [
473
+ "moonshotai/kimi-k2.6",
474
+ "z-ai/glm-5.1",
475
+ "minimaxai/minimax-m3",
476
+ "minimaxai/minimax-m2.7",
477
+ "meta/llama-3.1-405b-instruct",
478
  ]
479
 
480
  best_model = None
481
  best_latency = 999.0
482
 
483
+ # Mark stable models as permanently ONLINE in the status map
484
+ stable_models = [
485
+ "stepfun-ai/step-3.7-flash",
486
+ "nvidia/nemotron-3-ultra-550b-a55b",
487
+ "qwen/qwen2.5-coder-32b-instruct"
488
+ ]
489
+ for model in stable_models:
490
+ MODEL_STATUSES[model] = {"status": "ONLINE (Stable)", "latency": "Fast", "raw_latency": 0.1}
491
+
492
+ log_activity("Periodic health check started: verifying unstable frontier NIM models...")
493
  for model in models_to_test:
494
  start_time = time.time()
495
  try:
 
504
  MODEL_STATUSES[model] = {"status": "ONLINE", "latency": f"{latency:.2f}s", "raw_latency": latency}
505
  log_activity(f"Model checked: {model} is ONLINE ({latency:.2f}s)")
506
 
507
+ # Choose the fastest online unstable model
 
508
  if latency < best_latency:
509
  best_latency = latency
510
  best_model = model
 
515
 
516
  if best_model:
517
  RECOMMENDED_MODEL = best_model
518
+ log_activity(f"Best frontier model selected: {RECOMMENDED_MODEL} ({best_latency:.2f}s)")
519
  else:
520
+ # Fallback to the stable Step 3.7 Flash if all frontier models are offline/throttled
521
+ RECOMMENDED_MODEL = "stepfun-ai/step-3.7-flash"
522
+ log_activity(f"All frontier models offline. Falling back to stable recommended model: {RECOMMENDED_MODEL}")
523
 
524
  async def periodic_health_check_loop():
525
  # Wait 10 seconds after startup before the first check to let the space boot fully