Cyber Catalyst Team commited on
Commit
2e782b6
·
1 Parent(s): cbc1879

feat: disable dynamic completions health checks to preserve NIM key RPM

Browse files
Files changed (1) hide show
  1. backend.py +3 -63
backend.py CHANGED
@@ -558,70 +558,10 @@ def auth(authorization: str = None):
558
 
559
 
560
  async def check_models_health():
561
- global RECOMMENDED_MODEL
562
- # Test only the unstable frontier models (the rest).
563
- # The stable ones (Step 3.7 Flash, Nemotron 3 Ultra, Qwen 2.5 Coder) are always free/working.
564
- models_to_test = [
565
- "moonshotai/kimi-k2.6",
566
- "z-ai/glm-5.1",
567
- "minimaxai/minimax-m3",
568
- "minimaxai/minimax-m2.7",
569
- "meta/llama-3.1-405b-instruct",
570
- ]
571
-
572
- best_model = None
573
- best_latency = 999.0
574
-
575
- # Mark stable models as permanently ONLINE in the status map
576
- stable_models = [
577
- "stepfun-ai/step-3.7-flash",
578
- "nvidia/nemotron-3-ultra-550b-a55b",
579
- "qwen/qwen2.5-coder-32b-instruct"
580
- ]
581
- for model in stable_models:
582
  MODEL_STATUSES[model] = {"status": "ONLINE (Stable)", "latency": "Fast", "raw_latency": 0.1}
583
-
584
- log_activity("Periodic health check started: verifying unstable frontier NIM models...")
585
- for model in models_to_test:
586
- start_time = time.time()
587
- try:
588
- # Send a fast test prompt
589
- async with anyio.fail_after(15.0): # 15 seconds max timeout
590
- await nim_client.chat.completions.create(
591
- model=model,
592
- messages=[{"role": "user", "content": "1+1="}],
593
- max_tokens=3,
594
- )
595
- latency = time.time() - start_time
596
- MODEL_STATUSES[model] = {"status": "ONLINE", "latency": f"{latency:.2f}s", "raw_latency": latency}
597
- log_activity(f"Model checked: {model} is ONLINE ({latency:.2f}s)")
598
-
599
- # Choose the fastest online unstable model
600
- if latency < best_latency:
601
- best_latency = latency
602
- best_model = model
603
-
604
- except Exception as e:
605
- MODEL_STATUSES[model] = {"status": "OFFLINE", "latency": "N/A", "raw_latency": 999.0}
606
- log_activity(f"Model checked: {model} is OFFLINE / TIMEOUT: {e}")
607
-
608
- if best_model:
609
- RECOMMENDED_MODEL = best_model
610
- log_activity(f"Best frontier model selected: {RECOMMENDED_MODEL} ({best_latency:.2f}s)")
611
- else:
612
- # Fallback to the stable Step 3.7 Flash if all frontier models are offline/throttled
613
- RECOMMENDED_MODEL = "stepfun-ai/step-3.7-flash"
614
- log_activity(f"All frontier models offline. Falling back to stable recommended model: {RECOMMENDED_MODEL}")
615
-
616
- async def periodic_health_check_loop():
617
- # Wait 10 seconds after startup before the first check to let the space boot fully
618
- await asyncio.sleep(10)
619
- while True:
620
- try:
621
- await check_models_health()
622
- except Exception as e:
623
- log_activity(f"Health check loop error: {e}")
624
- await asyncio.sleep(900) # every 15 minutes (reduce frequency to save quota)
625
 
626
  @app.on_event("startup")
627
  async def startup():
 
558
 
559
 
560
  async def check_models_health():
561
+ # Mark all models as statically ONLINE to save API RPM quotas
562
+ for model in ALL_MODELS.keys():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
563
  MODEL_STATUSES[model] = {"status": "ONLINE (Stable)", "latency": "Fast", "raw_latency": 0.1}
564
+ log_activity("[Health Check] Zero-cost status check complete: all models marked ONLINE (No API calls made).")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
565
 
566
  @app.on_event("startup")
567
  async def startup():