Spaces:
Runtime error
Runtime error
Cyber Catalyst Team commited on
Commit ·
2e782b6
1
Parent(s): cbc1879
feat: disable dynamic completions health checks to preserve NIM key RPM
Browse files- backend.py +3 -63
backend.py
CHANGED
|
@@ -558,70 +558,10 @@ def auth(authorization: str = None):
|
|
| 558 |
|
| 559 |
|
| 560 |
async def check_models_health():
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
# The stable ones (Step 3.7 Flash, Nemotron 3 Ultra, Qwen 2.5 Coder) are always free/working.
|
| 564 |
-
models_to_test = [
|
| 565 |
-
"moonshotai/kimi-k2.6",
|
| 566 |
-
"z-ai/glm-5.1",
|
| 567 |
-
"minimaxai/minimax-m3",
|
| 568 |
-
"minimaxai/minimax-m2.7",
|
| 569 |
-
"meta/llama-3.1-405b-instruct",
|
| 570 |
-
]
|
| 571 |
-
|
| 572 |
-
best_model = None
|
| 573 |
-
best_latency = 999.0
|
| 574 |
-
|
| 575 |
-
# Mark stable models as permanently ONLINE in the status map
|
| 576 |
-
stable_models = [
|
| 577 |
-
"stepfun-ai/step-3.7-flash",
|
| 578 |
-
"nvidia/nemotron-3-ultra-550b-a55b",
|
| 579 |
-
"qwen/qwen2.5-coder-32b-instruct"
|
| 580 |
-
]
|
| 581 |
-
for model in stable_models:
|
| 582 |
MODEL_STATUSES[model] = {"status": "ONLINE (Stable)", "latency": "Fast", "raw_latency": 0.1}
|
| 583 |
-
|
| 584 |
-
log_activity("Periodic health check started: verifying unstable frontier NIM models...")
|
| 585 |
-
for model in models_to_test:
|
| 586 |
-
start_time = time.time()
|
| 587 |
-
try:
|
| 588 |
-
# Send a fast test prompt
|
| 589 |
-
async with anyio.fail_after(15.0): # 15 seconds max timeout
|
| 590 |
-
await nim_client.chat.completions.create(
|
| 591 |
-
model=model,
|
| 592 |
-
messages=[{"role": "user", "content": "1+1="}],
|
| 593 |
-
max_tokens=3,
|
| 594 |
-
)
|
| 595 |
-
latency = time.time() - start_time
|
| 596 |
-
MODEL_STATUSES[model] = {"status": "ONLINE", "latency": f"{latency:.2f}s", "raw_latency": latency}
|
| 597 |
-
log_activity(f"Model checked: {model} is ONLINE ({latency:.2f}s)")
|
| 598 |
-
|
| 599 |
-
# Choose the fastest online unstable model
|
| 600 |
-
if latency < best_latency:
|
| 601 |
-
best_latency = latency
|
| 602 |
-
best_model = model
|
| 603 |
-
|
| 604 |
-
except Exception as e:
|
| 605 |
-
MODEL_STATUSES[model] = {"status": "OFFLINE", "latency": "N/A", "raw_latency": 999.0}
|
| 606 |
-
log_activity(f"Model checked: {model} is OFFLINE / TIMEOUT: {e}")
|
| 607 |
-
|
| 608 |
-
if best_model:
|
| 609 |
-
RECOMMENDED_MODEL = best_model
|
| 610 |
-
log_activity(f"Best frontier model selected: {RECOMMENDED_MODEL} ({best_latency:.2f}s)")
|
| 611 |
-
else:
|
| 612 |
-
# Fallback to the stable Step 3.7 Flash if all frontier models are offline/throttled
|
| 613 |
-
RECOMMENDED_MODEL = "stepfun-ai/step-3.7-flash"
|
| 614 |
-
log_activity(f"All frontier models offline. Falling back to stable recommended model: {RECOMMENDED_MODEL}")
|
| 615 |
-
|
| 616 |
-
async def periodic_health_check_loop():
|
| 617 |
-
# Wait 10 seconds after startup before the first check to let the space boot fully
|
| 618 |
-
await asyncio.sleep(10)
|
| 619 |
-
while True:
|
| 620 |
-
try:
|
| 621 |
-
await check_models_health()
|
| 622 |
-
except Exception as e:
|
| 623 |
-
log_activity(f"Health check loop error: {e}")
|
| 624 |
-
await asyncio.sleep(900) # every 15 minutes (reduce frequency to save quota)
|
| 625 |
|
| 626 |
@app.on_event("startup")
|
| 627 |
async def startup():
|
|
|
|
| 558 |
|
| 559 |
|
| 560 |
async def check_models_health():
|
| 561 |
+
# Mark all models as statically ONLINE to save API RPM quotas
|
| 562 |
+
for model in ALL_MODELS.keys():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 563 |
MODEL_STATUSES[model] = {"status": "ONLINE (Stable)", "latency": "Fast", "raw_latency": 0.1}
|
| 564 |
+
log_activity("[Health Check] Zero-cost status check complete: all models marked ONLINE (No API calls made).")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 565 |
|
| 566 |
@app.on_event("startup")
|
| 567 |
async def startup():
|