bep40 commited on
Commit
c93bc67
·
verified ·
1 Parent(s): 43634cd

Add /api/ai/probe diagnostic to find fast available chat model

Browse files
Files changed (1) hide show
  1. ai_ext.py +24 -0
ai_ext.py CHANGED
@@ -528,6 +528,30 @@ def pollinations_image_url(topic: str) -> str:
528
  return "https://image.pollinations.ai/prompt/" + quote(prompt, safe="") + "?width=1024&height=576&nologo=true"
529
 
530
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
531
  # ===== QWEN AI (strict, concise) =====
532
  async def qwen_generate(prompt: str, image_url: Optional[str] = None, max_tokens: int = 500, image_urls: Optional[List[str]] = None):
533
  global LAST_QWEN_ERROR, HF_TOKEN
 
528
  return "https://image.pollinations.ai/prompt/" + quote(prompt, safe="") + "?width=1024&height=576&nologo=true"
529
 
530
 
531
+ @app.get("/api/ai/probe")
532
+ async def api_ai_probe():
533
+ """Diagnostic: test which chat models actually work on this token + their latency."""
534
+ import time as _t
535
+ tok = _hf_token()
536
+ out = []
537
+ cand = QWEN_TEXT_MODELS + [QWEN_VL_MODEL, "Qwen/Qwen2.5-VL-7B-Instruct",
538
+ "meta-llama/Llama-3.1-8B-Instruct", "google/gemma-2-9b-it",
539
+ "mistralai/Mistral-7B-Instruct-v0.3", "Qwen/Qwen2.5-32B-Instruct"]
540
+ seen = set()
541
+ for m in cand:
542
+ if not m or m in seen:
543
+ continue
544
+ seen.add(m)
545
+ t0 = _t.time()
546
+ try:
547
+ c = AsyncInferenceClient(provider="auto", api_key=tok, timeout=40)
548
+ r = await c.chat_completion(model=m, messages=[{"role": "user", "content": "Trả lời đúng 1 từ: xin chào"}], max_tokens=10)
549
+ out.append({"model": m, "ok": True, "sec": round(_t.time() - t0, 1), "txt": (r.choices[0].message.content or "")[:30]})
550
+ except Exception as e:
551
+ out.append({"model": m, "ok": False, "sec": round(_t.time() - t0, 1), "err": (type(e).__name__ + ": " + str(e))[:140]})
552
+ return JSONResponse({"results": out})
553
+
554
+
555
  # ===== QWEN AI (strict, concise) =====
556
  async def qwen_generate(prompt: str, image_url: Optional[str] = None, max_tokens: int = 500, image_urls: Optional[List[str]] = None):
557
  global LAST_QWEN_ERROR, HF_TOKEN