ai: Implementing parallel request with first-success strategy.

jarvis.py

@@ -55,8 +55,6 @@ META_TAGS = os.getenv("META_TAGS")
 
 ALLOWED_EXTENSIONS = json.loads(os.getenv("ALLOWED_EXTENSIONS", "[]"))
 
-ACTIVE_CANDIDATE = None
-
 class SessionWithID(requests.Session):
     def __init__(sess):
         super().__init__()
@@ -222,13 +220,19 @@ async def fetch_response_async(host, key, model, msgs, cfg, sid):
     marked_item(key, LINUX_SERVER_PROVIDER_KEYS_MARKED, LINUX_SERVER_PROVIDER_KEYS_ATTEMPTS)
     return None
 
+async def candidate_task(h, k, model, msgs, cfg, sid):
+    r = await fetch_response_async(h, k, model, msgs, cfg, sid)
+    return r, (h, k)
+
 async def chat_with_model_async(history, user_input, model_display, sess, custom_prompt):
     ensure_stop_event(sess)
     if not get_available_items(LINUX_SERVER_PROVIDER_KEYS, LINUX_SERVER_PROVIDER_KEYS_MARKED) or not get_available_items(LINUX_SERVER_HOSTS, LINUX_SERVER_HOSTS_ATTEMPTS):
         return RESPONSES["RESPONSE_3"]
-    if not hasattr(sess, "session_id"):
+    if not hasattr(sess, "session_id") or not sess.session_id:
         sess.session_id = str(uuid.uuid4())
         sess.stop_event = asyncio.Event()
+    if not hasattr(sess, "active_candidate"):
+        sess.active_candidate = None
     model_key = get_model_key(model_display)
     cfg = MODEL_CONFIG.get(model_key, DEFAULT_CONFIG)
     msgs = [{"role": "user", "content": u} for u, _ in history] + [{"role": "assistant", "content": a} for _, a in history if a]
@@ -238,21 +242,32 @@ async def chat_with_model_async(history, user_input, model_display, sess, custom
         prompt = custom_prompt or SYSTEM_PROMPT_MAPPING.get(model_key, SYSTEM_PROMPT_DEFAULT)
     msgs.insert(0, {"role": "system", "content": prompt})
     msgs.append({"role": "user", "content": user_input})
-    global ACTIVE_CANDIDATE
-    if ACTIVE_CANDIDATE:
-        res = await fetch_response_async(ACTIVE_CANDIDATE[0], ACTIVE_CANDIDATE[1], model_key, msgs, cfg, sess.session_id)
+    if sess.active_candidate:
+        res = await fetch_response_async(sess.active_candidate[0], sess.active_candidate[1], model_key, msgs, cfg, sess.session_id)
         if res:
             return res
-        ACTIVE_CANDIDATE = None
+        sess.active_candidate = None
     keys = get_available_items(LINUX_SERVER_PROVIDER_KEYS, LINUX_SERVER_PROVIDER_KEYS_MARKED)
     hosts = get_available_items(LINUX_SERVER_HOSTS, LINUX_SERVER_HOSTS_ATTEMPTS)
     cands = [(h, k) for h in hosts for k in keys]
     random.shuffle(cands)
-    for h, k in cands:
-        res = await fetch_response_async(h, k, model_key, msgs, cfg, sess.session_id)
-        if res:
-            ACTIVE_CANDIDATE = (h, k)
-            return res
+    tasks = [asyncio.create_task(candidate_task(h, k, model_key, msgs, cfg, sess.session_id)) for h, k in cands]
+    first_success = None
+    candidate = None
+    for task in asyncio.as_completed(tasks):
+        try:
+            res, cand = await task
+            if res is not None:
+                first_success = res
+                candidate = cand
+                break
+        except:
+            continue
+    for t in tasks:
+        t.cancel()
+    if first_success:
+        sess.active_candidate = candidate
+        return first_success
     return RESPONSES["RESPONSE_2"]
 
 async def respond_async(multi, history, model_display, sess, custom_prompt):
@@ -287,7 +302,7 @@ async def respond_async(multi, history, model_display, sess, custom_prompt):
             buffer.clear()
             last_update = current_time
             yield history, gr.update(interactive=False, submit_btn=False, stop_btn=True), sess
-            await asyncio.sleep(0.016)
+            await asyncio.sleep(0.020)
     if buffer:
         history[-1][1] += "".join(buffer)
         yield history, gr.update(interactive=False, submit_btn=False, stop_btn=True), sess