ai: Enhance API and Server randomization with fault tolerance.

jarvis.py

@@ -18,6 +18,7 @@ import fitz
 import io
 import uuid
 import concurrent.futures
+import itertools
 
 from openai import OpenAI
 
@@ -134,15 +135,20 @@ def chat_with_model(history, user_input, selected_model_display, sess):
     messages = [{"role": "user", "content": user} for user, _ in history]
     messages += [{"role": "assistant", "content": assistant} for _, assistant in history if assistant]
     messages.append({"role": "user", "content": user_input})
-    futures = []
-    with concurrent.futures.ThreadPoolExecutor(max_workers=len(LINUX_SERVER_HOSTS)) as executor:
-        for host, key in zip(LINUX_SERVER_HOSTS, LINUX_SERVER_PROVIDER_KEYS):
-            futures.append(executor.submit(fetch_response, host, key, selected_model, messages, model_config, sess.session_id))
-        done, not_done = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED)
-        for future in not_done:
-            future.cancel()
-        result = list(done)[0].result() if done else RESPONSES["RESPONSE_2"]
-    return result
+    candidates = [(host, key) for host in LINUX_SERVER_HOSTS for key in LINUX_SERVER_PROVIDER_KEYS]
+    random.shuffle(candidates)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=len(candidates)) as executor:
+        futures = {executor.submit(fetch_response, host, key, selected_model, messages, model_config, sess.session_id): (host, key) for (host, key) in candidates}
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                result = future.result()
+                for f in futures:
+                    if f is not future:
+                        f.cancel()
+                return result
+            except Exception:
+                continue
+    return RESPONSES["RESPONSE_2"]
 
 def respond(multi_input, history, selected_model_display, sess):
     message = {"text": multi_input.get("text", "").strip(), "files": multi_input.get("files", [])}