ai: Implement async functions.

jarvis.py

@@ -20,6 +20,8 @@ import uuid
 import concurrent.futures
 import itertools
 import threading
+import httpx
+import asyncio
 
 from openai import OpenAI
 
@@ -141,19 +143,21 @@ def process_ai_response(ai_text):
     except Exception:
         return ai_text
 
-def fetch_response(host, provider_key, selected_model, messages, model_config, session_id):
+async def fetch_response_async(host, provider_key, selected_model, messages, model_config, session_id):
     try:
-        client = OpenAI(base_url=host, api_key=provider_key)
-        data = {"model": selected_model, "messages": messages, **model_config}
-        response = client.chat.completions.create(extra_body={"optillm_approach": "rto|re2|cot_reflection|self_consistency|plansearch|leap|z3|bon|moa|mcts|mcp|router|privacy|executecode|json", "session_id": session_id}, **data)
-        ai_text = response.choices[0].message.content if response.choices and response.choices[0].message and response.choices[0].message.content else RESPONSES["RESPONSE_2"]
-        return process_ai_response(ai_text)
+        async with httpx.AsyncClient(timeout=1) as client:
+            data = {"model": selected_model, "messages": messages, **model_config}
+            extra = {"optillm_approach": "rto|re2|cot_reflection|self_consistency|plansearch|leap|z3|bon|moa|mcts|mcp|router|privacy|executecode|json", "session_id": session_id}
+            response = await client.post(f"{host}", json={**data, "extra_body": extra, "session_id": session_id}, headers={"Authorization": f"Bearer {provider_key}"})
+            response.raise_for_status()
+            resp_json = response.json()
+            ai_text = resp_json["choices"][0]["message"]["content"] if resp_json.get("choices") and resp_json["choices"][0].get("message") and resp_json["choices"][0]["message"].get("content") else RESPONSES["RESPONSE_2"]
+            return process_ai_response(ai_text)
     except Exception:
         marked_item(provider_key, LINUX_SERVER_PROVIDER_KEYS_MARKED, LINUX_SERVER_PROVIDER_KEYS_ATTEMPTS)
-        #marked_item(host, LINUX_SERVER_HOSTS_MARKED, LINUX_SERVER_HOSTS_ATTEMPTS)
         raise
 
-def chat_with_model(history, user_input, selected_model_display, sess):
+async def chat_with_model_async(history, user_input, selected_model_display, sess):
     global ACTIVE_CANDIDATE
     if not get_available_items(LINUX_SERVER_PROVIDER_KEYS, LINUX_SERVER_PROVIDER_KEYS_MARKED) or not get_available_items(LINUX_SERVER_HOSTS, LINUX_SERVER_HOSTS_MARKED):
         return RESPONSES["RESPONSE_3"]
@@ -166,28 +170,24 @@ def chat_with_model(history, user_input, selected_model_display, sess):
     messages.append({"role": "user", "content": user_input})
     if ACTIVE_CANDIDATE is not None:
         try:
-            return fetch_response(ACTIVE_CANDIDATE[0], ACTIVE_CANDIDATE[1], selected_model, messages, model_config, sess.session_id)
+            return await fetch_response_async(ACTIVE_CANDIDATE[0], ACTIVE_CANDIDATE[1], selected_model, messages, model_config, sess.session_id)
         except Exception:
             ACTIVE_CANDIDATE = None
     available_keys = get_available_items(LINUX_SERVER_PROVIDER_KEYS, LINUX_SERVER_PROVIDER_KEYS_MARKED)
     available_servers = get_available_items(LINUX_SERVER_HOSTS, LINUX_SERVER_HOSTS_MARKED)
     candidates = [(host, key) for host in available_servers for key in available_keys]
     random.shuffle(candidates)
-    with concurrent.futures.ThreadPoolExecutor(max_workers=len(candidates)) as executor:
-        futures = {executor.submit(fetch_response, host, key, selected_model, messages, model_config, sess.session_id): (host, key) for (host, key) in candidates}
-        for future in concurrent.futures.as_completed(futures):
-            try:
-                result = future.result()
-                ACTIVE_CANDIDATE = futures[future]
-                for f in futures:
-                    if f is not future:
-                        f.cancel()
-                return result
-            except Exception:
-                continue
+    tasks = [fetch_response_async(host, key, selected_model, messages, model_config, sess.session_id) for host, key in candidates]
+    for task in asyncio.as_completed(tasks):
+        try:
+            result = await task
+            ACTIVE_CANDIDATE = next(((host, key) for host, key in candidates if host and key), None)
+            return result
+        except Exception:
+            continue
     return RESPONSES["RESPONSE_2"]
 
-def respond(multi_input, history, selected_model_display, sess):
+async def respond_async(multi_input, history, selected_model_display, sess):
     message = {"text": multi_input.get("text", "").strip(), "files": multi_input.get("files", [])}
     if not message["text"] and not message["files"]:
         yield history, gr.MultimodalTextbox(value=None, interactive=True), sess
@@ -200,7 +200,7 @@ def respond(multi_input, history, selected_model_display, sess):
     if message["text"]:
         combined_input += message["text"]
     history.append([combined_input, ""])
-    ai_response = chat_with_model(history, combined_input, selected_model_display, sess)
+    ai_response = await chat_with_model_async(history, combined_input, selected_model_display, sess)
     history[-1][1] = ""
     def convert_to_string(data):
         if isinstance(data, (str, int, float)):
@@ -215,7 +215,7 @@ def respond(multi_input, history, selected_model_display, sess):
             return repr(data)
     for character in ai_response:
         history[-1][1] += convert_to_string(character)
-        time.sleep(0.0001)
+        await asyncio.sleep(0.0001)
         yield history, gr.MultimodalTextbox(value=None, interactive=True), sess
 
 def change_model(new_model_display):
@@ -229,8 +229,6 @@ with gr.Blocks(fill_height=True, fill_width=True, title=AI_TYPES["AI_TYPE_4"], h
     model_dropdown = gr.Dropdown(show_label=False, choices=MODEL_CHOICES, value=MODEL_CHOICES[0])
     with gr.Row():
         msg = gr.MultimodalTextbox(show_label=False, placeholder=RESPONSES["RESPONSE_5"], interactive=True, file_count="single", file_types=ALLOWED_EXTENSIONS)
-
     model_dropdown.change(fn=change_model, inputs=[model_dropdown], outputs=[user_history, user_session, selected_model])
-    msg.submit(fn=respond, inputs=[msg, user_history, selected_model, user_session], outputs=[chatbot, msg, user_session])
-
+    msg.submit(fn=respond_async, inputs=[msg, user_history, selected_model, user_session], outputs=[chatbot, msg, user_session])
 jarvis.launch(show_api=False, max_file_size="1mb")

requirements.txt

@@ -1,5 +1,6 @@
 gradio
 huggingface_hub
+httpx
 openai
 optillm
 pandas