Async requests

app.py

@@ -20,7 +20,7 @@ MODEL_LIB = {'TINYLLAMA': get_tinyllama, "QWEN2INS1B": get_qwen2ins1b, "RUBASE":
 GEN_LIB = {'TINYLLAMA': response_tinyllama, "QWEN2INS1B": response_qwen2ins1b, "RUBASE": response_gigachat}
 MODEL_LIST = ["TINYLLAMA", "QWEN2INS1B", "RUBASE"]
 
-def model_gen(
+async def model_gen(
         content,
         chat_history,
         model_name: str,
@@ -49,7 +49,7 @@ def model_gen(
         {"top_p": top_p, "temperature": temp, "max_tokens": max_tokens}
         )
 
-def model_regen(
+async def model_regen(
         content,
         chat_history,
         model_name: str,
@@ -227,7 +227,7 @@ def tab_offline_arena():
 def build_demo():
     # global original_dfs, available_models, gpt4t_dfs, haiku_dfs, llama_dfs
 
-    with gradio.Blocks(theme=gradio.themes.Base(), css=css, js=js_light) as demo:
+    with gradio.Blocks(theme=gradio.themes.Base(), concurrency_limit=4, css=css, js=js_light) as demo:
         # gradio.HTML(BANNER, elem_id="banner")
         # gradio.Markdown(HEADER_MD.replace("{model_num}", str(len(original_dfs["-1"]))), elem_classes="markdown-text")