Fix 7b

app.py

@@ -47,7 +47,8 @@ _ = (
     "golay" in platform.node()
     or "okteto" in platform.node()
     or Path("/kaggle").exists()
-    or psutil.cpu_count(logical=False) < 4
+    # or psutil.cpu_count(logical=False) < 4
+    or 1  # run 7b in hf
 )
 
 if _:
@@ -116,7 +117,7 @@ except Exception as exc_:
 LLM = AutoModelForCausalLM.from_pretrained(
     model_loc,
     model_type="llama",
-    threads=cpu_count,
+    # threads=cpu_count,
 )
 
 logger.info(f"done load llm {model_loc=} {file_size=}G")
@@ -145,7 +146,7 @@ class GenerationConfig:
     seed: int = 42
     reset: bool = False
     stream: bool = True
-    threads: int = cpu_count
+    # threads: int = cpu_count
     # stop: list[str] = field(default_factory=lambda: [stop_string])
 
 
@@ -237,7 +238,7 @@ def predict_api(prompt):
             seed=42,
             reset=True,  # reset history (cache)
             stream=False,
-            threads=cpu_count,
+            # threads=cpu_count,
             # stop=prompt_prefix[1:2],
         )
 
@@ -392,18 +393,18 @@ with gr.Blocks(
         fn=user,
         inputs=[msg, chatbot],
         outputs=[msg, chatbot],
-        queue=False,
+        queue=True,
         show_progress="full",
-        api_name=False,
+        api_name=None,
     ).then(bot, chatbot, chatbot, queue=False)
     submit.click(
         fn=lambda x, y: ("",) + user(x, y)[1:],  # clear msg
         inputs=[msg, chatbot],
         outputs=[msg, chatbot],
-        # queue=True,
-        queue=False,
+        queue=True,
+        # queue=False,
         show_progress="full",
-        api_name=False,
+        api_name=None,
     ).then(bot, chatbot, chatbot, queue=False)
 
     clear.click(lambda: None, None, chatbot, queue=False)
@@ -429,13 +430,16 @@ with gr.Blocks(
 # CPU UPGRADE cpu_count=8 32G, model 7G
 
 # does not work
+_ = """
 # _ = int(psutil.virtual_memory().total / 10**9 // file_size - 1)
 # concurrency_count = max(_, 1)
-
-if psutil.cpu_count(logical=False) > 8:
-    concurrency_count = max(int(32 / file_size) - 1, 1)
+if psutil.cpu_count(logical=False) >= 8:
+    # concurrency_count = max(int(32 / file_size) - 1, 1)
 else:
-    concurrency_count = max(int(16 / file_size) - 1, 1)
+    # concurrency_count = max(int(16 / file_size) - 1, 1)
+# """
+
+concurrency_count = 1
 logger.info(f"{concurrency_count=}")
 
-block.queue(concurrency_count=1, max_size=5).launch(debug=True)
+block.queue(concurrency_count=concurrency_count, max_size=5).launch(debug=True)