Update single model

app.py

@@ -118,11 +118,12 @@ def bot(chat_history, **kwargs):
         )
         """
         logger.debug("run model.chat...")
+        model.generation_config(**kwargs)
         response, chat_history = model.chat(
             tokenizer,
             message,
             chat_history[:-1],
-            **kwargs,
+            # **kwargs,
         )
         del response
         return chat_history
@@ -131,6 +132,21 @@ def bot(chat_history, **kwargs):
         chat_history[:-1].append(["message", str(exc)])
         return chat_history
 
+def bot_stream(chat_history, **kwargs):
+    try:
+        message = chat_history[-1][0]
+    except Exception as exc:
+        logger.error(f"{chat_history=}: {exc}")
+        raise gr.Error(f"{chat_history=}")
+        # yield chat_history
+
+    # for elm in model.chat_stream(tokenizer, message, chat_history):
+    model.generation_config(**kwargs)
+    for elm in model.chat_stream(tokenizer, message, chat_history):
+        chat_history[-1] = [message, elm]
+        yield chat_history
+
+
 
 SYSTEM_PROMPT = "You are a helpful assistant."
 MAX_MAX_NEW_TOKENS = 1024
@@ -146,7 +162,8 @@ class Config:
     top_p: float = 0.9
 
 
-stats_default = SimpleNamespace(llm=model, system_prompt=SYSTEM_PROMPT, config=Config())
+# stats_default = SimpleNamespace(llm=model, system_prompt=SYSTEM_PROMPT, config=Config())
+stats_default = SimpleNamespace(llm=None, system_prompt=SYSTEM_PROMPT, config=Config())
 
 theme = gr.themes.Soft(text_size="sm")
 with gr.Blocks(
@@ -158,20 +175,9 @@ with gr.Blocks(
     if not torch.cuda.is_available():
         raise gr.Error("GPU not available, cant run. Turn on GPU and restart")
 
-    # model_ = stats.value.llm  # OOM
-    config = stats.value.config
-    stats.value.llm.generation_config.update(**asdict(config))
-    def bot_stream(chat_history):
-        try:
-            message = chat_history[-1][0]
-        except Exception as exc:
-            logger.error(f"{chat_history=}: {exc}")
-            raise gr.Error(f"{chat_history=}")
-            # yield chat_history
-        # for elm in model.chat_stream(tokenizer, message, chat_history):
-        for elm in stats.value.llm.chat_stream(tokenizer, message, chat_history):
-            chat_history[-1] = [message, elm]
-            yield chat_history
+    config = asdict(stats.value.config)
+    def bot_stream_state(chat_history):
+        return bot_stream(chat_history, **config)
 
     with gr.Accordion("🎈 Info", open=False):
         gr.Markdown(
@@ -213,7 +219,7 @@ with gr.Blocks(
         queue=True,
         show_progress="full",
         # api_name=None,
-    ).then(bot_stream, chatbot, chatbot, queue=True)
+    ).then(bot_stream_state, chatbot, chatbot, queue=True)
     submit_click_event = submit.click(
         # fn=lambda x, y: ("",) + user(x, y)[1:],  # clear msg
         fn=user,  # clear msg
@@ -222,7 +228,7 @@ with gr.Blocks(
         queue=True,
         show_progress="full",
         # api_name=None,
-    ).then(bot_stream, chatbot, chatbot, queue=True)
+    ).then(bot_stream_state, chatbot, chatbot, queue=True)
     stop.click(
         fn=None,
         inputs=None,