Update app.py

app.py

@@ -25,7 +25,7 @@ css = """
 .dark.user {
     background: #0a1120 !important;
 }
-.dark.assistant, .dark.pending {
+.dark.assistant {
     background: transparent !important;
 }
 """
@@ -37,6 +37,8 @@ def respond(
     max_tokens,
     temperature,
     top_p,
+    top_k,
+    repeat_penalty,
     model,
 ):
     from llama_cpp import Llama
@@ -47,14 +49,13 @@ def respond(
     from llama_cpp_agent.chat_history.messages import Roles
     print(message)
     print(history)
-    print(max_tokens)
-    print(temperature)
-    print(top_p)
-    print(model)
     
     llm = Llama(
         model_path=f"models/{model}",
+        flash_attn=True,
+        n_threads=40,
         n_gpu_layers=81,
+        n_batch=1024,
         n_ctx=8192,
     )
     provider = LlamaCppPythonProvider(llm)
@@ -67,7 +68,11 @@ def respond(
     )
     
     settings = provider.get_provider_default_settings()
+    settings.temperature = temperature
+    settings.top_k = top_k
+    settings.top_p = top_p
     settings.max_tokens = max_tokens
+    settings.repeat_penalty = repeat_penalty
     settings.stream = True
 
     messages = BasicChatHistory()
@@ -81,7 +86,6 @@ def respond(
             'role': Roles.assistant,
             'content': msn[1]
         }
-
         messages.add_message(user)
         messages.add_message(assistant)
     
@@ -95,14 +99,28 @@ def respond(
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[
-        gr.Slider(minimum=1, maximum=8192, value=8192, step=1, label="Max new tokens"),
+        gr.Slider(minimum=1, maximum=8192, value=8192, step=1, label="Max tokens"),
         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
         gr.Slider(
             minimum=0.1,
             maximum=1.0,
             value=0.95,
             step=0.05,
-            label="Top-p (nucleus sampling)",
+            label="Top-p",
+        ),
+        gr.Slider(
+            minimum=0,
+            maximum=100,
+            value=40,
+            step=1,
+            label="Top-k",
+        ),
+        gr.Slider(
+            minimum=0.0,
+            maximum=2.0,
+            value=1.1,
+            step=0.1,
+            label="Repetition penalty",
         ),
         gr.Dropdown(['dolphin-2.9.1-yi-1.5-34b-Q6_K.gguf', 'dolphin-2.9.1-llama-3-70b.Q3_K_M.gguf'], value="dolphin-2.9.1-llama-3-70b.Q3_K_M.gguf", label="Model"),
     ],
@@ -121,7 +139,7 @@ demo = gr.ChatInterface(
     undo_btn="Undo",
     clear_btn="Clear",
     submit_btn="Send",
-    description="Cognitive Computation: 🐬 Chat multi llm"
+    description="Cognitive Computation: Chat Doplhin 🐬 2.9.1-llama-3-70b & 2.9.1-yi-1.5-34b"
 )
 
 if __name__ == "__main__":