Update main.py

main.py

@@ -1,21 +1,24 @@
 from ctransformers import AutoModelForCausalLM
-from gradio import Chatbot, Textbox, Button
+from gradio import Chatbot, Interface
 import gradio as gr
 
-# Load Qwen 2.5 GGUF model
+# Load the GGUF model
 llm = AutoModelForCausalLM.from_pretrained(
-    "Qwen2.5-7B-Q4_K_M.gguf",
-    model_type="qwen",
+    "zephyr-7b-beta.Q4_K_S.gguf",
+    model_type="mistral",
     max_new_tokens=1096,
     threads=3
 )
 
-# Define prompt format
+# Format prompt with system message and chat history
 def format_prompt(message, chat_history):
-    prompt = ""
+    system_prompt = "Below is an instruction that describes a task. Write a response that appropriately completes the request."
+    E_INST = "</s>"
+    user, assistant = "<|user|>", "<|assistant|>"
+    prompt = f"{system_prompt}{E_INST}\n"
     for user_msg, bot_msg in chat_history:
-        prompt += f"<|user|>\n{user_msg}\n<|assistant|>\n{bot_msg}\n"
-    prompt += f"<|user|>\n{message}\n<|assistant|>\n"
+        prompt += f"{user}\n{user_msg}{E_INST}\n{assistant}\n{bot_msg}{E_INST}\n"
+    prompt += f"{user}\n{message}{E_INST}\n{assistant}\n"
     return prompt
 
 # Define chatbot function
@@ -23,18 +26,26 @@ def respond(message, chat_history):
     formatted_prompt = format_prompt(message, chat_history)
     response = llm(formatted_prompt)
     chat_history.append((message, response))
-    return chat_history, chat_history  # Return both chatbot and state
+    return chat_history, chat_history
 
-# Gradio UI
+# Create Gradio Chatbot UI
+chatbot = Chatbot(
+    bubble_full_width=False,
+    height=500
+)
+
+# Launch interface
 with gr.Blocks() as demo:
-    gr.Markdown("## Qwen 2.5 Chat Interface")
-    chatbot = Chatbot()
-    msg = Textbox(label="Your Message")
-    clear = Button("Clear Chat")
+    gr.Markdown("## Zephyr LLM Chat Interface")
+    chatbot = gr.Chatbot()
+    msg = gr.Textbox(label="Your Message")
+    clear = gr.Button("Clear Chat")
+
+    state = gr.State([])
 
-    msg.submit(fn=respond, inputs=[msg, chatbot], outputs=[chatbot, chatbot])
-    clear.click(fn=lambda: ([], []), inputs=None, outputs=[chatbot, chatbot])
+    msg.submit(respond, [msg, state], [chatbot, state])
+    clear.click(lambda: ([], []), None, [chatbot, state])
 
-# Launch with share=True
+# Launch Gradio app
 if __name__ == "__main__":
     demo.launch(share=True)