Switch to Mistral 7B GGUF

app.py

@@ -1,14 +1,14 @@
 import gradio as gr
 from ctransformers import AutoModelForCausalLM
 
-MODEL_REPO = "Qwen/Qwen3-4B-GGUF"
-MODEL_FILE = "Qwen3-4B-Q4_K_M.gguf"
+MODEL_REPO = "TheBloke/Mistral-7B-Instruct-v0.2-GGUF"
+MODEL_FILE = "mistral-7b-instruct-v0.2.Q4_K_M.gguf"
 
 print("Loading model...")
 llm = AutoModelForCausalLM.from_pretrained(
     MODEL_REPO,
     model_file=MODEL_FILE,
-    model_type="qwen",
+    model_type="mistral",
     gpu_layers=0,
     context_length=2048,
 )
@@ -18,15 +18,18 @@ def respond(message: str, history: list[list[str]]) -> str:
     for user_msg, bot_msg in history:
         prompt += f"<|im_start|>user\n{user_msg}\n<|im_end|>\n"
         prompt += f"<|im_start|>assistant\n{bot_msg}\n<|im_end|>\n"
+
     prompt += f"<|im_start|>user\n{message}\n<|im_end|>\n<|im_start|>assistant\n"
+
     out = llm(
         prompt,
-        max_new_tokens=512,
+        max_new_tokens=256,
         temperature=0.7,
         top_p=0.9,
         stop=["<|im_end|>"],
     )
-    return out
+    return out["text"]
+
 
 if __name__ == "__main__":
-    gr.ChatInterface(respond).launch()
+    gr.ChatInterface(respond).launch()