Use ctransformers for Qwen

app.py

@@ -1,34 +1,32 @@
 import gradio as gr
-from llama_cpp import Llama
+from ctransformers import AutoModelForCausalLM
 
 MODEL_REPO = "Qwen/Qwen3-4B-GGUF"
 MODEL_FILE = "Qwen3-4B-Q4_K_M.gguf"
 
 print("Loading model...")
-llm = Llama.from_pretrained(
-    repo_id=MODEL_REPO,
-    filename=MODEL_FILE,
-    n_ctx=4096,
-    n_threads=2,
+llm = AutoModelForCausalLM.from_pretrained(
+    MODEL_REPO,
+    model_file=MODEL_FILE,
+    model_type="llama",
+    gpu_layers=0,
+    context_length=4096,
 )
 
-
 def respond(message: str, history: list[list[str]]) -> str:
-    messages = []
+    prompt = ""
     for user_msg, bot_msg in history:
-        messages.append({"role": "user", "content": user_msg})
-        messages.append({"role": "assistant", "content": bot_msg})
-
-    messages.append({"role": "user", "content": message})
-
-    out = llm.create_chat_completion(
-        messages=messages,
-        max_tokens=512,
+        prompt += f"<|im_start|>user\n{user_msg}\n<|im_end|>\n"
+        prompt += f"<|im_start|>assistant\n{bot_msg}\n<|im_end|>\n"
+    prompt += f"<|im_start|>user\n{message}\n<|im_end|>\n<|im_start|>assistant\n"
+    out = llm(
+        prompt,
+        max_new_tokens=512,
         temperature=0.7,
         top_p=0.9,
+        stop=["<|im_end|>"],
     )
+    return out
 
-    return out["choices"][0]["message"]["content"]
-
-
-gr.ChatInterface(respond).launch()
+if __name__ == "__main__":
+    gr.ChatInterface(respond).launch()