Update app.py

app.py

@@ -1,12 +1,15 @@
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import gradio as gr
 import os
+import gradio as gr
+from llama_cpp import Llama
 
-os.environ["HF_HOME"] = "/app/hf_cache"
-model_name = "Qwen/Qwen3.5-35B-A3B-FP8"
+os.environ["HF_HOME"] = "/tmp/hf_cache"
 
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", offload_folder="/tmp/offload")
+model = Llama.from_pretrained(
+    repo_id="bartowski/Qwen3.5-35B-A3B-GGUF",
+    filename="Qwen3.5-35B-A3B-Q3_K_M.gguf",
+    n_ctx=2048,
+    n_threads=2,
+)
 
 def chat(message, history):
     messages = []
@@ -15,10 +18,7 @@ def chat(message, history):
         messages.append({"role": "assistant", "content": assistant})
     messages.append({"role": "user", "content": message})
 
-    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-    inputs = tokenizer([text], return_tensors="pt").to(model.device)
-    outputs = model.generate(**inputs, max_new_tokens=512)
-    output = tokenizer.decode(outputs[0][inputs.input_ids.shape[-1]:], skip_special_tokens=True)
-    return output
+    response = model.create_chat_completion(messages=messages, max_tokens=512)
+    return response["choices"][0]["message"]["content"]
 
 gr.ChatInterface(chat).launch(server_name="0.0.0.0", server_port=7860)