Update app.py

app.py

@@ -1,38 +1,23 @@
 import os
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import torch
+from llama_cpp import Llama
 
-# Redirect cache to /tmp to avoid 50GB storage limit
-os.environ['HF_HOME'] = '/tmp/hf_home'
-os.environ['TRANSFORMERS_CACHE'] = '/tmp/hf_cache'
-
-model_name = "Qwen/Qwen2.5-Coder-14B-Instruct"
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-
-# Custom device map: Place most layers on GPU, offload rest to CPU
-device_map = {
-    "transformer": "cuda",  # Main layers on GPU
-    "lm_head": "cpu"       # Output layer to CPU
-}
-
-model = AutoModelForCausalLM.from_pretrained(
-    model_name,
-    load_in_4bit=True,
-    device_map=device_map,
-    llm_int8_enable_fp32_cpu_offload=True,  # Enable CPU offloading
-    torch_dtype=torch.float16,              # Reduce memory with FP16
-    trust_remote_code=True
+# No cache redirect needed for GGUF
+model_path = "Qwen/Qwen2.5-Coder-14B-Instruct-GGUF"  # Auto-downloads Q4_K_M (~9GB)
+llm = Llama(
+    model_path,
+    n_ctx=4096,        # Adjust for coding tasks
+    n_gpu_layers=99,   # Offload to T4 GPU
+    verbose=False
 )
 
 def chat(message, history):
-    messages = [{"role": "user", "content": message}]
-    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-    inputs = tokenizer(text, return_tensors="pt").to("cuda" if torch.cuda.is_available() else "cpu")
-    with torch.no_grad():
-        outputs = model.generate(**inputs, max_new_tokens=512, temperature=0.7)
-    response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
-    history.append((message, response))
+    response = llm.create_chat_completion(
+        messages=[{"role": "user", "content": message}],
+        max_tokens=512,
+        temperature=0.7
+    )
+    history.append((message, response["choices"][0]["message"]["content"]))
     return history, ""
 
 demo = gr.ChatInterface(chat)