Update app.py

app.py

@@ -1,68 +1,62 @@
 import time
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
 import gradio as gr
+from llama_cpp import Llama
 
 # --------------------------
-# Load 13B GPTQ model
+# Load CPU-friendly 4B model
 # --------------------------
-model_name = "TheBloke/Wizard-Vicuna-13B-Uncensored-GPTQ"
-
-tokenizer = AutoTokenizer.from_pretrained(model_name, legacy=False)
-model = AutoModelForCausalLM.from_pretrained(
-    model_name,
-    device_map="auto" if torch.cuda.is_available() else None,  # GPU if available
-    dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
+llm = Llama.from_pretrained(
+    repo_id="DavidAU/Gemma-3-it-4B-Uncensored-DBL-X-GGUF",
+    filename="Gemma-3-it-4B-Uncensored-D_AU-F16.gguf",
 )
 
 # --------------------------
-# Chat history & response
+# Chat history
 # --------------------------
 chat_history = []
 
-def generate_response(prompt, history):
+# --------------------------
+# Generate response
+# --------------------------
+def generate_response(prompt):
+    global chat_history
     # Combine previous conversation
     context = ""
-    for user_msg, ai_msg in history:
+    for user_msg, ai_msg in chat_history:
         context += f"User: {user_msg}\nAI: {ai_msg}\n"
     context += f"User: {prompt}\nAI:"
 
-    inputs = tokenizer(context, return_tensors="pt")
-    if torch.cuda.is_available():
-        inputs = {k: v.cuda() for k, v in inputs.items()}
-
-    # Generate output
-    outputs = model.generate(**inputs, max_new_tokens=150, do_sample=True, temperature=0.7)
-    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
-    response = response.split("AI:")[-1].strip()
+    # Generate text
+    output = llm(prompt=context, max_tokens=200)
+    response = output['choices'][0]['text'].strip()
+    
+    # Update history
+    chat_history.append((prompt, response))
     return response
 
 # --------------------------
-# Simulate live typing
+# Live typing simulation
 # --------------------------
 def live_typing(prompt):
-    global chat_history
-    response = generate_response(prompt, chat_history)
+    response = generate_response(prompt)
     displayed_text = ""
     for char in response:
         displayed_text += char
-        time.sleep(0.02)  # Typing speed
+        time.sleep(0.02)  # typing speed
         yield displayed_text
-    # Update chat history
-    chat_history.append((prompt, response))
 
 # --------------------------
 # Gradio UI
 # --------------------------
 with gr.Blocks() as demo:
-    gr.Markdown("## 🤖 Wizard-Vicuna 13B Chatbot with Avatars and Live Typing")
+    gr.Markdown("## 🤖 Gemma-3 Chatbot (CPU-Friendly) with Avatars and Live Typing")
 
     with gr.Row():
         with gr.Column(scale=1):
             user_avatar = gr.Image("user_avatar.png", elem_id="user-avatar")
         with gr.Column(scale=4):
             user_input = gr.Textbox(label="Your Message", placeholder="Type something...")
-    
+
     with gr.Row():
         with gr.Column(scale=1):
             ai_avatar = gr.Image("ai_avatar.png", elem_id="ai-avatar")