Update app.py

app.py

@@ -1,21 +1,36 @@
+import os
 import gradio as gr
-from transformers import pipeline
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
 
-# Load lightweight Flan-T5 model
-qa_pipeline = pipeline("text2text-generation", model="google/flan-t5-base")
+# Model info
+REPO = "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"
+FILENAME = "TinyLlama-1.1B-Chat-v1.0.Q4_K_M.gguf"
 
-def answer_question(question):
-    prompt = f"Answer the following programming question clearly:\n{question}"
-    result = qa_pipeline(prompt, max_length=200, do_sample=True, top_p=0.9, temperature=0.7)[0]["generated_text"]
-    return result.strip()
+# Download model (if not already)
+model_path = hf_hub_download(repo_id=REPO, filename=FILENAME, cache_dir="./models")
 
-demo = gr.Interface(
-    fn=answer_question,
-    inputs=gr.Textbox(lines=2, label="Ask a programming question"),
-    outputs=gr.Textbox(label="Answer"),
-    title="Lightweight Code Q&A (Flan-T5)",
-    description="Ask coding questions and get short, helpful answers using the Flan-T5 model."
+# Load model with llama-cpp
+llm = Llama(
+    model_path=model_path,
+    n_ctx=2048,
+    n_threads=4,
+    use_mlock=True
 )
 
-if __name__ == "__main__":
-    demo.launch()
+# Chat prompt wrapper
+def format_prompt(message, history):
+    conversation = ""
+    for user, bot in history:
+        conversation += f"<|user|>\n{user.strip()}\n<|assistant|>\n{bot.strip()}\n"
+    conversation += f"<|user|>\n{message.strip()}\n<|assistant|>\n"
+    return conversation
+
+def chat(message, history):
+    prompt = format_prompt(message, history)
+    output = llm(prompt, max_tokens=256, temperature=0.7, top_p=0.9, stop=["<|user|>", "<|assistant|>"])
+    reply = output["choices"][0]["text"].strip()
+    return reply
+
+# Gradio chat UI
+gr.ChatInterface(chat, title="TinyLlama CPU Chat", description="Lightweight local LLM (1.1B) powered by llama.cpp.").launch()