Update app.py

app.py

@@ -1,22 +1,25 @@
 import gradio as gr
-from transformers import pipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer
 
-# Load the Meta-Llama-3.1-8B-Instruct-GGUF model
+# Load the GGUF model and tokenizer
 model_name = "lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF"
-model = pipeline("text-generation", model=model_name, device=-1)  # -1 for CPU
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name)
 
-# Define the Gradio interface
+# Define a function to generate text using the GGUF model
 def generate_text(prompt):
-    output = model(prompt)[0]["generated_text"]
-    return output
+    inputs = tokenizer.encode(prompt, return_tensors="pt")
+    outputs = model.generate(inputs, max_length=50)
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)
 
-iface = gr.Interface(
-    fn=generate_text,
-    inputs=gr.Textbox(label="Prompt"),
-    outputs=gr.Textbox(label="Generated Text"),
-    title="Meta-Llama-3.1-8B-Instruct-GGUF Text Generation",
-    description="Enter a prompt to generate text using the Meta-Llama-3.1-8B-Instruct-GGUF model.",
+# Create a Gradio chatbot interface
+chatbot = gr.Chatbot(
+    generate_text,
+    title="GGUF Chatbot",
+    description="Talk to the GGUF model!",
+    width=800,
+    height=600,
 )
 
 # Launch the Gradio app
-iface.launch()
+chatbot.launch()