app.py

import gradio as gr
from llama_cpp import Llama

llm = Llama.from_pretrained(
    repo_id="bartowski/Llama-3.2-1B-Instruct-GGUF",
    filename="Llama-3.2-1B-Instruct-Q4_K_M.gguf",
    n_ctx=2048,
    n_threads=2,
    verbose=False,
)

def chat(message, history):
    response = llm.create_chat_completion(
        messages=[{"role": "user", "content": message}]
    )
    return response["choices"][0]["message"]["content"]

gr.ChatInterface(fn=chat).launch()