Spaces:
Runtime error
Runtime error
File size: 1,952 Bytes
32a82ba 71e83b6 0e2b6ab 32a82ba 71e83b6 32a82ba 71e83b6 32a82ba 71e83b6 32a82ba 71e83b6 32a82ba 71e83b6 32a82ba 71e83b6 32a82ba 71e83b6 32a82ba 71e83b6 32a82ba | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | import gradio as gr
from llama_cpp import Llama
# -----------------------------
# Load GGUF model
# -----------------------------
def load_gguf():
model_name = "TheBloke/Wizard-Vicuna-30B-Uncensored-GGUF"
gguf_file = "wizard-vicuna-30b-uncensored.Q4_K_M.gguf"
llm = Llama.from_pretrained(
repo_id=model_name,
filename=gguf_file,
n_ctx=2048,
n_threads=8,
n_gpu_layers=0, # CPU only
verbose=False,
)
return llm
# -----------------------------
# Chat function
# -----------------------------
def chat_gguf(llm, history, message):
# Build conversation prompt
prompt = ""
for user, bot in history:
prompt += f"User: {user}\nAssistant: {bot}\n"
prompt += f"User: {message}\nAssistant:"
# Run inference
result = llm(
prompt,
max_tokens=512,
temperature=0.7,
top_p=0.9,
)
reply = result["choices"][0]["text"].strip()
return reply
# -----------------------------
# Gradio UI
# -----------------------------
def create_interface():
with gr.Blocks(title="Wizard Vicuna 30B GGUF Chat") as demo:
gr.Markdown(
"""
# 🧙♂️ Wizard‑Vicuna‑30B (GGUF)
Running on CPU using llama.cpp
"""
)
chatbot = gr.Chatbot(height=500)
msg = gr.Textbox(label="Your message")
clear = gr.Button("Clear Chat")
# Load model once at startup
state_model = gr.State(load_gguf())
# Chat handler
def respond(message, history, model):
reply = chat_gguf(model, history, message)
history.append([message, reply])
return history
msg.submit(
respond,
inputs=[msg, chatbot, state_model],
outputs=chatbot,
)
clear.click(lambda: [], None, chatbot)
return demo
demo = create_interface()
demo.launch()
|