File size: 1,323 Bytes
82e52f9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | import os
import gradio as gr
from huggingface_hub import hf_hub_download
from llama_cpp import Llama
MODEL_REPO = "unsloth/Qwen3-VL-2B-Instruct-GGUF"
MODEL_FILE = "Qwen3-VL-2B-Instruct-UD-Q8_K_XL.gguf"
HF_TOKEN = os.getenv("HF_TOKEN")
model_path = hf_hub_download(
repo_id=MODEL_REPO,
filename=MODEL_FILE,
token=HF_TOKEN
)
llm = Llama(
model_path=model_path,
n_ctx=2048,
n_threads=2,
n_gpu_layers=0,
n_batch=128,
verbose=False
)
SYSTEM_PROMPT = """És um assistente útil, claro e directo. Responde em português quando o utilizador escrever em português."""
def respond(message, history):
prompt = f"System: {SYSTEM_PROMPT}\n\n"
for item in history:
if item["role"] == "user":
prompt += f"User: {item['content']}\n"
elif item["role"] == "assistant":
prompt += f"Assistant: {item['content']}\n"
prompt += f"User: {message}\nAssistant:"
output = llm(
prompt,
max_tokens=256,
temperature=0.6,
top_p=0.9,
stop=["User:", "System:"]
)
return output["choices"][0]["text"].strip()
demo = gr.ChatInterface(
fn=respond,
type="messages",
title="Gemma 4 E4B - CPU Basic",
description="Demo em Hugging Face Spaces CPU Basic usando GGUF + llama.cpp."
)
demo.launch() |