Spaces:
Sleeping
Sleeping
File size: 1,179 Bytes
3f5e50b e8388b3 3f5e50b e8388b3 3f5e50b e8388b3 3f5e50b e01c5fe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | import gradio as gr
from huggingface_hub import hf_hub_download
from llama_cpp import Llama
# Fetch the highly optimized 4-bit quantized developer model
model_path = hf_hub_download(
repo_id="Qwen/Qwen2.5-Coder-3B-Instruct-GGUF",
filename="qwen2.5-coder-3b-instruct-q4_k_m.gguf"
)
# Initialize Llama.cpp constrained to 2 vCPUs with a 4096 token context window
llm = Llama(model_path=model_path, n_ctx=4096, n_threads=2, verbose=False)
def generate(formatted_prompt):
"""
Accepts a ChatML formatted string, streams token generation from
Llama.cpp, and yields cumulative text chunks back to the Gradio client.
"""
stream = llm(
formatted_prompt,
max_tokens=1024,
stop=["<|im_end|>"],
stream=True
)
text = ""
for chunk in stream:
text += chunk['choices'][0]['text']
yield text
# Use Blocks to explicitly define the API name
with gr.Blocks() as demo:
prompt_input = gr.Textbox(label="Prompt")
output_text = gr.Textbox(label="Generated Text")
btn = gr.Button("Generate")
btn.click(fn=generate, inputs=prompt_input, outputs=output_text, api_name="predict")
demo.launch()
|