Spaces:
Runtime error
Runtime error
File size: 2,444 Bytes
c51faba 67c17af c51faba 67c17af c51faba 67c17af c51faba 67c17af c51faba 67c17af 4ffb95c 67c17af c51faba 67c17af c51faba 4ffb95c c51faba | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | import os
import gradio as gr
from huggingface_hub import hf_hub_download
from llama_cpp import Llama
# 1. Download the highly optimized Llama 3.2 3B model from Hugging Face repository
print("Downloading model...")
model_path = hf_hub_download(
repo_id="bartowski/Llama-3.2-3B-Instruct-GGUF",
filename="Llama-3.2-3B-Instruct-Q4_K_M.gguf"
)
# 2. Initialize the model on the CPU
print("Initializing model...")
llm = Llama(
model_path=model_path,
n_ctx=2048, # Context length
n_threads=2 # Utilizes both free CPU cores fully
)
# 3. Define the chatbot logic
def respond(message, chat_history):
# Format the prompt to match Llama 3.2 structural rules
formatted_prompt = "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n"
formatted_prompt += "You are a helpful, direct, and honest AI assistant.<|eot_id|>"
# Inject chat history so the bot remembers the conversation context
for turn in chat_history:
# Check if history is structured as objects or dicts (Gradio 6 style)
user_msg = turn.get("text") if isinstance(turn, dict) else turn
bot_msg = turn.get("text") if isinstance(turn, dict) else turn
if user_msg:
formatted_prompt += f"<|start_header_id|>user<|end_header_id|>\n{user_msg}<|eot_id|>"
if bot_msg:
formatted_prompt += f"<|start_header_id|>assistant<|end_header_id|>\n{bot_msg}<|eot_id|>"
# Add the newest user message
formatted_prompt += f"<|start_header_id|>user<|end_header_id|>\n{message}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n"
# Generate response tokens streamingly
output = llm(
formatted_prompt,
max_tokens=512,
stop=["<|eot_id|>"],
stream=True
)
token_accumulator = ""
for token in output:
token_text = token["choices"]["text"]
token_accumulator += token_text
yield token_accumulator
# 4. Create the web dashboard layout using a global theme block wrapper
with gr.Blocks(theme="soft") as demo:
gr.ChatInterface(
fn=respond,
title="🤖 Free Llama 3.2 CPU Chatbot",
description="Running 24/7/365 for free on Hugging Face Spaces using CPU inference.",
examples=["Explain quantum computing simply.", "Write a short poem about coding."]
)
if __name__ == "__main__":
demo.launch(server_name="0.0.0.0", server_port=7860)
|