File size: 1,339 Bytes
040e231 528fa7d 040e231 528fa7d b76c6a3 040e231 6f5c507 528fa7d 6f5c507 528fa7d 6f5c507 528fa7d 6f5c507 b76c6a3 6f5c507 528fa7d 6f5c507 528fa7d 6f5c507 528fa7d 6f5c507 040e231 6f5c507 040e231 b76c6a3 528fa7d b76c6a3 040e231 b76c6a3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | import os
from functools import lru_cache
import gradio as gr
from huggingface_hub import hf_hub_download
from llama_cpp import Llama
MODEL_REPO = "microsoft/Phi-3-mini-4k-instruct-gguf"
MODEL_FILE = "Phi-3-mini-4k-instruct-q4.gguf"
@lru_cache(maxsize=1)
def get_model():
model_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILE)
return Llama(
model_path=model_path,
n_ctx=4096,
n_threads=min(4, os.cpu_count() or 2),
n_batch=128,
use_mmap=True,
verbose=False,
)
def build_prompt(message, history):
prompt = ""
for user, assistant in history:
prompt += f"User: {user}\nAssistant: {assistant}\n"
prompt += f"User: {message}\nAssistant:"
return prompt
def chat(message, history):
try:
model = get_model()
prompt = build_prompt(message, history)
output = model(
prompt,
max_tokens=200,
temperature=0.7,
top_p=0.9,
stop=["User:", "</s>"],
)
text = output["choices"][0]["text"].strip()
if not text:
return "..."
return text
except Exception as e:
return f"Error: {str(e)}"
demo = gr.ChatInterface(
fn=chat,
title="Phi-3 Mini CPU Chat",
)
if __name__ == "__main__":
demo.launch() |