File size: 1,463 Bytes
5495267 bf09cd6 5495267 bf09cd6 5495267 530c337 bf09cd6 5495267 bf09cd6 5495267 bf09cd6 5495267 bf09cd6 5495267 bf09cd6 5495267 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | import gradio as gr
from transformers import AutoModelForCausalLM, AutoTokenizer
# מזהה המאגר ושם קובץ ה-GGUF
MODEL_ID = "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF"
GGUF_FILE = "qwen2.5-coder-7b-instruct-q4_k_m.gguf"
# 1. טעינת הטוקנייזר (ללא הפרמטר gguf_file)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
# 2. טעינת המודל בפורמט GGUF
model = AutoModelForCausalLM.from_pretrained(MODEL_ID, gguf_file=GGUF_FILE)
def answer(message, history):
messages = []
for item in history:
if isinstance(item, dict):
messages.append(item)
elif isinstance(item, (list, tuple)) and len(item) == 2:
u_msg, b_msg = item
if u_msg: messages.append({"role": "user", "content": u_msg})
if b_msg: messages.append({"role": "assistant", "content": b_msg})
messages.append({"role": "user", "content": message})
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = tokenizer(prompt, return_tensors="pt")
outputs = model.generate(**inputs, max_new_tokens=512, temperature=0.7)
response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
return response
demo = gr.ChatInterface(
fn=answer,
title="Qwen GGUF Assistant",
description="הרצת GGUF נקייה ומהירה על CPU"
)
if __name__ == "__main__":
demo.launch() |