| import gradio as gr |
| from transformers import AutoModelForCausalLM, AutoTokenizer |
|
|
| |
| MODEL_ID = "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF" |
| GGUF_FILE = "qwen2.5-coder-7b-instruct-q4_k_m.gguf" |
|
|
| |
| tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) |
|
|
| |
| model = AutoModelForCausalLM.from_pretrained(MODEL_ID, gguf_file=GGUF_FILE) |
|
|
| def answer(message, history): |
| messages = [] |
| for item in history: |
| if isinstance(item, dict): |
| messages.append(item) |
| elif isinstance(item, (list, tuple)) and len(item) == 2: |
| u_msg, b_msg = item |
| if u_msg: messages.append({"role": "user", "content": u_msg}) |
| if b_msg: messages.append({"role": "assistant", "content": b_msg}) |
|
|
| messages.append({"role": "user", "content": message}) |
|
|
| prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) |
| inputs = tokenizer(prompt, return_tensors="pt") |
| |
| outputs = model.generate(**inputs, max_new_tokens=512, temperature=0.7) |
| response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True) |
| |
| return response |
|
|
| demo = gr.ChatInterface( |
| fn=answer, |
| title="Qwen GGUF Assistant", |
| description="הרצת GGUF נקייה ומהירה על CPU" |
| ) |
|
|
| if __name__ == "__main__": |
| demo.launch() |
