File size: 662 Bytes
5ea2900 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
# app.py
from transformers import AutoModelForCausalLM, AutoTokenizer
from PIL import Image
import torch
MODEL_ID = "unsloth/qwen2.5-vl-7b-instruct"
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID, torch_dtype=torch.float16, device_map="auto", trust_remote_code=True
)
def infer(request):
messages = request.get("messages", [])
images = request.get("images", [])
inputs = tokenizer.apply_chat_template(messages, return_tensors="pt").to(model.device)
outputs = model.generate(**inputs, max_new_tokens=512)
return {"text": tokenizer.decode(outputs[0])}
|