all2

app.py

@@ -1,58 +0,0 @@
-import torch
-from transformers import AutoProcessor, AutoModelForVision2Seq
-from PIL import Image
-import base64
-import io
-
-# Load model & processor once at startup
-MODEL_NAME = "unsloth/qwen-2.5-vl-7b-instruct"
-
-device = "cuda" if torch.cuda.is_available() else "cpu"
-
-processor = AutoProcessor.from_pretrained(MODEL_NAME)
-model = AutoModelForVision2Seq.from_pretrained(MODEL_NAME).to(device)
-model.eval()
-
-def decode_base64_image(base64_str: str) -> Image.Image:
-    """Decode a base64 string to a PIL image"""
-    image_bytes = base64.b64decode(base64_str)
-    return Image.open(io.BytesIO(image_bytes)).convert("RGB")
-
-def handler(inputs: dict):
-    """
-    Expected input:
-    {
-        "image": "<base64 string>",
-        "prompt": "Describe this image"
-    }
-    """
-    try:
-        # Extract inputs
-        img_b64 = inputs.get("image")
-        prompt = inputs.get("prompt", "")
-
-        if not img_b64:
-            return {"error": "Missing 'image' field"}
-
-        image = decode_base64_image(img_b64)
-
-        # Preprocess
-        pixel_values = processor(images=image, return_tensors="pt").pixel_values.to(device)
-        input_ids = processor(prompt, return_tensors="pt").input_ids.to(device)
-
-        # Generate
-        outputs = model.generate(
-            input_ids=input_ids,
-            pixel_values=pixel_values,
-            max_new_tokens=256,
-            do_sample=True,
-            temperature=0.7
-        )
-
-        # Decode output
-        text = processor.batch_decode(outputs, skip_special_tokens=True)[0]
-
-        return {"generated_text": text}
-
-    except Exception as e:
-        return {"error": str(e)}

handler.py

@@ -0,0 +1,19 @@
+# app.py
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from PIL import Image
+import torch
+
+MODEL_ID = "unsloth/qwen2.5-vl-7b-instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_ID, torch_dtype=torch.float16, device_map="auto", trust_remote_code=True
+)
+
+def infer(request):
+    messages = request.get("messages", [])
+    images = request.get("images", [])
+
+    inputs = tokenizer.apply_chat_template(messages, return_tensors="pt").to(model.device)
+    outputs = model.generate(**inputs, max_new_tokens=512)
+    return {"text": tokenizer.decode(outputs[0])}

requirements.txt

@@ -1,5 +1,4 @@
-torch>=2.1.0
-transformers>=4.56.2
-Pillow
-safetensors
+transformers>=4.56.3
+torch>=2.2.0
 accelerate
+pillow