Add Qwen3VL & moondream3

app.py

@@ -9,22 +9,20 @@ from qwen_vl_utils import process_vision_info
 from transformers import (
     AutoModelForCausalLM,
     AutoProcessor,
-    Qwen2_5_VLForConditionalGeneration,
+    Qwen3VLForConditionalGeneration,
 )
 
 from spaces import GPU
 import supervision as sv
 
-# --- Config ---
-model_qwen_id = "Qwen/Qwen2.5-VL-3B-Instruct"
-model_moondream_id = "vikhyatk/moondream2"
+model_qwen_id = "Qwen/Qwen3-VL-4B-Instruct"
+model_moondream_id = "moondream/moondream3-preview"
 
-model_qwen = Qwen2_5_VLForConditionalGeneration.from_pretrained(
-    model_qwen_id, torch_dtype="auto", device_map="auto"
+model_qwen = Qwen3VLForConditionalGeneration.from_pretrained(
+    model_qwen_id, torch_dtype="auto", device_map="auto",
 )
 model_moondream = AutoModelForCausalLM.from_pretrained(
     model_moondream_id,
-    revision="2025-06-21",
     trust_remote_code=True,
     device_map={"": "cuda"},
 )
@@ -34,15 +32,11 @@ def extract_model_short_name(model_id):
     return model_id.split("/")[-1].replace("-", " ").replace("_", " ")
 
 
-model_qwen_name = extract_model_short_name(model_qwen_id)  # → "Qwen2.5 VL 3B Instruct"
-model_moondream_name = extract_model_short_name(model_moondream_id)  # → "moondream2"
+model_qwen_name = extract_model_short_name(model_qwen_id)  
+model_moondream_name = extract_model_short_name(model_moondream_id)  
 
 
-min_pixels = 224 * 224
-max_pixels = 1024 * 1024
-processor_qwen = AutoProcessor.from_pretrained(
-    "Qwen/Qwen2.5-VL-3B-Instruct", min_pixels=min_pixels, max_pixels=max_pixels
-)
+processor_qwen = AutoProcessor.from_pretrained(model_qwen_id)
 
 
 def create_annotated_image(image, json_data, height, width):
@@ -112,7 +106,6 @@ def create_annotated_image_normalized(image, json_data, label="object"):
     original_width, original_height = image.size
     annotated_image = np.array(image.convert("RGB"))
 
-    # Handle points for keypoint detection
     points = []
     if "points" in json_data:
         for point in json_data.get("points", []):
@@ -154,6 +147,79 @@ def create_annotated_image_normalized(image, json_data, label="object"):
 
     return Image.fromarray(annotated_image)
 
+
+def parse_qwen3_json(json_output):
+    lines = json_output.splitlines()
+    for i, line in enumerate(lines):
+        if line == "```json":
+            json_output = "\n".join(lines[i+1:])  
+            json_output = json_output.split("```")[0]  
+            break 
+    
+    try:
+        boxes = json.loads(json_output)
+    except json.JSONDecodeError:
+        end_idx = json_output.rfind('"}') + len('"}')
+        truncated_text = json_output[:end_idx] + "]"
+        boxes = json.loads(truncated_text)
+    
+    if not isinstance(boxes, list):
+        boxes = [boxes]
+    
+    return boxes
+
+
+def create_annotated_image_qwen3(image, json_output):
+    try:
+        boxes = parse_qwen3_json(json_output)
+    except Exception as e:
+        print(f"Error parsing JSON: {e}")
+        return image
+    
+    if not boxes:
+        return image
+    
+    original_width, original_height = image.size
+    annotated_image = np.array(image.convert("RGB"))
+    
+    xyxy = []
+    labels = []
+    
+    for box in boxes:
+        if "bbox_2d" in box and "label" in box:
+            x1, y1, x2, y2 = box["bbox_2d"]
+            scale = 1000
+            x1 = max(0, min(scale, x1)) / scale * original_width
+            y1 = max(0, min(scale, y1)) / scale * original_height
+            x2 = max(0, min(scale, x2)) / scale * original_width
+            y2 = max(0, min(scale, y2)) / scale * original_height
+            # Ensure x1 <= x2 and y1 <= y2
+            if x1 > x2: x1, x2 = x2, x1
+            if y1 > y2: y1, y2 = y2, y1
+            xyxy.append([int(x1), int(y1), int(x2), int(y2)])
+            labels.append(box["label"])
+    
+    if not xyxy:
+        return image
+    
+    detections = sv.Detections(
+        xyxy=np.array(xyxy),
+        class_id=np.arange(len(xyxy))
+    )
+    
+    bounding_box_annotator = sv.BoxAnnotator(color_lookup=sv.ColorLookup.INDEX)
+    label_annotator = sv.LabelAnnotator(color_lookup=sv.ColorLookup.INDEX)
+    
+    annotated_image = bounding_box_annotator.annotate(
+        scene=annotated_image, detections=detections
+    )
+    annotated_image = label_annotator.annotate(
+        scene=annotated_image, detections=detections, labels=labels
+    )
+    
+    return Image.fromarray(annotated_image)
+
+
 @GPU
 def detect_qwen(image, prompt):
     messages = [
@@ -167,16 +233,12 @@ def detect_qwen(image, prompt):
     ]
 
     t0 = time.perf_counter()
-    text = processor_qwen.apply_chat_template(
-        messages, tokenize=False, add_generation_prompt=True
-    )
-    image_inputs, video_inputs = process_vision_info(messages)
-    inputs = processor_qwen(
-        text=[text],
-        images=image_inputs,
-        videos=video_inputs,
-        padding=True,
-        return_tensors="pt",
+    inputs = processor_qwen.apply_chat_template(
+        messages,
+        tokenize=True,
+        add_generation_prompt=True,
+        return_dict=True,
+        return_tensors="pt"
     ).to(model_qwen.device)
 
     generated_ids = model_qwen.generate(**inputs, max_new_tokens=1024)
@@ -186,20 +248,15 @@ def detect_qwen(image, prompt):
     ]
     output_text = processor_qwen.batch_decode(
         generated_ids_trimmed,
-        do_sample=True,
         skip_special_tokens=True,
         clean_up_tokenization_spaces=False,
     )[0]
     elapsed_ms = (time.perf_counter() - t0) * 1_000
 
-    input_height = inputs["image_grid_thw"][0][1] * 14
-    input_width = inputs["image_grid_thw"][0][2] * 14
-
-    annotated_image = create_annotated_image(
-        image, output_text, input_height, input_width
-    )
+    annotated_image = create_annotated_image_qwen3(image, output_text)
 
     time_taken = f"**Inference time ({model_qwen_name}):** {elapsed_ms:.0f} ms"
+    
     return annotated_image, output_text, time_taken
 
 
@@ -251,15 +308,14 @@ button#gradio-share-link-button-0 {
 }
 """
 
-# --- Gradio Interface ---
 with gr.Blocks(theme=Ocean(), css=css_hide_share) as demo:
     gr.Markdown("# 👓 Object Understanding with Vision Language Models")
     gr.Markdown(
         "### Explore object detection, visual grounding, keypoint detection, and/or object counting through natural language prompts."
     )
     gr.Markdown("""
-    *Powered by [Qwen2.5-VL 3B](https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct) and [Moondream 2B (revision="2025-06-21")](https://huggingface.co/vikhyatk/moondream2). Inspired by the tutorial [Object Detection and Visual Grounding with Qwen 2.5](https://pyimagesearch.com/2025/06/09/object-detection-and-visual-grounding-with-qwen-2-5/) on PyImageSearch.*
-    *Moondream 2B uses the [moondream.py API](https://huggingface.co/vikhyatk/moondream2/blob/main/moondream.py), selecting `detect` for categories with "Object Detection" `point` for the ones with "Keypoint Detection", and reasoning-based querying for all others.*
+    *Powered by [Qwen3-VL 4B](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct) and [Moondream 3 Preview](https://huggingface.co/moondream/moondream3-preview). Inspired by the tutorial [Object Detection and Visual Grounding with Qwen 2.5](https://pyimagesearch.com/2025/06/09/object-detection-and-visual-grounding-with-qwen-2-5/) on PyImageSearch.*
+    *Moondream 3 uses the [moondream-preview](https://huggingface.co/vikhyatk/moondream2/blob/main/moondream.py), selecting `detect` for categories with "Object Detection" `point` for the ones with "Keypoint Detection", and reasoning-based querying for all others.*
     """)
 
     with gr.Row():
@@ -312,13 +368,13 @@ with gr.Blocks(theme=Ocean(), css=css_hide_share) as demo:
     example_prompts = [
         [
             "examples/example_1.jpg",
-            "Detect all objects in the image and return their locations and labels.",
+            "locate every instance in the image. Report bbox coordinates in JSON format.",
             "objects",
             "Object Detection",
         ],
         [
             "examples/example_2.JPG",
-            "Detect all the individual candies in the image and return their locations and labels.",
+            'locate every instance that belongs to the following categories: "candy, hand". Report bbox coordinates in JSON format.',
             "candies",
             "Object Detection",
         ],
@@ -336,7 +392,7 @@ with gr.Blocks(theme=Ocean(), css=css_hide_share) as demo:
         ],
         [
             "examples/example_1.jpg",
-            "Identify the red cars in this image, detect their key points and return their positions in the form of points.",
+            'locate every instance that belongs to the following categories: "red car". Report bbox coordinates in JSON format..',
             "red cars",
             "Visual Grounding + Keypoint Detection",
         ],
@@ -348,27 +404,26 @@ with gr.Blocks(theme=Ocean(), css=css_hide_share) as demo:
         ],
         [
             "examples/example_1.jpg",
-            "Detect the red car that is leading in this image and return its location and label.",
+            'locate every instance that belongs to the following categories: "leading red car". Report bbox coordinates in JSON format..',
             "leading red car",
             "Visual Grounding + Object Detection",
         ],
         [
             "examples/example_2.JPG",
-            "Detect the blue candy located at the top of the group in this image and return its location and label.",
+            'locate every instance that belongs to the following categories: "blue candy located at the top of the group". Report bbox coordinates in JSON format.',
             "blue candy located at the top of the group",
             "Visual Grounding + Object Detection",
         ],
     ]
-
     gr.Examples(
-        examples=example_prompts,
-        inputs=[
-            image_input,
-            prompt_input_model_1,
-            prompt_input_model_2,
-            category_input,
-        ],
-        label="Click an example to populate the input",
+         examples=example_prompts,
+         inputs=[
+             image_input,
+             prompt_input_model_1,
+             prompt_input_model_2,
+             category_input,
+         ],
+         label="Click an example to populate the input",
     )
 
     generate_btn.click(
@@ -390,4 +445,4 @@ with gr.Blocks(theme=Ocean(), css=css_hide_share) as demo:
     )
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()