Add ZeroGPU gr.Workflow app for North-Micro-Vision-Instruct

README.md

@@ -5,7 +5,7 @@ colorFrom: indigo
 colorTo: indigo
 sdk: gradio
 sdk_version: 6.22.0
-app_file: run.py
+app_file: app.py
 pinned: false
 hf_oauth: true
 ---

app.py

@@ -0,0 +1,133 @@
+import os
+
+import spaces
+import torch
+import gradio as gr
+from transformers import AutoModelForImageTextToText, AutoProcessor
+
+MODEL_ID = "CohereLabs/North-Micro-Vision-Instruct"
+
+# Load once at startup. On ZeroGPU the weights stay resident and
+# @spaces.GPU allocates a worker per call.
+print(f"Loading {MODEL_ID} ...")
+processor = AutoProcessor.from_pretrained(MODEL_ID)
+model = AutoModelForImageTextToText.from_pretrained(
+    MODEL_ID,
+    dtype=torch.bfloat16,
+    device_map="cuda",
+)
+print("Model loaded!")
+
+
+def _load_image(image):
+    """Accept a workflow image value (dict with 'path', or a path/URL string)
+    and return a PIL.Image."""
+    from PIL import Image
+
+    if isinstance(image, dict):
+        image = image.get("path") or image.get("url")
+    if isinstance(image, str) and image.startswith(("http://", "https://")):
+        import requests
+        from io import BytesIO
+
+        return Image.open(BytesIO(requests.get(image, timeout=30).content)).convert("RGB")
+    return Image.open(image).convert("RGB")
+
+
+def _estimate_duration(image, prompt, max_new_tokens, temperature, top_p, top_k) -> int:
+    """Rough wall-clock estimate (seconds) for one VLM call. Requesting less
+    than the 60s default raises queue priority and frees the GPU slot sooner
+    for the next visitor. Scaled by max_new_tokens; clamped to a safe range."""
+    seconds = 10 + int(max_new_tokens) * 0.15
+    return max(20, min(int(seconds), 120))
+
+
+def _friendly_gpu_error(err: Exception) -> str:
+    msg = (str(err) or "").lower()
+    capacity_hints = (
+        "gpu limit", "reached its gpu limit", "gpu quota", "out of quota",
+        "quota", "no gpu", "could not allocate", "gpu is busy", "too many",
+        "concurrent",
+    )
+    if any(h in msg for h in capacity_hints):
+        return (
+            "⛔ This demo's shared GPU is at capacity right now — it's not a "
+            "problem with your input or your account. Please wait a minute and retry."
+        )
+    if "out of memory" in msg or "oom" in msg:
+        return (
+            "💥 Ran out of GPU memory. Try a smaller image or fewer max new "
+            "tokens, then retry."
+        )
+    return "⚠️ Generation failed. Please try again in a moment."
+
+
+@spaces.GPU(duration=_estimate_duration)
+def _run_vlm_gpu(image, prompt, max_new_tokens, temperature, top_p, top_k):
+    """GPU worker: runs only under a ZeroGPU allocation."""
+    if not prompt or not prompt.strip():
+        raise gr.Error("Please enter a prompt.")
+    if image is None:
+        raise gr.Error("Please provide an image.")
+
+    pil_image = _load_image(image)
+
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "image"},
+                {"type": "text", "text": prompt},
+            ],
+        }
+    ]
+
+    inputs = processor.apply_chat_template(
+        messages,
+        tokenize=True,
+        add_generation_prompt=True,
+        return_tensors="pt",
+        return_dict=True,
+    )
+    inputs = {k: v.to(model.device) if hasattr(v, "to") else v for k, v in inputs.items()}
+    inputs["pixel_values"] = inputs["pixel_values"].to(torch.bfloat16)
+
+    do_sample = float(temperature) > 0
+    gen_kwargs = dict(max_new_tokens=int(max_new_tokens), do_sample=do_sample)
+    if do_sample:
+        gen_kwargs.update(
+            temperature=float(temperature),
+            top_p=float(top_p),
+            top_k=int(top_k),
+        )
+
+    outputs = model.generate(**inputs, **gen_kwargs)
+    generated_ids = outputs[0][inputs["input_ids"].shape[1]:]
+    return processor.decode(
+        generated_ids,
+        skip_special_tokens=True,
+        clean_up_tokenization_spaces=False,
+    )
+
+
+def run_vlm(image, prompt, max_new_tokens, temperature, top_p, top_k):
+    """Workflow-facing wrapper bound to the canvas as a `fn` operator node.
+    Catches ZeroGPU allocator rejections and rewords them for users."""
+    try:
+        return _run_vlm_gpu(image, prompt, max_new_tokens, temperature, top_p, top_k)
+    except gr.Error:
+        raise
+    except Exception as e:
+        raise gr.Error(_friendly_gpu_error(e)) from e
+
+
+# The workflow (workflow.json) wires `run_vlm` as a `fn` operator:
+#   Image, Prompt, Max New Tokens, Temperature, Top P, Top K ─▶
+#   run_vlm (fn operator, kind="fn") ─▶ Response
+demo = gr.Workflow(
+    graph="workflow.json",
+    bind={"run_vlm": run_vlm},
+)
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,7 @@
+torch
+accelerate
+pillow
+requests
+spaces
+gradio
+git+https://github.com/huggingface/transformers.git

run.py

@@ -1,6 +0,0 @@
-import gradio as gr
-
-demo = gr.Workflow()
-
-if __name__ == "__main__":
-    demo.launch()

workflow.json

@@ -1 +1,137 @@
-{"schema_version":"2","name":"My Workflow","runtime":{"default":"client"},"references":[],"operators":[],"subjects":[],"edges":[],"view":{"default":"canvas"}}
+{
+  "schema_version": "2",
+  "name": "North Micro Vision Instruct",
+  "description": "Visual question answering, captioning, OCR, and grounding with CohereLabs/North-Micro-Vision-Instruct (2.4B native-resolution VLM) on ZeroGPU, driven by a gr.Workflow fn-bound @spaces.GPU function. Single Space: the workflow frontend and the GPU worker share one process.",
+  "runtime": { "default": "client" },
+  "view": { "default": "canvas" },
+  "references": [
+    {
+      "id": "ref_image",
+      "label": "Input Image",
+      "role": "reference",
+      "asset_type": "image",
+      "inputs": [{ "id": "in", "label": "Image", "type": "image" }],
+      "outputs": [{ "id": "out", "label": "Image", "type": "image" }],
+      "x": 60,
+      "y": 120,
+      "width": 240,
+      "height": 140,
+      "data": {}
+    },
+    {
+      "id": "ref_prompt",
+      "label": "Prompt",
+      "role": "reference",
+      "asset_type": "text",
+      "inputs": [{ "id": "in", "label": "Prompt", "type": "text" }],
+      "outputs": [{ "id": "out", "label": "Prompt", "type": "text" }],
+      "x": 60,
+      "y": 300,
+      "width": 240,
+      "height": 120,
+      "data": { "out": "What do you see in this image?" }
+    },
+    {
+      "id": "ref_max_tokens",
+      "label": "Max New Tokens",
+      "role": "reference",
+      "asset_type": "number",
+      "inputs": [{ "id": "in", "label": "Max New Tokens", "type": "number" }],
+      "outputs": [{ "id": "out", "label": "Max New Tokens", "type": "number" }],
+      "x": 60,
+      "y": 460,
+      "width": 200,
+      "height": 90,
+      "data": { "out": 256 }
+    },
+    {
+      "id": "ref_temperature",
+      "label": "Temperature",
+      "role": "reference",
+      "asset_type": "number",
+      "inputs": [{ "id": "in", "label": "Temperature", "type": "number" }],
+      "outputs": [{ "id": "out", "label": "Temperature", "type": "number" }],
+      "x": 60,
+      "y": 570,
+      "width": 200,
+      "height": 90,
+      "data": { "out": 0.7 }
+    },
+    {
+      "id": "ref_top_p",
+      "label": "Top P",
+      "role": "reference",
+      "asset_type": "number",
+      "inputs": [{ "id": "in", "label": "Top P", "type": "number" }],
+      "outputs": [{ "id": "out", "label": "Top P", "type": "number" }],
+      "x": 60,
+      "y": 680,
+      "width": 200,
+      "height": 90,
+      "data": { "out": 0.8 }
+    },
+    {
+      "id": "ref_top_k",
+      "label": "Top K",
+      "role": "reference",
+      "asset_type": "number",
+      "inputs": [{ "id": "in", "label": "Top K", "type": "number" }],
+      "outputs": [{ "id": "out", "label": "Top K", "type": "number" }],
+      "x": 60,
+      "y": 790,
+      "width": 200,
+      "height": 90,
+      "data": { "out": 20 }
+    }
+  ],
+  "operators": [
+    {
+      "id": "op_vlm",
+      "label": "run_vlm",
+      "role": "operator",
+      "kind": "fn",
+      "source": "fn",
+      "fn": "run_vlm",
+      "inputs": [
+        { "id": "in_0", "label": "image", "type": "image", "required": true },
+        { "id": "in_1", "label": "prompt", "type": "text", "required": true },
+        { "id": "in_2", "label": "max_new_tokens", "type": "number" },
+        { "id": "in_3", "label": "temperature", "type": "number" },
+        { "id": "in_4", "label": "top_p", "type": "number" },
+        { "id": "in_5", "label": "top_k", "type": "number" }
+      ],
+      "outputs": [
+        { "id": "out_0", "label": "response", "type": "text", "output_index": 0 }
+      ],
+      "x": 420,
+      "y": 380,
+      "width": 280,
+      "height": 220,
+      "data": {}
+    }
+  ],
+  "subjects": [
+    {
+      "id": "sub_response",
+      "label": "Response",
+      "role": "subject",
+      "asset_type": "text",
+      "inputs": [{ "id": "in", "label": "Response", "type": "text" }],
+      "outputs": [{ "id": "out", "label": "Response", "type": "text" }],
+      "x": 820,
+      "y": 380,
+      "width": 280,
+      "height": 140,
+      "data": {}
+    }
+  ],
+  "edges": [
+    { "id": "e_image",       "from_node_id": "ref_image",       "from_port_id": "out",   "to_node_id": "op_vlm",       "to_port_id": "in_0", "type": "image" },
+    { "id": "e_prompt",      "from_node_id": "ref_prompt",      "from_port_id": "out",   "to_node_id": "op_vlm",       "to_port_id": "in_1", "type": "text" },
+    { "id": "e_max_tokens",  "from_node_id": "ref_max_tokens",  "from_port_id": "out",   "to_node_id": "op_vlm",       "to_port_id": "in_2", "type": "number" },
+    { "id": "e_temperature", "from_node_id": "ref_temperature", "from_port_id": "out",   "to_node_id": "op_vlm",       "to_port_id": "in_3", "type": "number" },
+    { "id": "e_top_p",       "from_node_id": "ref_top_p",       "from_port_id": "out",   "to_node_id": "op_vlm",       "to_port_id": "in_4", "type": "number" },
+    { "id": "e_top_k",       "from_node_id": "ref_top_k",       "from_port_id": "out",   "to_node_id": "op_vlm",       "to_port_id": "in_5", "type": "number" },
+    { "id": "e_response",    "from_node_id": "op_vlm",          "from_port_id": "out_0", "to_node_id": "sub_response", "to_port_id": "in",   "type": "text" }
+  ]
+}