ZeroGPU

README.md

@@ -4,22 +4,15 @@ emoji: 💬
 colorFrom: yellow
 colorTo: purple
 sdk: gradio
-sdk_version: "6.5.1"
-python_version: "3.11.13"
+sdk_version: "6.11.0"
+python_version: "3.12.12"
 app_file: app.py
 pinned: false
-hf_oauth: true
-hf_oauth_scopes:
-  - inference-api
 license: apache-2.0
 short_description: "a compact Vision-Language Model"
-startup_duration_timeout: 1h
-preload_from_hub:
-  - tencent/Penguin-VL-8B
 ---
 
-This Space runs Penguin-VL-8B and preloads model artifacts from `tencent/Penguin-VL-8B`.
+This Space runs Penguin-VL-8B on ZeroGPU.
 
-On dedicated GPU hardware, the app also loads the model into GPU memory during startup so users can interact without waiting for the first request to initialize the model.
-
-If you move this Space back to ZeroGPU later, restore the `@spaces.GPU(...)` path and set `PRELOAD_MODEL_ON_STARTUP=0` in the Space variables.
+The model is loaded into CPU memory at startup. ZeroGPU transparently migrates
+it to GPU when a request is processed via `@spaces.GPU`.

app.py

@@ -1,9 +1,10 @@
+import spaces  # noqa: F401 — Must precede torch for ZeroGPU monkey-patching
 import os
 import warnings
 
 from inference.interface import PenguinVLQwen3GradioInterface
 from inference.server import PenguinVLQwen3DirectClient
-from inference.server.direct_client import ensure_flash_attn_installed, preload_model
+from inference.server.direct_client import preload_model
 
 warnings.filterwarnings(
     "ignore",
@@ -14,12 +15,7 @@ warnings.filterwarnings(
 
 def main():
     model_path = os.getenv("MODEL_PATH", "tencent/Penguin-VL-8B")
-    ensure_flash_attn_installed()
-    if os.getenv("PRELOAD_MODEL_ON_STARTUP", "1") == "1":
-        try:
-            preload_model(model_path)
-        except Exception as exc:
-            print(f"Startup model preload skipped: {exc}")
+    preload_model(model_path)
 
     model_client = PenguinVLQwen3DirectClient(
         model_path=model_path,

inference/interface/gradio_interface.py

@@ -1,4 +1,3 @@
-import os
 import os.path as osp
 
 import gradio as gr
@@ -10,233 +9,156 @@ Developed by [Penguin-VL](https://github.com/tencent-ailab/Penguin-VL) team at T
 
 Note: speed on ZeroGPU does not reflect real model speed and may be influenced by the shared environment. For stable and fast Gradio Space deployment and running, please visit [the local UI instructions](https://github.com/tencent-ailab/Penguin-VL?tab=readme-ov-file#-gradio-demo-local-ui). For usage examples and expected results, please refer to [here](https://github.com/tencent-ailab/Penguin-VL/blob/master/inference/notebooks/01_penguinvl_inference_recipes.public.ipynb).
 
-Please login with your Hugging Face account first. We provide some example images and videos for easier trials.
+Please login with your Hugging Face account first.
 """
 
+IMAGE_FORMATS = ("png", "jpg", "jpeg")
+VIDEO_FORMATS = ("mp4", "mov")
 
-class PenguinVLQwen3GradioInterface(object):
+# (filename, prompt) pairs sourced from the official inference notebook.
+EXAMPLE_PAIRS = [
+    ("leetcode.png", "please think this problem step by step and give the python code solution"),
+    ("newspaper.png", "please output the text in the image"),
+    ("horse_poet.png", "Write a short poem inspired by this image. Capture the relationship between the man and the horse, as well as the traditional, historical atmosphere of the painting."),
+    ("2b_table_result.png", "please output the table content in markdown format."),
+    ("chart_understanding.png", "Look at the 'Nonmetropolitan' line. In what approximate year does it reach its absolute lowest point on the chart, and what is the approximate percent change at that time?"),
+    ("video-example.mp4", "please describe the video in details"),
+    ("polar_bear.mp4", "Describe what happens in this video."),
+]
+
+
+class PenguinVLQwen3GradioInterface:
 
     def __init__(self, model_client, example_dir=None, default_system_prompt="You are a helpful assistant developed by Tencent AI Lab PenguinVL team.", **server_kwargs):
         self.model_client = model_client
         self.server_kwargs = server_kwargs
         self.default_system_prompt = (default_system_prompt or "").strip()
 
-        self.image_formats = ("png", "jpg", "jpeg")
-        self.video_formats = ("mp4", "mov")
-        image_examples, video_examples = [], []
-        if example_dir is not None:
-            example_files = [
-                osp.join(example_dir, f) for f in os.listdir(example_dir)
-            ]
-            for example_file in example_files:
-                if example_file.endswith(self.image_formats):
-                    image_examples.append([example_file])
-                elif example_file.endswith(self.video_formats):
-                    video_examples.append([example_file])
-
-        with gr.Blocks() as self.interface:
-            gr.Markdown(HEADER)
-            with gr.Row():
-                chatbot_kwargs = {"elem_id": "chatbot", "height": 710}
-                try:
-                    chatbot = gr.Chatbot(type="messages", **chatbot_kwargs)
-                except TypeError:
-                    # Gradio 6 uses OpenAI-style messages by default and removed the `type` arg.
-                    chatbot = gr.Chatbot(**chatbot_kwargs)
-
-                with gr.Column():
-                    with gr.Tab(label="Input"):
-
-                        with gr.Row():
-                            input_video = gr.Video(sources=["upload"], label="Upload Video")
-                            input_image = gr.Image(sources=["upload"], type="filepath", label="Upload Image")
-                        
-                        if len(image_examples):
-                            gr.Examples(image_examples, inputs=[input_image], label="Example Images")
-                        if len(video_examples):
-                            gr.Examples(video_examples, inputs=[input_video], label="Example Videos")
-
-                        input_text = gr.Textbox(label="Input Text", placeholder="Type your message here and press enter to submit")
-
-                        submit_button = gr.Button("Generate")
-
-                    with gr.Tab(label="Configure"):
-                        with gr.Accordion("Prompt Config", open=True):
-                            system_prompt = gr.Textbox(
-                                value=self.default_system_prompt,
-                                label="System Prompt",
-                                lines=4,
-                                placeholder="Optional: system instruction prepended to each request",
-                            )
-
-                        with gr.Accordion("Generation Config", open=True):
-                            do_sample = gr.Checkbox(value=True, label="Do Sample")
-                            temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.1, label="Temperature")
-                            top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.9, label="Top P")
-                            max_new_tokens = gr.Slider(minimum=0, maximum=4096, value=1536, step=1, label="Max New Tokens")
-
-                        with gr.Accordion("Video Config", open=True):
-                            fps = gr.Slider(minimum=0.0, maximum=10.0, value=1, label="FPS")
-                            max_frames = gr.Slider(minimum=0, maximum=256, value=180, step=1, label="Max Frames")
-
-            input_video.change(self._on_video_upload, [chatbot, input_video], [chatbot, input_video])
-            input_image.change(self._on_image_upload, [chatbot, input_image], [chatbot, input_image])
-            input_text.submit(
-                self._predict,
-                [
-                    chatbot, input_text, system_prompt, do_sample, temperature, top_p, max_new_tokens,
-                    fps, max_frames,
-                ],
-                [chatbot, input_text],
-            )
-            submit_button.click(
-                self._predict,
-                [
-                    chatbot, input_text, system_prompt, do_sample, temperature, top_p, max_new_tokens,
-                    fps, max_frames,
-                ],
-                [chatbot, input_text],
-            )
-
-    def _on_video_upload(self, messages, video):
-        messages = messages or []
-        if video is not None:
-            # messages.append({"role": "user", "content": gr.Video(video)})
-            messages.append({"role": "user", "content": {"path": video}})
-        return messages, None
-
-    def _on_image_upload(self, messages, image):
-        messages = messages or []
-        if image is not None:
-            # messages.append({"role": "user", "content": gr.Image(image)})
-            messages.append({"role": "user", "content": {"path": image}})
-        return messages, None
-
-    def _on_text_submit(self, messages, text):
-        messages = messages or []
-        messages.append({"role": "user", "content": text})
-        return messages, ""
-
-    def _extract_media_path(self, content):
-        if isinstance(content, dict):
-            if content.get("type") == "text" and isinstance(content.get("text"), str):
-                raise ValueError(f"Text content is not media: {content}")
-            media_path = content.get("path")
-            if media_path:
-                return media_path
-            for value in content.values():
-                try:
-                    return self._extract_media_path(value)
-                except ValueError:
-                    continue
-
-        if isinstance(content, (list, tuple)) and len(content) > 0:
-            for item in content:
-                try:
-                    return self._extract_media_path(item)
-                except ValueError:
-                    continue
-
-        raise ValueError(f"Unsupported media content: {content}")
-
-    def _extract_text_content(self, content):
+        examples = []
+        if example_dir:
+            for filename, prompt in EXAMPLE_PAIRS:
+                path = osp.join(example_dir, filename)
+                if osp.isfile(path):
+                    examples.append([{"text": prompt, "files": [path]}])
+
+        self.interface = gr.ChatInterface(
+            fn=self._predict,
+            multimodal=True,
+            description=HEADER,
+            additional_inputs=[
+                gr.Textbox(label="System Prompt", value=self.default_system_prompt, lines=4),
+                gr.Checkbox(label="Do Sample", value=True),
+                gr.Slider(label="Temperature", minimum=0.0, maximum=1.0, value=0.1),
+                gr.Slider(label="Top P", minimum=0.0, maximum=1.0, value=0.9),
+                gr.Slider(label="Max New Tokens", minimum=0, maximum=4096, value=1536, step=1),
+                gr.Slider(label="FPS (video)", minimum=0.0, maximum=10.0, value=1),
+                gr.Slider(label="Max Frames (video)", minimum=0, maximum=256, value=180, step=1),
+            ],
+            additional_inputs_accordion=gr.Accordion(label="Settings", open=False),
+            examples=examples or None,
+        )
+
+    def _classify_file(self, path):
+        ext = osp.splitext(path)[1].lower().lstrip(".")
+        if ext in VIDEO_FORMATS:
+            return "video"
+        if ext in IMAGE_FORMATS:
+            return "image"
+        return None
+
+    def _normalize_content(self, content, fps, max_frames):
+        """Convert a single history content entry to model conversation format."""
         if isinstance(content, str):
-            return content
+            return [{"type": "text", "text": content}]
 
         if isinstance(content, dict):
-            if content.get("type") == "text" and isinstance(content.get("text"), str):
-                return content["text"]
+            path = content.get("path") or content.get("url", "")
+            if path:
+                media_type = self._classify_file(path)
+                if media_type == "video":
+                    return [{"type": "video", "video": {"video_path": path, "fps": fps, "max_frames": max_frames}}]
+                if media_type == "image":
+                    return [{"type": "image", "image": {"image_path": path}}]
             text = content.get("text")
             if isinstance(text, str):
-                return text
+                return [{"type": "text", "text": text}]
 
-        if isinstance(content, (list, tuple)) and len(content) > 0:
-            text_parts = []
+        if isinstance(content, list):
+            result = []
             for item in content:
-                try:
-                    text_parts.append(self._extract_text_content(item))
-                except ValueError:
-                    continue
-            if text_parts:
-                return "\n".join(part for part in text_parts if part)
+                result.extend(self._normalize_content(item, fps, max_frames))
+            return result
 
-        raise ValueError(f"Unsupported text content: {content}")
+        return []
 
-    def _normalize_user_content(self, content, fps, max_frames):
-        if isinstance(content, str):
-            return [{"type": "text", "text": content}]
-
-        if isinstance(content, (list, tuple)):
-            normalized_items = []
-            for item in content:
-                normalized_items.extend(self._normalize_user_content(item, fps, max_frames))
-            return normalized_items
-
-        if isinstance(content, dict):
-            try:
-                text = self._extract_text_content(content)
-            except ValueError:
-                text = None
-            else:
-                return [{"type": "text", "text": text}]
+    def _build_conversation(self, message, history, system_prompt, fps, max_frames):
+        conversation = []
 
-            media_path = self._extract_media_path(content)
-            media_ext = osp.splitext(media_path)[1].lower().lstrip(".")
-            if media_ext in self.video_formats:
-                return [{"type": "video", "video": {"video_path": media_path, "fps": fps, "max_frames": max_frames}}]
-            if media_ext in self.image_formats:
-                return [{"type": "image", "image": {"image_path": media_path}}]
-            raise ValueError(f"Unsupported media type: {media_path}")
-
-        raise ValueError(f"Unsupported user content: {content}")
-
-    def _predict(self, messages, input_text, system_prompt, do_sample, temperature, top_p, max_new_tokens,
-                 fps, max_frames):
-        messages = list(messages or [])
-        input_text = input_text or ""
-        if input_text and len(input_text) > 0:
-            messages.append({"role": "user", "content": input_text})
-        new_messages = []
         active_system_prompt = (system_prompt or self.default_system_prompt).strip()
         if active_system_prompt:
-            new_messages.append({
+            conversation.append({
                 "role": "system",
                 "content": [{"type": "text", "text": active_system_prompt}],
             })
 
-        contents = []
-        for message in messages:
-            if message["role"] == "assistant":
-                if len(contents):
-                    new_messages.append({"role": "user", "content": contents})
-                    contents = []
-                new_messages.append(message)
-            elif message["role"] == "user":
-                contents.extend(self._normalize_user_content(message["content"], fps, max_frames))
+        # History — merge consecutive user messages into one turn
+        for entry in history:
+            role = entry["role"]
+            if role == "assistant":
+                conversation.append({"role": "assistant", "content": entry["content"]})
+            elif role == "user":
+                normalized = self._normalize_content(entry["content"], fps, max_frames)
+                if not normalized:
+                    continue
+                if conversation and conversation[-1]["role"] == "user":
+                    conversation[-1]["content"].extend(normalized)
+                else:
+                    conversation.append({"role": "user", "content": normalized})
+
+        # Current message
+        current_content = []
+        for f in message.get("files") or []:
+            path = f if isinstance(f, str) else f.get("path", "")
+            media_type = self._classify_file(path)
+            if media_type == "video":
+                current_content.append({"type": "video", "video": {"video_path": path, "fps": fps, "max_frames": max_frames}})
+            elif media_type == "image":
+                current_content.append({"type": "image", "image": {"image_path": path}})
+        text = (message.get("text") or "").strip()
+        if text:
+            current_content.append({"type": "text", "text": text})
+
+        if current_content:
+            if conversation and conversation[-1]["role"] == "user":
+                conversation[-1]["content"].extend(current_content)
+            else:
+                conversation.append({"role": "user", "content": current_content})
+
+        return conversation
 
-        if len(contents):
-            new_messages.append({"role": "user", "content": contents})
+    def _predict(self, message, history, system_prompt, do_sample, temperature, top_p, max_new_tokens, fps, max_frames):
+        conversation = self._build_conversation(message, history, system_prompt, fps, max_frames)
 
-        if len(new_messages) == 0 or new_messages[-1]["role"] != "user":
-            return messages
+        if not conversation or conversation[-1]["role"] != "user":
+            yield ""
+            return
 
         generation_config = {
             "do_sample": do_sample,
             "temperature": temperature,
             "top_p": top_p,
-            "max_new_tokens": max_new_tokens
+            "max_new_tokens": max_new_tokens,
         }
 
-        response = self.model_client.submit({"conversation": new_messages, "generation_config": generation_config})
+        response = self.model_client.submit({"conversation": conversation, "generation_config": generation_config})
         if isinstance(response, str):
-            messages.append({"role": "assistant", "content": response})
-            yield messages, ""
+            yield response
             return
 
-        messages.append({"role": "assistant", "content": ""})
+        text = ""
         for token in response:
-            messages[-1]['content'] += token
-            yield messages, ""
+            text += token
+            yield text
 
     def launch(self):
-        self.interface.launch(**self.server_kwargs)
+        self.interface.launch(ssr_mode=False, **self.server_kwargs)

inference/server/direct_client.py

@@ -1,10 +1,7 @@
-import importlib
-import importlib.util
 import os
-import subprocess
-import sys
 from threading import Lock, Thread
 
+import spaces
 import torch
 from transformers import AutoModelForCausalLM, AutoProcessor, TextIteratorStreamer
 
@@ -13,42 +10,16 @@ _MODEL = None
 _PROCESSOR = None
 _MODEL_PATH = None
 _MODEL_LOCK = Lock()
-_FLASH_ATTN_LOCK = Lock()
-_FLASH_ATTN_PACKAGE = "flash_attn"
-_FLASH_ATTN_REQUIREMENT = os.getenv("FLASH_ATTN_REQUIREMENT", "flash-attn==2.8.3")
 
 
 def _get_attn_implementation():
-    return os.getenv("ATTN_IMPLEMENTATION", "sdpa")
+    return os.getenv("ATTN_IMPLEMENTATION", "flash_attention_2")
 
 
 def _get_model_revision():
     return os.getenv("MODEL_REVISION")
 
 
-def ensure_flash_attn_installed():
-    if importlib.util.find_spec(_FLASH_ATTN_PACKAGE) is not None:
-        return
-
-    with _FLASH_ATTN_LOCK:
-        if importlib.util.find_spec(_FLASH_ATTN_PACKAGE) is not None:
-            return
-
-        install_cmd = [
-            sys.executable,
-            "-m",
-            "pip",
-            "install",
-            _FLASH_ATTN_REQUIREMENT,
-            "--no-build-isolation",
-        ]
-        print(f"Installing {_FLASH_ATTN_REQUIREMENT} with --no-build-isolation...")
-        subprocess.check_call(install_cmd, env=os.environ.copy())
-        importlib.invalidate_caches()
-        if importlib.util.find_spec(_FLASH_ATTN_PACKAGE) is None:
-            raise RuntimeError(f"Failed to import {_FLASH_ATTN_PACKAGE} after installation.")
-
-
 def _ensure_model_loaded(model_path):
     global _MODEL, _PROCESSOR, _MODEL_PATH
 
@@ -59,7 +30,6 @@ def _ensure_model_loaded(model_path):
         if _MODEL is not None and _PROCESSOR is not None and _MODEL_PATH == model_path:
             return _MODEL, _PROCESSOR
 
-        ensure_flash_attn_installed()
         attn_implementation = _get_attn_implementation()
         revision = _get_model_revision()
 
@@ -88,6 +58,7 @@ def preload_model(model_path):
     return _ensure_model_loaded(model_path)
 
 
+@spaces.GPU(duration=120)
 def _run_generation_stream(payload):
     model_path = payload["model_path"]
     model, processor = _ensure_model_loaded(model_path)

pre-requirements.txt

@@ -1,5 +0,0 @@
-# Build helpers recommended by the FlashAttention installation guide.
-packaging
-psutil
-ninja
-wheel

requirements.txt

@@ -1,9 +1,5 @@
 --extra-index-url https://download.pytorch.org/whl/cu124
 
-# Base runtime for Transformers inference and the Gradio demo.
-# Training, notebook, and vLLM-specific extras were removed from this file.
-# The previous full list is preserved in requirements.original.txt.
-
 # Core model runtime
 torch==2.5.1
 torchvision==0.20.1
@@ -13,7 +9,7 @@ accelerate==1.10.1
 huggingface_hub==0.34.4
 sentencepiece==0.1.99
 timm==1.0.3
-numpy==1.24.4
+numpy>=1.26.0
 Pillow
 einops==0.6.1
 einops-exts==0.0.4
@@ -22,16 +18,12 @@ einops-exts==0.0.4
 decord==0.6.0
 imageio==2.34.0
 imageio-ffmpeg==0.4.9
-opencv-python-headless==4.6.0.66
+opencv-python-headless
 ffmpeg-python
 requests
 
 # UI
 gradio>=5.44.1,<7
 
-# FlashAttention is installed separately with:
-# pip install flash-attn==2.8.3 --no-build-isolation
-# This cannot be expressed in a standard Gradio Space requirements install step.
-
-# Optional extras
-# vllm==0.11.0
+# FlashAttention — pre-built wheel from GitHub (no nvcc required)
+https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu12torch2.5cxx11abiFALSE-cp312-cp312-linux_x86_64.whl