Merge branch 'main' of https://huggingface.co/spaces/build-small-hackathon/PregoPal

_test_inference.py

@@ -1,17 +1,38 @@
-import sys, json, httpx, os
+"""
+Test inference endpoint for PregoPal MiniCPM-o API on Modal
+
+Usage:
+    python _test_inference.py
+    
+    # Or with custom URL:
+    python _test_inference.py https://andrew-jiabin--prego-pal-minicpm-serve.modal.run
+"""
+import sys
+import json
+import httpx
+import os
 
 os.environ["PYTHONIOENCODING"] = "utf-8"
+
+# Default URL - update after deployment
 URL = "https://andrew-jiabin--prego-pal-minicpm-serve.modal.run"
 
+if len(sys.argv) > 1:
+    URL = sys.argv[1].rstrip("/")
+
 payload = {
-    "model": "minicpm",
+    "model": "MiniCPM-o-4_5",
     "messages": [{"role": "user", "content": "你好，请用一句话回答：1+1等于几？"}],
     "max_tokens": 50,
     "temperature": 0.7
 }
 
-print("Sending request to:", URL + "/v1/chat/completions")
-print("This may take 2-8 minutes for cold start...")
+print("=" * 60)
+print("PregoPal MiniCPM-o API 测试")
+print("=" * 60)
+print(f"URL: {URL}/v1/chat/completions")
+print("注意：冷启动需要 2-10 分钟加载模型...")
+print()
 
 try:
     r = httpx.post(
@@ -19,14 +40,18 @@ try:
         json=payload,
         timeout=600  # 10 minutes for cold start
     )
-    print(f"\nStatus: {r.status_code}")
+    print(f"状态码: {r.status_code}")
     if r.status_code == 200:
         data = r.json()
         content = data["choices"][0]["message"]["content"]
-        print(f"\n=== MODEL RESPONSE ===\n{content}\n======================")
+        print(f"\n=== 模型响应 ===")
+        print(content)
+        print("=" * 20)
     else:
-        print(f"\nResponse body (first 500 chars):\n{r.text[:500]}")
+        print(f"\n响应 (前500字符):")
+        print(r.text[:500])
 except httpx.TimeoutException:
-    print("\nRequest timed out (600s) - container still initializing, check modal logs")
+    print("\n请求超时 (600s) - 容器仍在初始化，检查 Modal 日志")
+    print("  modal app logs $(modal app list | grep prego-pal | awk '{print $1}')")
 except Exception as e:
-    print(f"\nError: {e}")
+    print(f"\n错误: {e}")

modal_deploy/README.md

@@ -9,8 +9,9 @@
 ```
 用户请求 → FastAPI (ASGI) → llama-server (OpenAI 兼容) → MiniCPM-o-4_5 GGUF
                               ↓
-                    纯文本: /v1/chat/completions
-                    多模态: /v1/multimodal/chat (通过 llama-mtmd-cli)
+                    /v1/chat/completions (文本 + 图片多模态)
+                    /v1/embeddings
+                    /health
 ```
 
 ## 前置条件
@@ -21,26 +22,26 @@
 
 ## 模型文件
 
-需要 10 个 GGUF 文件（约 4.3 GB），从 HuggingFace 下载：
+部署需要 3 个核心 GGUF 文件（Volume 中需包含）：
+
+| 文件 | 大小 | 用途 |
+|------|------|------|
+| `MiniCPM-o-4_5-Q4_K_M.gguf` | ~12 GB | 主模型（文本推理） |
+| `vision/MiniCPM-o-4_5-vision-F16.gguf` | ~1 GB | 视觉投影层（可选，用于图片理解） |
+| `audio/MiniCPM-o-4_5-audio-F16.gguf` | ~630 MB | 音频投影层（当前未使用） |
+
+⚠️ **关键变更**：`--mmproj` 只传 vision 投影层，**不能**传入多个 .gguf 文件。旧代码使用 `find_mmproj_files()` 递归搜索导致传入 9 个文件而崩溃。
+
+### 上传步骤
 
 ```bash
-# 下载到本地
-python _download_models.py  # 在 PregoPal 根目录
+# 上传模型到 Modal Volume（只需一次）
+modal volume put minicpm-o-4_5-models /path/to/models/MiniCPM-o-4_5-gguf /
 
-# 上传到 Modal Volume
-modal volume put minicpm-o-4_5-models ../models/MiniCPM-o-4_5-gguf /
+# 验证
+modal volume ls minicpm-o-4_5-models /
 ```
 
-模型文件列表：
-| 文件 | 大小 | 用途 |
-|------|------|------|
-| `MiniCPM-o-4_5-Q4_K_M.gguf` | 688 MB | 主模型（文本推理） |
-| `vision/MiniCPM-o-4_5-vision-F16.gguf` | 1044 MB | 视觉投影层 |
-| `audio/MiniCPM-o-4_5-audio-F16.gguf` | 630 MB | 音频投影层 |
-| `tts/MiniCPM-o-4_5-tts-F16.gguf` | 1104 MB | TTS 语音合成 |
-| `tts/MiniCPM-o-4_5-projector-F16.gguf` | 14 MB | 投影层 |
-| `token2wav-gguf/*.gguf` (5 文件) | 875 MB | 语音合成 token→wav |
-
 ## 部署
 
 ```bash
@@ -85,11 +86,13 @@ Content-Type: application/json
 ```
 
 ### 4. 多模态对话（图片理解）
+llama-server (b4690+) 原生支持 OpenAI 多模态格式，通过 `/v1/chat/completions` 即可：
 ```bash
-POST /v1/multimodal/chat
+POST /v1/chat/completions
 Content-Type: application/json
 
 {
+  "model": "MiniCPM-o-4_5",
   "messages": [
     {"role": "user", "content": [
       {"type": "text", "text": "这张图片里有什么食物？"},
@@ -153,4 +156,5 @@ def ask_minicpm(prompt: str) -> str:
 1. **冷启动**：无请求 >5 分钟后容器关闭，下次请求需等待 ~30-60 秒（模型加载）
 2. **成本**：A100 约 $1.10/小时，按实际使用计费
 3. **模型上传**：Volume 是持久化的，模型只需上传一次
-4. **并发**：`allow_concurrent_inputs=10` 支持 10 个并发请求
+4. **并发**：`@modal.concurrent(max_inputs=10)` 支持 10 个并发请求
+5. **修复说明**：旧 deploy.py 的 `find_mmproj_files()` 递归搜索到 9 个 .gguf 文件并全部作为 `--mmproj` 传入，导致 llama-server 启动失败。新版已明确指定 `--mmproj` 使用 vision 投影层（仅一个文件）。

modal_deploy/client.py

@@ -145,7 +145,7 @@ class MiniCPMClient:
             "max_tokens": max_tokens or self.default_max_tokens,
             "temperature": temperature or self.default_temperature,
         }
-        return self._post("/v1/multimodal/chat", body)
+        return self._post("/v1/chat/completions", body)
 
     def describe_image(self, image_base64: str, image_format: str = "jpeg") -> str:
         """简化调用：描述图片内容"""
@@ -198,5 +198,5 @@ SYSTEM_PROMPTS = {
 2. 与孕期推荐标准的对比
 3. 需要补充或调整的方向
 4. 具体的一周饮食改善建议
-请用结构化格式输出，便于前端可视化""",
-}
+5. 用结构化格式输出，便于前端可视化""",
+}

modal_deploy/deploy.py

@@ -1,120 +1,132 @@
 """
 Modal deployment for MiniCPM-o-4_5 via llama.cpp server
-Serves OpenAI-compatible chat API + multimodal (text/image/audio) endpoints
+Serves OpenAI-compatible chat API + multimodal (vision) endpoints
+
+Usage:
+    # Deploy to Modal (builds image, uploads code, starts app)
+    modal deploy modal_deploy.deploy
+
+    # Test inference on Modal GPU
+    modal run modal_deploy.deploy::test_inference
+
+Architecture:
+    User Request → Modal ASGI (FastAPI) → llama-server (OpenAI-compatible)
+                                ↓
+                          Modal Volume (GGUF model persistent storage)
 """
 import os
 import sys
 import subprocess
 import modal
-from modal import Image, App, Volume, Secret, asgi_app
+from modal import Image, App, Volume, asgi_app
 
-# ── constants ──────────────────────────────────────────────
-MODEL_DIR = "/models"
-MODEL_SUBDIR = f"{MODEL_DIR}/MiniCPM-o-4_5-gguf"
-MODEL_NAME = "MiniCPM-o-4_5"
-# Model files expected in the volume
-MAIN_MODEL    = f"{MODEL_SUBDIR}/MiniCPM-o-4_5-Q4_K_M.gguf"
-# Projection files are auto-discovered by find_mmproj_files()
+# ═══════════════════════════════════════════════════════════════
+# 1. BASE IMAGE: llama.cpp compiled once, cached by Modal
+# ═══════════════════════════════════════════════════════════════
 
-# Image with llama.cpp compiled
-llamacpp_image = (
+_llamacpp_image = (
     Image.debian_slim(python_version="3.11")
-    .apt_install("curl", "git", "build-essential", "cmake", "libcurl4-openssl-dev",
-                  "software-properties-common", "wget")
+    .apt_install(
+        "curl", "git", "build-essential", "cmake",
+        "libcurl4-openssl-dev", "software-properties-common", "wget"
+    )
     .run_commands(
-        # Install CUDA toolkit (including nvcc + runtime libraries) for GPU inference on A100
+        # Install CUDA toolkit for GPU inference on A100
         "wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb",
         "dpkg -i cuda-keyring_1.1-1_all.deb",
         "apt-get update -qq",
         "apt-get install -y -qq cuda-toolkit-12-1 cuda-compiler-12-1 2>&1 | tail -5",
-        # Ensure CUDA runtime libs are discoverable at runtime
-        "echo '/usr/local/cuda-12.1/lib64' >> /etc/ld.so.conf.d/cuda.conf",
-        "ldconfig",
     )
-    .pip_install("fastapi", "uvicorn", "httpx", "numpy", "Pillow", "soundfile")
     .env({"CUDACXX": "/usr/local/cuda/bin/nvcc",
           "CUDA_HOME": "/usr/local/cuda-12.1",
           "LD_LIBRARY_PATH": "/usr/local/cuda-12.1/lib64"})
+    .run_commands(
+        "echo '/usr/local/cuda-12.1/lib64' >> /etc/ld.so.conf.d/cuda.conf",
+        "ldconfig",
+    )
+    .pip_install("fastapi", "uvicorn", "httpx", "numpy", "Pillow", "soundfile")
     .run_commands(
         "git clone --depth 1 https://github.com/ggerganov/llama.cpp /llama.cpp",
-        "cd /llama.cpp && cmake -B build -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON -DLLAMA_CURL=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF",
-        "cd /llama.cpp && cmake --build build --config Release -j $(nproc) --target llama-server llama-mtmd-cli",
+        # Build with CUDA - single job to prevent OOM on build server
+        # NOTE: -DCMAKE_CUDA_ARCHITECTURES=80-real = A100 only (faster build)
+        "cd /llama.cpp && cmake -B build -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON -DLLAMA_CURL=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF -DCMAKE_CUDA_ARCHITECTURES=80-real",
+        # Use -j1 to prevent OOM (each nvcc process ~1.5-2GB, build server ~8GB RAM)
+        "cd /llama.cpp && cmake --build build --config Release -j1 --target llama-server llama-mtmd-cli",
     )
 )
 
+# ═══════════════════════════════════════════════════════════════
+# 2. CONSTANTS
+# ═══════════════════════════════════════════════════════════════
+
+MODEL_DIR = "/models"
+MODEL_SUBDIR = f"{MODEL_DIR}/MiniCPM-o-4_5-gguf"
+MAIN_GGUF = "MiniCPM-o-4_5-Q4_K_M.gguf"
+VISION_MMPROJ = f"vision/MiniCPM-o-4_5-vision-F16.gguf"
+AUDIO_MMPROJ = f"audio/MiniCPM-o-4_5-audio-F16.gguf"
+
 # Volume to store models (persisted across restarts)
 model_volume = Volume.from_name("minicpm-o-4_5-models", create_if_missing=True)
 
+# Application
 app = App("prego-pal-minicpm")
 
-# ── Helpers ────────────────────────────────────────���────────
-
-def find_mmproj_files(model_dir: str) -> list[str]:
-    """Locate ALL mmproj GGUF files in model directory structure.
-    
-    Scans subdirectories (vision/, audio/, tts/) for .gguf files 
-    that are NOT the main language model.
-    """
-    mmproj = []
-    main_basename = os.path.basename(MAIN_MODEL)
-    for root, dirs, files in os.walk(model_dir):
-        for f in sorted(files):
-            if f.endswith(".gguf") and f != main_basename:
-                mmproj.append(os.path.join(root, f))
-    return mmproj
-
+# ═══════════════════════════════════════════════════════════════
+# 3. BUILD SERVER ARGS
+# ═══════════════════════════════════════════════════════════════
 
 def build_server_args(model_dir: str, port: int = 8080) -> list[str]:
     """Construct llama-server command line arguments.
     
-    All model paths are derived from the model_dir parameter,
-    not from hardcoded constants.
+    CRITICAL: --mmproj passes a *single* multimodal projection file to
+    enable vision support via /v1/chat/completions with image_url content.
+    
+    Do NOT pass audio/tts/token2wav .gguf files here - those are loaded
+    separately by llama-mtmd-cli for speech tasks only.
     """
-    main_model_path = os.path.join(model_dir, "MiniCPM-o-4_5-Q4_K_M.gguf")
+    main_model_path = os.path.join(model_dir, MAIN_GGUF)
+    vision_path = os.path.join(model_dir, VISION_MMPROJ)
+
     args = [
         "/llama.cpp/build/bin/llama-server",
         "-m", main_model_path,
         "--host", "0.0.0.0",
         "--port", str(port),
-        "-ngl", "99",            # offload all layers to GPU if available
-        "-c", "8192",             # context size
+        "-ngl", "99",            # offload all layers to GPU
+        "-c", "8192",            # context size
+        "--no-mmap",             # compatibility with Modal tmpfs
     ]
 
-    # Attach mmproj files for multimodal support (vision + audio + tts)
-    mmproj_files = find_mmproj_files(model_dir)
-    for mf in mmproj_files:
-        args.extend(["--mmproj", mf])
+    # Attach vision mmproj for multimodal support (image understanding)
+    if os.path.isfile(vision_path):
+        args.extend(["--mmproj", vision_path])
+        print(f"[PregoPal] Vision mmproj: {vision_path}")
+    else:
+        print(f"[PregoPal] WARNING: Vision mmproj not found at {vision_path}")
+        print(f"[PregoPal] Available files in {model_dir}:")
+        for root, dirs, files in os.walk(model_dir):
+            for f in files:
+                if f.endswith(".gguf"):
+                    print(f"  {os.path.join(root, f)}")
 
     return args
 
 
-def find_vision_mmproj(model_dir: str) -> str | None:
-    """Find the vision projection GGUF file."""
-    vision_dir = os.path.join(model_dir, "vision")
-    if not os.path.isdir(vision_dir):
-        return None
-    for f in sorted(os.listdir(vision_dir)):
-        if f.endswith(".gguf"):
-            return os.path.join(vision_dir, f)
-    return None
-
-
-# ── Entrypoint / API ────────────────────────────────────────
+# ═══════════════════════════════════════════════════════════════
+# 4. MAIN ASGI ENTRYPOINT
+# ═══════════════════════════════════════════════════════════════
 
 @app.function(
-    image=llamacpp_image,
+    image=_llamacpp_image,
     volumes={MODEL_DIR: model_volume},
     scaledown_window=300,                # shutdown after 5min idle
-    gpu="A100",                          # MiniCPM-o-4_5 needs GPU
-    timeout=600,
+    gpu="A100",                          # MiniCPM-o-4_5 needs A100 for 12GB model
+    timeout=1200,                        # 20 min timeout for cold start model loading
 )
+@modal.concurrent(max_inputs=10)
 @asgi_app()
 def serve():
-    """FastAPI app wrapping llama-server for OpenAI-compatible endpoints.
-    
-    NOTE: @modal.concurrent cannot be combined with @asgi_app().
-    Modal handles concurrency at the function level for ASGI apps automatically.
-    """
+    """FastAPI app wrapping llama-server for OpenAI-compatible endpoints."""
     import asyncio
     import httpx
     from fastapi import FastAPI, Request
@@ -145,21 +157,30 @@ def serve():
 
     llama_url = f"http://127.0.0.1:{server_port}"
 
-    # Wait for server to be ready
-    async def wait_for_server(timeout: float = 300.0):
-        async with httpx.AsyncClient() as client:
+    # Wait for server to be ready (long timeout for cold start)
+    async def wait_for_server(timeout: float = 900.0):
+        async with httpx.AsyncClient(timeout=30) as client:
             start = asyncio.get_event_loop().time()
             while True:
                 try:
-                    r = await client.get(f"{llama_url}/health", timeout=5)
+                    r = await client.get(f"{llama_url}/health", timeout=10)
                     if r.status_code == 200:
                         print("[PregoPal] llama-server ready!")
                         return
-                except Exception:
-                    pass
-                if asyncio.get_event_loop().time() - start > timeout:
-                    raise RuntimeError("llama-server startup timed out")
-                await asyncio.sleep(1)
+                except Exception as e:
+                    print(f"[PregoPal] Waiting for server... ({type(e).__name__})")
+                elapsed = asyncio.get_event_loop().time() - start
+                if elapsed > timeout:
+                    # Dump server logs on timeout to diagnose
+                    print("[PregoPal] TIMEOUT! Dumping server output:")
+                    if proc.stdout:
+                        try:
+                            output = proc.stdout.read(2048)
+                            print(output[-2048:])
+                        except Exception:
+                            pass
+                    raise RuntimeError(f"llama-server startup timed out after {timeout}s")
+                await asyncio.sleep(5)
 
     @web_app.on_event("startup")
     async def startup():
@@ -174,15 +195,22 @@ def serve():
         except subprocess.TimeoutExpired:
             proc.kill()
 
-    # ── Proxy endpoints to llama-server ──────────────────
+    # ═══════════════════════════════════════════════════
+    # 4a. PROXY ENDPOINTS
+    # ═══════════════════════════════════════════════════
 
     @web_app.post("/v1/chat/completions")
     async def chat_completions(request: Request):
-        """OpenAI-compatible chat completions (text)."""
+        """OpenAI-compatible chat completions.
+        
+        llama-server >= b4690 supports multimodal natively via /v1/chat/completions
+        with content parts including image_url (base64).
+        
+        This proxy passes requests directly through to llama-server.
+        """
         body = await request.json()
         async with httpx.AsyncClient(timeout=300) as client:
             r = await client.post(f"{llama_url}/v1/chat/completions", json=body)
-        # Stream if requested
         if body.get("stream", False):
             return StreamingResponse(
                 r.aiter_bytes(),
@@ -211,150 +239,32 @@ def serve():
             r = await client.get(f"{llama_url}/v1/models")
         return JSONResponse(r.json(), status_code=r.status_code)
 
-    # ── Multimodal endpoint ──────────────────────────────
-
-    @web_app.post("/v1/multimodal/chat")
-    async def multimodal_chat(request: Request):
-        """
-        Custom multimodal chat endpoint.
-        
-        NOTE: llama-server (>=b4690) natively supports multimodal via /v1/chat/completions
-        with content parts including image_url. Only use llama-mtmd-cli as fallback
-        when the direct API doesn't support the modality needed.
-        
-        For now, we proxy to llama-server directly and let it handle multimodal
-        if it's a new enough build. If that fails, fall back to llama-mtmd-cli.
-        
-        Request format:
-        {
-            "messages": [
-                {"role": "user", "content": [
-                    {"type": "text", "text": "Describe this image"},
-                    {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}
-                ]}
-            ],
-            "max_tokens": 1024,
-            "temperature": 0.7
-        }
-        """
-        import json as json_mod
-
-        body = await request.json()
-        
-        # First try: proxy directly to llama-server (it may support multimodal natively)
-        try:
-            async with httpx.AsyncClient(timeout=300) as client:
-                r = await client.post(f"{llama_url}/v1/chat/completions", json=body)
-            if r.status_code == 200:
-                if body.get("stream", False):
-                    return StreamingResponse(
-                        r.aiter_bytes(),
-                        media_type="text/event-stream",
-                        headers=dict(r.headers),
-                    )
-                return JSONResponse(r.json(), status_code=r.status_code)
-        except Exception:
-            pass
-        
-        # Fallback: use llama-mtmd-cli for multimodal if native path fails
-        import base64
-        import tempfile
-
-        messages = body.get("messages", [])
-        max_tokens = body.get("max_tokens", 1024)
-        temperature = body.get("temperature", 0.7)
-
-        # Extract text prompt and image
-        prompt_parts = []
-        image_path = None
-
-        for msg in messages:
-            if msg["role"] == "user":
-                content = msg["content"]
-                if isinstance(content, str):
-                    prompt_parts.append(content)
-                elif isinstance(content, list):
-                    for item in content:
-                        if item["type"] == "text":
-                            prompt_parts.append(item["text"])
-                        elif item["type"] == "image_url":
-                            data_url = item["image_url"]["url"]
-                            if data_url.startswith("data:image/"):
-                                # Parse base64 image
-                                header, b64 = data_url.split(",", 1)
-                                fmt = header.split("/")[1].split(";")[0]
-                                img_data = base64.b64decode(b64)
-                                fd, image_path = tempfile.mkstemp(suffix=f".{fmt}")
-                                os.close(fd)
-                                with open(image_path, "wb") as f:
-                                    f.write(img_data)
-
-        prompt = "\n".join(prompt_parts)
-
-        # Dynamically find vision mmproj
-        vision_mmproj = find_vision_mmproj(MODEL_SUBDIR)
-        if not vision_mmproj:
-            return JSONResponse(
-                {"error": "Vision projection model not found"}, status_code=500
-            )
-
-        # Build llama-mtmd-cli command (without --json-schema which may not exist)
-        cmd = [
-            "/llama.cpp/build/bin/llama-mtmd-cli",
-            "-m", os.path.join(MODEL_SUBDIR, "MiniCPM-o-4_5-Q4_K_M.gguf"),
-            "--mmproj", vision_mmproj,
-            "-ngl", "99",
-            "-c", "8192",
-            "-n", str(max_tokens),
-            "--temp", str(temperature),
-            "-p", prompt,
-        ]
-
-        if image_path:
-            cmd.extend(["--image", image_path])
-
-        try:
-            r_proc = await asyncio.create_subprocess_exec(
-                *cmd,
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-            )
-            stdout, stderr = await asyncio.wait_for(
-                r_proc.communicate(), timeout=120
-            )
-            if image_path:
-                os.unlink(image_path)
-
-            output = stdout.decode("utf-8", errors="replace").strip()
-            # Try to parse as JSON (llama-mtmd-cli may output raw text)
-            try:
-                result = json_mod.loads(output)
-                return JSONResponse(result)
-            except json_mod.JSONDecodeError:
-                return JSONResponse({"response": output, "raw": True})
-
-        except asyncio.TimeoutError:
-            if image_path and os.path.exists(image_path):
-                os.unlink(image_path)
-            return JSONResponse(
-                {"error": "Request timed out"}, status_code=504
-            )
-        except Exception as e:
-            if image_path and os.path.exists(image_path):
-                os.unlink(image_path)
-            return JSONResponse({"error": str(e)}, status_code=500)
+    # ═══════════════════════════════════════════════════
+    # 4b. HEALTH & INFO
+    # ═══════════════════════════════════════════════════
 
     @web_app.get("/")
     async def root():
-        return {"service": "PregoPal MiniCPM-o-4_5 API", "version": "1.0.0"}
+        return {
+            "service": "PregoPal MiniCPM-o-4_5 API",
+            "version": "1.0.0",
+            "endpoints": {
+                "chat": "/v1/chat/completions (POST) - OpenAI-compatible, supports text + image",
+                "embeddings": "/v1/embeddings (POST)",
+                "models": "/v1/models (GET)",
+                "health": "/health (GET)",
+            }
+        }
 
     return web_app
 
 
-# ── Deploy helpers ─────────────────────────────────────────
+# ═══════════════════════════════════════════════════════════════
+# 5. DEPLOY HELPERS
+# ═══════════════════════════════════════════════════════════════
 
 @app.function(
-    image=llamacpp_image,
+    image=_llamacpp_image,
     volumes={MODEL_DIR: model_volume},
     gpu="A100",
     timeout=3600,
@@ -365,22 +275,22 @@ def upload_models():
     Run: modal run modal_deploy/deploy.py::upload_models
     Requires models in ../models/MiniCPM-o-4_5-gguf/
     """
-    import shutil
-    local_dir = "/local_models"
-    
-    # This function assumes models are mounted locally
-    # On Modal, you'd use modal volume put instead
-    print("Model upload placeholder - use 'modal volume put' CLI")
-    print(f"  modal volume put minicpm-o-4_5-models ../models/MiniCPM-o-4_5-gguf /")
+    print("Upload models using CLI:")
+    print("  modal volume put minicpm-o-4_5-models ../models/MiniCPM-o-4_5-gguf /")
+    print()
+    print("Verify with:")
+    print("  modal volume ls minicpm-o-4_5-models /")
 
 
-# ── Health check function ───────────────────────────────────
+# ═══════════════════════════════════════════════════════════════
+# 6. TEST INFERENCE (runs on Modal GPU)
+# ═══════════════════════════════════════════════════════════════
 
 @app.function(
-    image=llamacpp_image,
+    image=_llamacpp_image,
     volumes={MODEL_DIR: model_volume},
     gpu="A100",
-    timeout=300,
+    timeout=600,
 )
 def test_inference():
     """Quick test to verify model loads and runs.
@@ -392,26 +302,29 @@ def test_inference():
 
     port = 8081
     args = build_server_args(MODEL_SUBDIR, port)
-    
+
     print(f"Starting server: {' '.join(args)}")
     proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    
+
     url = f"http://127.0.0.1:{port}"
-    deadline = time.time() + 120
+    deadline = time.time() + 180  # 3 min for model load
     while time.time() < deadline:
         try:
             r = httpx.get(f"{url}/health", timeout=5)
             if r.status_code == 200:
                 print("Server healthy!")
                 break
-        except Exception:
+        except Exception as e:
             pass
         time.sleep(2)
     else:
+        # Dump server output
+        if proc.stdout:
+            print("Server output:", proc.stdout.read(1024))
         proc.terminate()
         raise RuntimeError("Server failed to start")
 
-    # Test completion
+    # Test text completion
     r = httpx.post(
         f"{url}/v1/chat/completions",
         json={
@@ -424,5 +337,12 @@ def test_inference():
         },
         timeout=120,
     )
-    print(f"Response: {r.json()}")
-    proc.terminate()
+    print(f"Text response: {r.json()}")
+
+    # Test multimodal if vision mmproj exists
+    vision_path = os.path.join(MODEL_SUBDIR, VISION_MMPROJ)
+    if os.path.isfile(vision_path):
+        print("Vision mmproj found, skipping multimodal test (no test image available)")
+    
+    proc.terminate()
+    print("\n✅ Test passed!")