fix(deploy): 使用预编译CUDA wheel替代源码编译

.gitignore

@@ -38,4 +38,9 @@ debug*.txt
 # 检查脚本（一次性使用）
 _check_hf.py
 _download_models.py
-"cookbook_ref/"  
+"cookbook_ref/"
+
+# MiniCPM-V-Cookbook 参考仓库（仅供本地参考，不提交）
+MiniCPM-V-Cookbook/
+temp_minicpm/
+cookbook_ref/

modal_deploy/deploy.py

@@ -1,64 +1,55 @@
 """
-Modal deployment for MiniCPM-o-4_5 via llama-cpp-python (CUDA).
-Compiles with GGML_CUDA=ON during image build (~5min, within Modal's 15min limit).
-
-Usage:
-    modal deploy modal_deploy.deploy              # Deploy (~5min build)
-    modal run modal_deploy.deploy::test_inference  # Test
-
-Architecture:
-    User → FastAPI (ASGI, OpenAI-compatible) → llama-cpp-python (CUDA)
-                                                ⬇
-                                         Modal Volume (GGUF models)
+PregoPal × MiniCPM-o-4_5 — Modal 部署 (预编译 llama-cpp-python)
+
+架构:
+  FastAPI (ASGI) ←→ llama-cpp-python (CUDA via pre-built wheel)
+                         ↕
+                  Modal Volume: GGUF models
+
+用法:
+    pip install modal      # 安装 Modal CLI
+    modal token new        # 登录 Modal
+    modal deploy modal_deploy.deploy    # 部署 (2-3 min)
+
+测试:
+    modal run modal_deploy.deploy::test_inference
+
+API:
+    POST /v1/chat/completions   — OpenAI 兼容 (支持 streaming)
+    POST /v1/completions        — Text completion
+    POST /v1/embeddings         — Embeddings
+    POST /v1/vision             — 多模态 (图片+文字)
+    GET  /health                — 健康检查
+    GET  /v1/models             — 模型列表
 """
+
 import os
 import modal
 from modal import Image, App, Volume, asgi_app
 
-# ═══════════════════════════════════════════════════════════════
-# 1. BASE IMAGE
-# ═══════════════════════════════════════════════════════════════
-# Compile llama-cpp-python with CUDA support at image build time.
-# Modal provides 15min build timeout — sufficient for CUDA compilation.
-# ═══════════════════════════════════════════════════════════════
+# ════════════════════════════════════════════════════════════════════
+# 1. IMAGE — 预编译 CUDA wheel (不从头编译)
+# ════════════════════════════════════════════════════════════════════
 
 _image = (
     Image.debian_slim(python_version="3.11")
-    .apt_install("build-essential", "cmake", "curl", "wget", "git",
-                 "libcurl4-openssl-dev")
-    # Install CUDA toolkit (needed for llama.cpp CUDA compilation)
-    .run_commands(
-        "wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb",
-        "dpkg -i cuda-keyring_1.1-1_all.deb",
-        "apt-get update -qq",
-        "apt-get install -y -qq cuda-compiler-12-1 cuda-cudart-dev-12-1 2>&1 | tail -3",
-    )
-    .env({
-        "CUDA_HOME": "/usr/local/cuda-12.1",
-        "PATH": "/usr/local/cuda-12.1/bin:${PATH}",
-        "LD_LIBRARY_PATH": "/usr/local/cuda-12.1/lib64:${LD_LIBRARY_PATH}",
-        "CMAKE_ARGS": "-DGGML_CUDA=ON -DGGML_CUDA_ARCHS=sm_80",
-        "FORCE_CMAKE": "1",
-    })
+    # → 只安装 Python 依赖，不安装 cmake/gcc/CUDA toolkit
+    .pip_install("fastapi", "uvicorn[standard]", "httpx", "numpy", "Pillow")
+    # → 从 ggml-org 官方索引安装预编译 CUDA wheel（几秒完成）
     .pip_install(
-        "fastapi",
-        "uvicorn[standard]",
-        "httpx",
-        "numpy",
-        "Pillow",
-        # Install llama-cpp-python with CUDA (compiles at build time)
         "llama-cpp-python",
-        extra_args=["--force-reinstall", "--no-cache-dir"],
+        extra_index_url="https://ggml-org.github.io/llama-cpp-python/whl/cu121",
+        force_build=True,
     )
+    # → 验证安装
     .run_commands(
-        # Verify CUDA is available
-        "python -c 'from llama_cpp import Llama; print(f\"CUDA available: {Llama.supports_gpu()}\")'",
+        "python -c 'from llama_cpp import Llama; print(f\"llama-cpp OK, GPU: {Llama.supports_gpu()}\")'",
     )
 )
 
-# ═══════════════════════════════════════════════════════════════
+# ════════════════════════════════════════════════════════════════════
 # 2. CONSTANTS
-# ═══════════════════════════════════════════════════════════════
+# ════════════════════════════════════════════════════════════════════
 
 MODEL_DIR = "/models"
 MODEL_SUBDIR = f"{MODEL_DIR}/MiniCPM-o-4_5-gguf"
@@ -68,12 +59,9 @@ VISION_MMPROJ = "vision/MiniCPM-o-4_5-vision-F16.gguf"
 model_volume = Volume.from_name("minicpm-o-4_5-models", create_if_missing=True)
 app = App("prego-pal-minicpm")
 
-# ═══════════════════════════════════════════════════════════════
-# 3. MODEL HELPER
-# ═══════════════════════════════════════════════════════════════
 
 def get_model_paths(base_dir: str) -> dict:
-    """Return validated model paths."""
+    """返回经验证的模型路径."""
     main_path = os.path.join(base_dir, MAIN_GGUF)
     vision_path = os.path.join(base_dir, VISION_MMPROJ)
     paths = {"main": main_path, "vision": vision_path}
@@ -81,24 +69,27 @@ def get_model_paths(base_dir: str) -> dict:
         print(f"[PregoPal] {key}: {path} (exists={os.path.isfile(path)})")
     return paths
 
-# ═══════════════════════════════════════════════════════════════
-# 4. ASGI APP (OpenAI-compatible API)
-# ═══════════════════════════════════════════════════════════════
+
+# ════════════════════════════════════════════════════════════════════
+# 3. ASGI APP  — 多模态 API
+# ════════════════════════════════════════════════════════════════════
 
 @app.function(
     image=_image,
     volumes={MODEL_DIR: model_volume},
-    scaledown_window=300,      # Keep warm 5min after last request
-    gpu="A100",                 # Use A100 GPU via Modal
-    timeout=1200,               # Max function execution time
-    allow_concurrent_inputs=10, # Up to 10 parallel requests
-    container_idle_timeout=60,  # Keep container 60s between requests
+    scaledown_window=300,
+    gpu="A100",
+    timeout=1200,
+    container_idle_timeout=300,
+    allow_concurrent_inputs=10,
 )
 @asgi_app()
 def serve():
     import asyncio
     import json
     import logging
+    import base64
+    from io import BytesIO
     from pathlib import Path
     from fastapi import FastAPI, Request
     from fastapi.responses import StreamingResponse, JSONResponse
@@ -117,27 +108,68 @@ def serve():
         allow_headers=["*"],
     )
 
-    # ── Model Loading ──
+    # ── Model Loading ──────────────────────────────────────────────
     paths = get_model_paths(MODEL_SUBDIR)
     model_path = paths["main"]
     vision_path = paths["vision"]
 
     kwargs: dict = dict(
         model_path=model_path,
-        n_gpu_layers=-1,                     # Offload ALL layers to GPU
-        n_ctx=8192,                          # Context window
-        verbose=False,                        # Keep logs clean
+        n_gpu_layers=-1,
+        n_ctx=8192,
+        verbose=False,
         n_threads=os.cpu_count() or 4,
     )
     if os.path.isfile(vision_path):
         kwargs["mmproj"] = vision_path
-        logger.info("[PregoPal] Vision mmproj enabled")
-
-    logger.info("[PregoPal] Loading model (this may take 30-90s)...")
-    llm = Llama(**kwargs)
-    logger.info("[PregoPal] Model loaded! Ready for inference.")
-
-    # ── Endpoints ──
+        logger.info("[PregoPal] ✅ Vision mmproj enabled")
+    else:
+        logger.warning(f"[PregoPal] ⚠️  mmproj not found at {vision_path} — vision disabled")
+
+    logger.info("[PregoPal] Loading model (30-90s)...")
+    try:
+        llm = Llama(**kwargs)
+        logger.info("[PregoPal] ✅ Model loaded!")
+    except Exception as e:
+        logger.error(f"[PregoPal] ❌ Failed to load model: {e}")
+        raise
+
+    # ── Helpers ────────────────────────────────────────────────────
+
+    def _parse_messages(messages: list) -> str:
+        """Convert messages list to a prompt string."""
+        texts = []
+        for msg in messages:
+            role = msg.get("role", "user")
+            content = msg.get("content", "")
+            if isinstance(content, list):
+                parts = []
+                for part in content:
+                    if isinstance(part, dict):
+                        if part.get("type") == "text":
+                            parts.append(part.get("text", ""))
+                        elif part.get("type") == "image_url":
+                            parts.append("[IMAGE]")
+                    else:
+                        parts.append(str(part))
+                content = " ".join(parts)
+            texts.append(f"<|{role}|>\n{content}\n<|assistant|>\n")
+        return "".join(texts)
+
+    def _extract_image(messages: list) -> bytes | None:
+        """Extract the first base64 image from messages."""
+        for msg in messages:
+            content = msg.get("content", "")
+            if isinstance(content, list):
+                for part in content:
+                    if isinstance(part, dict) and part.get("type") == "image_url":
+                        url = part.get("image_url", {}).get("url", "")
+                        if url.startswith("data:image"):
+                            _, b64 = url.split(",", 1)
+                            return base64.b64decode(b64)
+        return None
+
+    # ── Endpoints ──────────────────────────────────────────────────
 
     @web_app.post("/v1/chat/completions")
     async def chat_completions(request: Request):
@@ -147,7 +179,6 @@ def serve():
         max_tokens = body.get("max_tokens", 512)
         temperature = body.get("temperature", 0.7)
         top_p = body.get("top_p", 0.9)
-        model = body.get("model", "MiniCPM-o-4_5")
 
         if stream:
             async def event_stream():
@@ -176,12 +207,11 @@ def serve():
         body = await request.json()
         prompt = body.get("prompt", "")
         max_tokens = body.get("max_tokens", 256)
-        temperature = body.get("temperature", 0.7)
 
         result = llm.create_completion(
             prompt=prompt,
             max_tokens=max_tokens,
-            temperature=temperature,
+            temperature=body.get("temperature", 0.7),
             stream=False,
         )
         return JSONResponse(result)
@@ -195,13 +225,39 @@ def serve():
         )
         return JSONResponse(result)
 
+    @web_app.post("/v1/vision")
+    async def vision(request: Request):
+        """
+        多模态推理：接收图片（base64）和文本提示。
+        如果 llm 未加载 mmproj，返回 400。
+        """
+        body = await request.json()
+        messages = body.get("messages", [])
+        max_tokens = body.get("max_tokens", 512)
+        temperature = body.get("temperature", 0.7)
+
+        if not os.path.isfile(vision_path):
+            return JSONResponse(
+                {"error": "Vision mmproj not loaded — deploy the model with mmproj file"},
+                status_code=400,
+            )
+
+        # llama-cpp-python 的 create_chat_completion 原生支持多模态
+        result = llm.create_chat_completion(
+            messages=messages,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            stream=False,
+        )
+        return JSONResponse(result)
+
     @web_app.get("/health")
     async def health():
-        # Quick check: model loaded
         return {
             "status": "ok",
             "model": "MiniCPM-o-4_5",
             "cuda": True,
+            "vision": os.path.isfile(vision_path),
         }
 
     @web_app.get("/v1/models")
@@ -213,30 +269,31 @@ def serve():
                 "object": "model",
                 "created": 1,
                 "owned_by": "prego-pal",
-                "permission": [],
             }],
         }
 
     @web_app.get("/")
     async def root():
         return {
-            "service": "PregoPal MiniCPM-o-4_5 API (CUDA)",
-            "version": "1.2.0",
-            "model": "MiniCPM-o-4_5-Q4_K_M",
+            "service": "PregoPal MiniCPM-o-4_5 API",
+            "version": "2.0.0",
+            "model": MAIN_GGUF,
             "endpoints": {
-                "chat": "/v1/chat/completions (POST, streaming+non-streaming)",
-                "completions": "/v1/completions (POST)",
-                "embeddings": "/v1/embeddings (POST)",
-                "models": "/v1/models (GET)",
-                "health": "/health (GET)",
-            }
+                "chat": "POST /v1/chat/completions",
+                "completions": "POST /v1/completions",
+                "embeddings": "POST /v1/embeddings",
+                "vision": "POST /v1/vision (多模态)",
+                "models": "GET /v1/models",
+                "health": "GET /health",
+            },
         }
 
     return web_app
 
-# ═══════════════════════════════════════════════════════════════
-# 5. MODEL UPLOAD HELPER
-# ═══════════════════════════════════════════════════════════════
+
+# ═══════════════════════════════════════════���════════════════════════
+# 4. MODEL UPLOAD 指引
+# ════════════════════════════════════════════════════════════════════
 
 @app.function(
     image=_image,
@@ -244,21 +301,25 @@ def serve():
     timeout=3600,
 )
 def upload_models():
-    """Print instructions for uploading models to Modal Volume."""
+    """打印上传模型指引."""
     print("=" * 60)
-    print("Upload model files to Modal Volume:")
+    print("📦 上传模型至 Modal Volume:")
     print()
     print("  # From your local models directory:")
     print("  modal volume put minicpm-o-4_5-models \\")
-    print("      ../models/MiniCPM-o-4_5-gguf /")
+    print("      ./models/MiniCPM-o-4_5-gguf /MiniCPM-o-4_5-gguf")
     print()
     print("  # Verify:")
     print("  modal volume ls minicpm-o-4_5-models /")
+    print(f"  # Expected files:")
+    print(f"  #   {MAIN_GGUF}")
+    print(f"  #   {VISION_MMPROJ}")
     print("=" * 60)
 
-# ═══════════════════════════════════════════════════════════════
-# 6. TEST INFERENCE (One-shot, not as ASGI)
-# ═══════════════════════════════════════════════════════════════
+
+# ════════════════════════════════════════════════════════════════════
+# 5. TEST INFERENCE
+# ════════════════════════════════════════════════════════════════════
 
 @app.function(
     image=_image,
@@ -267,32 +328,28 @@ def upload_models():
     timeout=600,
 )
 def test_inference():
-    """Test inference on Modal (downloads model from Volume)."""
+    """在 Modal 上测试推理."""
     import time
     import json
     from llama_cpp import Llama
 
     print("[PregoPal] ========== TEST INFERENCE ==========")
-    print(f"[PregoPal] Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S')}")
     print(f"[PregoPal] GPU: A100 (via Modal)")
-    print(f"[PregoPal] Model base: {MODEL_SUBDIR}")
 
     paths = get_model_paths(MODEL_SUBDIR)
     main_path = paths["main"]
     vision_path = paths["vision"]
 
     if not os.path.isfile(main_path):
-        print(f"[PregoPal] ERROR: Model not found at {main_path}")
-        print("[PregoPal] Upload models first: modal run modal_deploy.deploy::upload_models")
+        print(f"[PregoPal] ❌ Model not found at {main_path}")
         return
 
-    # Load model
     t0 = time.time()
     kwargs = dict(
         model_path=main_path,
         n_gpu_layers=-1,
         n_ctx=4096,
-        verbose=True,
+        verbose=False,
     )
     if os.path.isfile(vision_path):
         kwargs["mmproj"] = vision_path
@@ -300,9 +357,9 @@ def test_inference():
     print("[PregoPal] Loading model...")
     llm = Llama(**kwargs)
     load_time = time.time() - t0
-    print(f"[PregoPal] Model loaded in {load_time:.1f}s")
+    print(f"[PregoPal] ✅ Model loaded in {load_time:.1f}s")
 
-    # ── Test 1: Text generation ──
+    # Test 1
     print("\n[Test 1] Chinese greeting...")
     t0 = time.time()
     result = llm.create_chat_completion(
@@ -314,7 +371,7 @@ def test_inference():
     content = result.get("choices", [{}])[0].get("message", {}).get("content", "")
     print(f"Response ({elapsed:.1f}s): {content}")
 
-    # ── Test 2: English text ──
+    # Test 2
     print("\n[Test 2] English instruction...")
     t0 = time.time()
     result = llm.create_chat_completion(
@@ -328,5 +385,4 @@ def test_inference():
 
     print(f"\n{'='*50}")
     print(f"✅ Test complete! Loading: {load_time:.1f}s")
-    print(f"✅ Inference speed: {elapsed:.1f}s per response (CUDA)")
     print(f"{'='*50}")