重构modal部署使用CUDA编译, streaming修复, cookbook_ref gitignored

.gitignore

@@ -38,3 +38,4 @@ debug*.txt
 # 检查脚本（一次性使用）
 _check_hf.py
 _download_models.py
+"cookbook_ref/"  

modal_deploy/deploy.py

@@ -1,57 +1,58 @@
 """
-Modal deployment for MiniCPM-o-4_5 via llama.cpp server
-Serves OpenAI-compatible chat API + multimodal (vision) endpoints
+Modal deployment for MiniCPM-o-4_5 via llama-cpp-python (CUDA).
+Compiles with GGML_CUDA=ON during image build (~5min, within Modal's 15min limit).
 
 Usage:
-    # Deploy to Modal (builds image, uploads code, starts app)
-    modal deploy modal_deploy.deploy
-
-    # Test inference on Modal GPU
-    modal run modal_deploy.deploy::test_inference
+    modal deploy modal_deploy.deploy              # Deploy (~5min build)
+    modal run modal_deploy.deploy::test_inference  # Test
 
 Architecture:
-    User Request → Modal ASGI (FastAPI) → llama-server (OpenAI-compatible)
-                                ↓
-                          Modal Volume (GGUF model persistent storage)
+    User → FastAPI (ASGI, OpenAI-compatible) → llama-cpp-python (CUDA)
+                                                ⬇
+                                         Modal Volume (GGUF models)
 """
 import os
-import sys
-import subprocess
 import modal
 from modal import Image, App, Volume, asgi_app
 
 # ═══════════════════════════════════════════════════════════════
-# 1. BASE IMAGE: llama.cpp compiled once, cached by Modal
+# 1. BASE IMAGE
+# ═══════════════════════════════════════════════════════════════
+# Compile llama-cpp-python with CUDA support at image build time.
+# Modal provides 15min build timeout — sufficient for CUDA compilation.
 # ═══════════════════════════════════════════════════════════════
 
-_llamacpp_image = (
+_image = (
     Image.debian_slim(python_version="3.11")
-    .apt_install(
-        "curl", "git", "build-essential", "cmake",
-        "libcurl4-openssl-dev", "software-properties-common", "wget"
-    )
+    .apt_install("build-essential", "cmake", "curl", "wget", "git",
+                 "libcurl4-openssl-dev")
+    # Install CUDA toolkit (needed for llama.cpp CUDA compilation)
     .run_commands(
-        # Install CUDA toolkit for GPU inference on A100
         "wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb",
         "dpkg -i cuda-keyring_1.1-1_all.deb",
         "apt-get update -qq",
-        "apt-get install -y -qq cuda-toolkit-12-1 cuda-compiler-12-1 2>&1 | tail -5",
+        "apt-get install -y -qq cuda-compiler-12-1 cuda-cudart-dev-12-1 2>&1 | tail -3",
     )
-    .env({"CUDACXX": "/usr/local/cuda/bin/nvcc",
-          "CUDA_HOME": "/usr/local/cuda-12.1",
-          "LD_LIBRARY_PATH": "/usr/local/cuda-12.1/lib64"})
-    .run_commands(
-        "echo '/usr/local/cuda-12.1/lib64' >> /etc/ld.so.conf.d/cuda.conf",
-        "ldconfig",
+    .env({
+        "CUDA_HOME": "/usr/local/cuda-12.1",
+        "PATH": "/usr/local/cuda-12.1/bin:${PATH}",
+        "LD_LIBRARY_PATH": "/usr/local/cuda-12.1/lib64:${LD_LIBRARY_PATH}",
+        "CMAKE_ARGS": "-DGGML_CUDA=ON -DGGML_CUDA_ARCHS=sm_80",
+        "FORCE_CMAKE": "1",
+    })
+    .pip_install(
+        "fastapi",
+        "uvicorn[standard]",
+        "httpx",
+        "numpy",
+        "Pillow",
+        # Install llama-cpp-python with CUDA (compiles at build time)
+        "llama-cpp-python",
+        extra_args=["--force-reinstall", "--no-cache-dir"],
     )
-    .pip_install("fastapi", "uvicorn", "httpx", "numpy", "Pillow", "soundfile")
     .run_commands(
-        "git clone --depth 1 https://github.com/ggerganov/llama.cpp /llama.cpp",
-        # Build with CUDA - single job to prevent OOM on build server
-        # NOTE: -DCMAKE_CUDA_ARCHITECTURES=80-real = A100 only (faster build)
-        "cd /llama.cpp && cmake -B build -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON -DLLAMA_CURL=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF -DCMAKE_CUDA_ARCHITECTURES=80-real",
-        # Use -j1 to prevent OOM (each nvcc process ~1.5-2GB, build server ~8GB RAM)
-        "cd /llama.cpp && cmake --build build --config Release -j1 --target llama-server llama-mtmd-cli",
+        # Verify CUDA is available
+        "python -c 'from llama_cpp import Llama; print(f\"CUDA available: {Llama.supports_gpu()}\")'",
     )
 )
 
@@ -62,79 +63,52 @@ _llamacpp_image = (
 MODEL_DIR = "/models"
 MODEL_SUBDIR = f"{MODEL_DIR}/MiniCPM-o-4_5-gguf"
 MAIN_GGUF = "MiniCPM-o-4_5-Q4_K_M.gguf"
-VISION_MMPROJ = f"vision/MiniCPM-o-4_5-vision-F16.gguf"
-AUDIO_MMPROJ = f"audio/MiniCPM-o-4_5-audio-F16.gguf"
+VISION_MMPROJ = "vision/MiniCPM-o-4_5-vision-F16.gguf"
 
-# Volume to store models (persisted across restarts)
 model_volume = Volume.from_name("minicpm-o-4_5-models", create_if_missing=True)
-
-# Application
 app = App("prego-pal-minicpm")
 
 # ═══════════════════════════════════════════════════════════════
-# 3. BUILD SERVER ARGS
+# 3. MODEL HELPER
 # ═══════════════════════════════════════════════════════════════
 
-def build_server_args(model_dir: str, port: int = 8080) -> list[str]:
-    """Construct llama-server command line arguments.
-    
-    CRITICAL: --mmproj passes a *single* multimodal projection file to
-    enable vision support via /v1/chat/completions with image_url content.
-    
-    Do NOT pass audio/tts/token2wav .gguf files here - those are loaded
-    separately by llama-mtmd-cli for speech tasks only.
-    """
-    main_model_path = os.path.join(model_dir, MAIN_GGUF)
-    vision_path = os.path.join(model_dir, VISION_MMPROJ)
-
-    args = [
-        "/llama.cpp/build/bin/llama-server",
-        "-m", main_model_path,
-        "--host", "0.0.0.0",
-        "--port", str(port),
-        "-ngl", "99",            # offload all layers to GPU
-        "-c", "8192",            # context size
-        "--no-mmap",             # compatibility with Modal tmpfs
-    ]
-
-    # Attach vision mmproj for multimodal support (image understanding)
-    if os.path.isfile(vision_path):
-        args.extend(["--mmproj", vision_path])
-        print(f"[PregoPal] Vision mmproj: {vision_path}")
-    else:
-        print(f"[PregoPal] WARNING: Vision mmproj not found at {vision_path}")
-        print(f"[PregoPal] Available files in {model_dir}:")
-        for root, dirs, files in os.walk(model_dir):
-            for f in files:
-                if f.endswith(".gguf"):
-                    print(f"  {os.path.join(root, f)}")
-
-    return args
-
+def get_model_paths(base_dir: str) -> dict:
+    """Return validated model paths."""
+    main_path = os.path.join(base_dir, MAIN_GGUF)
+    vision_path = os.path.join(base_dir, VISION_MMPROJ)
+    paths = {"main": main_path, "vision": vision_path}
+    for key, path in paths.items():
+        print(f"[PregoPal] {key}: {path} (exists={os.path.isfile(path)})")
+    return paths
 
 # ═══════════════════════════════════════════════════════════════
-# 4. MAIN ASGI ENTRYPOINT
+# 4. ASGI APP (OpenAI-compatible API)
 # ═══════════════════════════════════════════════════════════════
 
 @app.function(
-    image=_llamacpp_image,
+    image=_image,
     volumes={MODEL_DIR: model_volume},
-    scaledown_window=300,                # shutdown after 5min idle
-    gpu="A100",                          # MiniCPM-o-4_5 needs A100 for 12GB model
-    timeout=1200,                        # 20 min timeout for cold start model loading
+    scaledown_window=300,      # Keep warm 5min after last request
+    gpu="A100",                 # Use A100 GPU via Modal
+    timeout=1200,               # Max function execution time
+    allow_concurrent_inputs=10, # Up to 10 parallel requests
+    container_idle_timeout=60,  # Keep container 60s between requests
 )
-@modal.concurrent(max_inputs=10)
 @asgi_app()
 def serve():
-    """FastAPI app wrapping llama-server for OpenAI-compatible endpoints."""
     import asyncio
-    import httpx
+    import json
+    import logging
+    from pathlib import Path
     from fastapi import FastAPI, Request
     from fastapi.responses import StreamingResponse, JSONResponse
     from fastapi.middleware.cors import CORSMiddleware
+    from llama_cpp import Llama
 
-    web_app = FastAPI(title="PregoPal MiniCPM-o API")
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger("prego-pal")
 
+    web_app = FastAPI(title="PregoPal MiniCPM-o-4_5 API")
     web_app.add_middleware(
         CORSMiddleware,
         allow_origins=["*"],
@@ -143,113 +117,115 @@ def serve():
         allow_headers=["*"],
     )
 
-    # Start llama-server in background
-    server_port = 8080
-    server_args = build_server_args(MODEL_SUBDIR, server_port)
-
-    print(f"[PregoPal] Starting llama-server: {' '.join(server_args)}")
-    proc = subprocess.Popen(
-        server_args,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        text=True,
+    # ── Model Loading ──
+    paths = get_model_paths(MODEL_SUBDIR)
+    model_path = paths["main"]
+    vision_path = paths["vision"]
+
+    kwargs: dict = dict(
+        model_path=model_path,
+        n_gpu_layers=-1,                     # Offload ALL layers to GPU
+        n_ctx=8192,                          # Context window
+        verbose=False,                        # Keep logs clean
+        n_threads=os.cpu_count() or 4,
     )
+    if os.path.isfile(vision_path):
+        kwargs["mmproj"] = vision_path
+        logger.info("[PregoPal] Vision mmproj enabled")
+
+    logger.info("[PregoPal] Loading model (this may take 30-90s)...")
+    llm = Llama(**kwargs)
+    logger.info("[PregoPal] Model loaded! Ready for inference.")
 
-    llama_url = f"http://127.0.0.1:{server_port}"
-
-    # Wait for server to be ready (long timeout for cold start)
-    async def wait_for_server(timeout: float = 900.0):
-        async with httpx.AsyncClient(timeout=30) as client:
-            start = asyncio.get_event_loop().time()
-            while True:
-                try:
-                    r = await client.get(f"{llama_url}/health", timeout=10)
-                    if r.status_code == 200:
-                        print("[PregoPal] llama-server ready!")
-                        return
-                except Exception as e:
-                    print(f"[PregoPal] Waiting for server... ({type(e).__name__})")
-                elapsed = asyncio.get_event_loop().time() - start
-                if elapsed > timeout:
-                    # Dump server logs on timeout to diagnose
-                    print("[PregoPal] TIMEOUT! Dumping server output:")
-                    if proc.stdout:
-                        try:
-                            output = proc.stdout.read(2048)
-                            print(output[-2048:])
-                        except Exception:
-                            pass
-                    raise RuntimeError(f"llama-server startup timed out after {timeout}s")
-                await asyncio.sleep(5)
-
-    @web_app.on_event("startup")
-    async def startup():
-        # Wait longer for cold start (loading 12GB model over Volume mount)
-        await wait_for_server()
-
-    @web_app.on_event("shutdown")
-    async def shutdown():
-        proc.terminate()
-        try:
-            proc.wait(timeout=10)
-        except subprocess.TimeoutExpired:
-            proc.kill()
-
-    # ═══════════════════════════════════════════════════
-    # 4a. PROXY ENDPOINTS
-    # ═══════════════════════════════════════════════════
+    # ── Endpoints ──
 
     @web_app.post("/v1/chat/completions")
     async def chat_completions(request: Request):
-        """OpenAI-compatible chat completions.
-        
-        llama-server >= b4690 supports multimodal natively via /v1/chat/completions
-        with content parts including image_url (base64).
-        
-        This proxy passes requests directly through to llama-server.
-        """
         body = await request.json()
-        async with httpx.AsyncClient(timeout=300) as client:
-            r = await client.post(f"{llama_url}/v1/chat/completions", json=body)
-        if body.get("stream", False):
-            return StreamingResponse(
-                r.aiter_bytes(),
-                media_type="text/event-stream",
-                headers=dict(r.headers),
-            )
-        return JSONResponse(r.json(), status_code=r.status_code)
+        stream = body.get("stream", False)
+        messages = body.get("messages", [])
+        max_tokens = body.get("max_tokens", 512)
+        temperature = body.get("temperature", 0.7)
+        top_p = body.get("top_p", 0.9)
+        model = body.get("model", "MiniCPM-o-4_5")
+
+        if stream:
+            async def event_stream():
+                for chunk in llm.create_chat_completion(
+                    messages=messages,
+                    max_tokens=max_tokens,
+                    temperature=temperature,
+                    top_p=top_p,
+                    stream=True,
+                ):
+                    yield f"data: {json.dumps(chunk)}\n\n"
+                yield "data: [DONE]\n\n"
+            return StreamingResponse(event_stream(), media_type="text/event-stream")
+
+        result = llm.create_chat_completion(
+            messages=messages,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            stream=False,
+        )
+        return JSONResponse(result)
+
+    @web_app.post("/v1/completions")
+    async def completions(request: Request):
+        body = await request.json()
+        prompt = body.get("prompt", "")
+        max_tokens = body.get("max_tokens", 256)
+        temperature = body.get("temperature", 0.7)
+
+        result = llm.create_completion(
+            prompt=prompt,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            stream=False,
+        )
+        return JSONResponse(result)
 
     @web_app.post("/v1/embeddings")
     async def embeddings(request: Request):
-        """OpenAI-compatible embeddings."""
         body = await request.json()
-        async with httpx.AsyncClient(timeout=120) as client:
-            r = await client.post(f"{llama_url}/v1/embeddings", json=body)
-        return JSONResponse(r.json(), status_code=r.status_code)
+        result = llm.create_embedding(
+            input=body.get("input", ""),
+            model=body.get("model", "MiniCPM-o-4_5"),
+        )
+        return JSONResponse(result)
 
     @web_app.get("/health")
     async def health():
-        async with httpx.AsyncClient() as client:
-            r = await client.get(f"{llama_url}/health")
-        return JSONResponse(r.json(), status_code=r.status_code)
+        # Quick check: model loaded
+        return {
+            "status": "ok",
+            "model": "MiniCPM-o-4_5",
+            "cuda": True,
+        }
 
     @web_app.get("/v1/models")
     async def list_models():
-        async with httpx.AsyncClient() as client:
-            r = await client.get(f"{llama_url}/v1/models")
-        return JSONResponse(r.json(), status_code=r.status_code)
-
-    # ═══════════════════════════════════════════════════
-    # 4b. HEALTH & INFO
-    # ═══════════════════════════════════════════════════
+        return {
+            "object": "list",
+            "data": [{
+                "id": "MiniCPM-o-4_5",
+                "object": "model",
+                "created": 1,
+                "owned_by": "prego-pal",
+                "permission": [],
+            }],
+        }
 
     @web_app.get("/")
     async def root():
         return {
-            "service": "PregoPal MiniCPM-o-4_5 API",
-            "version": "1.0.0",
+            "service": "PregoPal MiniCPM-o-4_5 API (CUDA)",
+            "version": "1.2.0",
+            "model": "MiniCPM-o-4_5-Q4_K_M",
             "endpoints": {
-                "chat": "/v1/chat/completions (POST) - OpenAI-compatible, supports text + image",
+                "chat": "/v1/chat/completions (POST, streaming+non-streaming)",
+                "completions": "/v1/completions (POST)",
                 "embeddings": "/v1/embeddings (POST)",
                 "models": "/v1/models (GET)",
                 "health": "/health (GET)",
@@ -258,91 +234,99 @@ def serve():
 
     return web_app
 
-
 # ═══════════════════════════════════════════════════════════════
-# 5. DEPLOY HELPERS
+# 5. MODEL UPLOAD HELPER
 # ═══════════════════════════════════════════════════════════════
 
 @app.function(
-    image=_llamacpp_image,
+    image=_image,
     volumes={MODEL_DIR: model_volume},
-    gpu="A100",
     timeout=3600,
 )
 def upload_models():
-    """One-time function to upload model files to Modal Volume.
-    
-    Run: modal run modal_deploy/deploy.py::upload_models
-    Requires models in ../models/MiniCPM-o-4_5-gguf/
-    """
-    print("Upload models using CLI:")
-    print("  modal volume put minicpm-o-4_5-models ../models/MiniCPM-o-4_5-gguf /")
+    """Print instructions for uploading models to Modal Volume."""
+    print("=" * 60)
+    print("Upload model files to Modal Volume:")
+    print()
+    print("  # From your local models directory:")
+    print("  modal volume put minicpm-o-4_5-models \\")
+    print("      ../models/MiniCPM-o-4_5-gguf /")
     print()
-    print("Verify with:")
+    print("  # Verify:")
     print("  modal volume ls minicpm-o-4_5-models /")
-
+    print("=" * 60)
 
 # ═══════════════════════════════════════════════════════════════
-# 6. TEST INFERENCE (runs on Modal GPU)
+# 6. TEST INFERENCE (One-shot, not as ASGI)
 # ═══════════════════════════════════════════════════════════════
 
 @app.function(
-    image=_llamacpp_image,
+    image=_image,
     volumes={MODEL_DIR: model_volume},
     gpu="A100",
     timeout=600,
 )
 def test_inference():
-    """Quick test to verify model loads and runs.
-    
-    Run: modal run modal_deploy/deploy.py::test_inference
-    """
-    import httpx
+    """Test inference on Modal (downloads model from Volume)."""
     import time
-
-    port = 8081
-    args = build_server_args(MODEL_SUBDIR, port)
-
-    print(f"Starting server: {' '.join(args)}")
-    proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
-    url = f"http://127.0.0.1:{port}"
-    deadline = time.time() + 180  # 3 min for model load
-    while time.time() < deadline:
-        try:
-            r = httpx.get(f"{url}/health", timeout=5)
-            if r.status_code == 200:
-                print("Server healthy!")
-                break
-        except Exception as e:
-            pass
-        time.sleep(2)
-    else:
-        # Dump server output
-        if proc.stdout:
-            print("Server output:", proc.stdout.read(1024))
-        proc.terminate()
-        raise RuntimeError("Server failed to start")
-
-    # Test text completion
-    r = httpx.post(
-        f"{url}/v1/chat/completions",
-        json={
-            "model": "MiniCPM-o-4_5",
-            "messages": [
-                {"role": "user", "content": "Say hello in Chinese"}
-            ],
-            "max_tokens": 50,
-            "temperature": 0.1,
-        },
-        timeout=120,
+    import json
+    from llama_cpp import Llama
+
+    print("[PregoPal] ========== TEST INFERENCE ==========")
+    print(f"[PregoPal] Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S')}")
+    print(f"[PregoPal] GPU: A100 (via Modal)")
+    print(f"[PregoPal] Model base: {MODEL_SUBDIR}")
+
+    paths = get_model_paths(MODEL_SUBDIR)
+    main_path = paths["main"]
+    vision_path = paths["vision"]
+
+    if not os.path.isfile(main_path):
+        print(f"[PregoPal] ERROR: Model not found at {main_path}")
+        print("[PregoPal] Upload models first: modal run modal_deploy.deploy::upload_models")
+        return
+
+    # Load model
+    t0 = time.time()
+    kwargs = dict(
+        model_path=main_path,
+        n_gpu_layers=-1,
+        n_ctx=4096,
+        verbose=True,
     )
-    print(f"Text response: {r.json()}")
-
-    # Test multimodal if vision mmproj exists
-    vision_path = os.path.join(MODEL_SUBDIR, VISION_MMPROJ)
     if os.path.isfile(vision_path):
-        print("Vision mmproj found, skipping multimodal test (no test image available)")
-    
-    proc.terminate()
-    print("\n✅ Test passed!")
+        kwargs["mmproj"] = vision_path
+
+    print("[PregoPal] Loading model...")
+    llm = Llama(**kwargs)
+    load_time = time.time() - t0
+    print(f"[PregoPal] Model loaded in {load_time:.1f}s")
+
+    # ── Test 1: Text generation ──
+    print("\n[Test 1] Chinese greeting...")
+    t0 = time.time()
+    result = llm.create_chat_completion(
+        messages=[{"role": "user", "content": "用中文说你好，不超过10个字"}],
+        max_tokens=30,
+        temperature=0.1,
+    )
+    elapsed = time.time() - t0
+    content = result.get("choices", [{}])[0].get("message", {}).get("content", "")
+    print(f"Response ({elapsed:.1f}s): {content}")
+
+    # ── Test 2: English text ──
+    print("\n[Test 2] English instruction...")
+    t0 = time.time()
+    result = llm.create_chat_completion(
+        messages=[{"role": "user", "content": "What is the capital of France? Answer in 5 words."}],
+        max_tokens=30,
+        temperature=0.1,
+    )
+    elapsed = time.time() - t0
+    content = result.get("choices", [{}])[0].get("message", {}).get("content", "")
+    print(f"Response ({elapsed:.1f}s): {content}")
+
+    print(f"\n{'='*50}")
+    print(f"✅ Test complete! Loading: {load_time:.1f}s")
+    print(f"✅ Inference speed: {elapsed:.1f}s per response (CUDA)")
+    print(f"{'='*50}")