Merge branch 'main' of https://huggingface.co/spaces/build-small-hackathon/PregoPal

modal_deploy/deploy_omni.py

@@ -0,0 +1,558 @@
+"""
+PregoPal x MiniCPM-o-4_5 - Modal deploy (llama.cpp-omni full-duplex voice upgrade)
+
+Architecture:
+  FastAPI (ASGI) <-> llama-server (OpenBMB/llama.cpp-omni subprocess)
+                         |
+                  Modal Volume: GGUF models (vision + audio + TTS)
+
+Usage:
+    pip install modal
+    modal token new
+    modal deploy modal_deploy.deploy_omni
+
+Test:
+    modal run -m modal_deploy.deploy_omni::test_inference
+    modal run -m modal_deploy.deploy_omni::diagnose_volume
+
+API:
+    POST /v1/chat/completions   - OpenAI compatible (text + multimodal, streaming)
+    POST /v1/audio/speech       - TTS: text -> voice WAV
+    POST /v1/audio/transcriptions - STT: voice -> text
+    POST /v1/embeddings         - Embeddings
+    GET  /health                - Health check (audio/vision/TTS status)
+    GET  /v1/models             - Model list
+"""
+
+import os
+import modal
+from modal import Image, App, Volume, asgi_app
+
+# ============================================================================
+# 1. IMAGE - Build OpenBMB/llama.cpp-omni from source
+#    Source is copied from local llamacpp_omni/ (repo no longer public on GitHub)
+# ============================================================================
+
+_omni_image = (
+    Image.debian_slim(python_version="3.11")
+    .apt_install(
+        "curl",
+    )
+    # Install CUDA Toolkit for compiling llama.cpp CUDA kernels
+    .run_commands(
+        "curl -L -o /tmp/cuda-keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb",
+        "dpkg -i /tmp/cuda-keyring.deb",
+        "apt-get update",
+        "apt-get install -y cuda-toolkit-12-4 cuda-compiler-12-4",
+    )
+    .apt_install(
+        "curl",
+        "git",
+        "build-essential",
+        "cmake",
+        "libcurl4-openssl-dev",
+        "libsndfile1",
+        "libasound2-dev",
+        "pkg-config",
+    )
+    .pip_install(
+        "fastapi",
+        "uvicorn[standard]",
+        "httpx",
+        "numpy",
+        "Pillow",
+        "soundfile",
+    )
+    # Copy local llamacpp_omni source into image (repo no longer public)
+    .add_local_dir(
+        os.path.join(os.path.dirname(os.path.abspath(__file__)), "llamacpp_omni"),
+        "/llama.cpp-omni",
+        copy=True,
+    )
+.run_commands(
+        "cd /llama.cpp-omni && cmake -B build "
+        "-DGGML_CUDA=ON "
+        "-DLLAMA_CURL=ON "
+        "-DLLAMA_BUILD_SERVER=ON "
+        "-DLLAMA_BUILD_TESTS=OFF "
+        "-DLLAMA_BUILD_EXAMPLES=OFF "
+        "-DLLAMA_CUDA_FORCE_MMQ=ON "
+        "-DCMAKE_CUDA_ARCHITECTURES='75;89' "
+        "-DCMAKE_BUILD_TYPE=Release "
+        "-DCMAKE_CUDA_COMPILER=/usr/local/cuda-12/bin/nvcc",
+        "cd /llama.cpp-omni && cmake --build build --config Release -j $(nproc) "
+        "--target llama-server",
+        "ls -lh /llama.cpp-omni/build/bin/llama-server",
+    )
+)
+
+# ============================================================================
+# 2. CONSTANTS
+# ============================================================================
+
+MODEL_DIR = "/models"
+MODEL_SUBDIR = f"{MODEL_DIR}/MiniCPM-o-4_5-gguf"
+
+MAIN_GGUF = "MiniCPM-o-4_5-Q4_K_M.gguf"
+VISION_MMPROJ = "vision/MiniCPM-o-4_5-vision-F16.gguf"
+AUDIO_MMPROJ = "audio/MiniCPM-o-4_5-audio-F16.gguf"
+TTS_BASE_LM = "tts/MiniCPM-o-4_5-tts-F16.gguf"
+TTS_ACOUSTIC = "tts/MiniCPM-o-4_5-projector-F16.gguf"
+TOKEN2WAV_DIR = "token2wav-gguf"
+
+LLAMA_SERVER_PORT = 8081
+
+model_volume = Volume.from_name("minicpm-o-4_5-models", create_if_missing=True)
+app = App("prego-pal-minicpm-omni")
+
+
+def get_model_paths(base_dir: str) -> dict:
+    paths = {
+        "main": os.path.join(base_dir, MAIN_GGUF),
+        "vision": os.path.join(base_dir, VISION_MMPROJ),
+        "audio": os.path.join(base_dir, AUDIO_MMPROJ),
+        "tts_base_lm": os.path.join(base_dir, TTS_BASE_LM),
+        "tts_acoustic": os.path.join(base_dir, TTS_ACOUSTIC),
+        "token2wav_dir": os.path.join(base_dir, TOKEN2WAV_DIR),
+    }
+    for key, path in paths.items():
+        if key == "token2wav_dir":
+            exists = os.path.isdir(path)
+        else:
+            exists = os.path.isfile(path)
+        print(f"[PregoPal] {key}: {path} (exists={exists})")
+    return paths
+
+
+# ============================================================================
+# 3. ASGI APP - FastAPI lifespan + llama-server subprocess
+# ============================================================================
+
+@app.function(
+    image=_omni_image,
+    volumes={MODEL_DIR: model_volume},
+    gpu="T4",
+    timeout=1200,
+    scaledown_window=300,
+)
+@modal.concurrent(max_inputs=10)
+@asgi_app()
+def serve():
+    """
+    FastAPI ASGI app. Launches llama-server subprocess in lifespan.
+    serve() is sync; async logic lives in lifespan context manager.
+    """
+    import asyncio
+    import json
+    import logging
+    import subprocess
+    from contextlib import asynccontextmanager
+    from fastapi import FastAPI, Request
+    from fastapi.responses import StreamingResponse, JSONResponse
+    from fastapi.middleware.cors import CORSMiddleware
+    import httpx
+
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger("prego-pal-omni")
+
+    paths = get_model_paths(MODEL_SUBDIR)
+
+    # Build llama-server command
+    llama_server_bin = "/llama.cpp-omni/build/bin/llama-server"
+    if not os.path.isfile(llama_server_bin):
+        llama_server_bin = "/llama.cpp-omni/build/bin/Release/llama-server"
+
+    cmd = [
+        llama_server_bin,
+        "-m", paths["main"],
+        "--mmproj", paths["vision"],
+        "--mmproj", paths["audio"],
+        "--voxcpm2-base-lm", paths["tts_base_lm"],
+        "--voxcpm2-acoustic", paths["tts_acoustic"],
+        "--host", "127.0.0.1",
+        "--port", str(LLAMA_SERVER_PORT),
+        "-ngl", "99",
+        "-c", "8192",
+        "--no-mmap",
+        "--jinja",
+    ]
+
+    # Check token2wav directory
+    t2w_ok = os.path.isdir(paths["token2wav_dir"])
+    if t2w_ok:
+        t2w_files = os.listdir(paths["token2wav_dir"])
+        logger.info(f"[PregoPal] token2wav files ({len(t2w_files)}): {t2w_files}")
+    else:
+        logger.warning("[PregoPal] token2wav dir NOT FOUND - TTS disabled")
+
+    @asynccontextmanager
+    async def lifespan(web_app: FastAPI):
+        """Async lifecycle: start llama-server subprocess, cleanup on shutdown."""
+        logger.info("[PregoPal] Starting llama-server...")
+        server_process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+
+        # Poll /health until ready (max 90s)
+        base_url = f"http://127.0.0.1:{LLAMA_SERVER_PORT}"
+        ready = False
+        for i in range(45):
+            await asyncio.sleep(2)
+            try:
+                async with httpx.AsyncClient(timeout=5.0) as client:
+                    r = await client.get(f"{base_url}/health")
+                    if r.status_code == 200:
+                        ready = True
+                        logger.info(f"[PregoPal] llama-server ready (attempt {i+1})")
+                        break
+            except Exception:
+                if i > 0 and i % 5 == 0:
+                    logger.info(f"[PregoPal] Waiting for llama-server (attempt {i+1})...")
+
+        if not ready:
+            stderr_lines = []
+            try:
+                for _ in range(20):
+                    line = server_process.stderr.readline()
+                    if line:
+                        stderr_lines.append(line.strip())
+            except Exception:
+                pass
+            logger.error("[PregoPal] llama-server failed to start.\n"
+                          + "\n".join(stderr_lines[-10:]))
+            server_process.terminate()
+            raise RuntimeError("llama-server failed to start within 90s")
+
+        web_app.state.llama_base_url = base_url
+        web_app.state.llama_client = httpx.AsyncClient(base_url=base_url, timeout=120.0)
+
+        yield
+
+        logger.info("[PregoPal] Shutting down llama-server...")
+        server_process.terminate()
+        server_process.wait(timeout=30)
+        await web_app.state.llama_client.aclose()
+        logger.info("[PregoPal] Shutdown complete")
+
+    web_app = FastAPI(
+        title="PregoPal MiniCPM-o-4_5 Omni API",
+        lifespan=lifespan,
+    )
+    web_app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    base_url = f"http://127.0.0.1:{LLAMA_SERVER_PORT}"
+
+    # ---- Proxy Endpoints ----
+
+    @web_app.post("/v1/chat/completions")
+    async def chat_completions(request: Request):
+        body = await request.json()
+        stream = body.get("stream", False)
+        client = web_app.state.llama_client
+
+        if stream:
+            async def event_stream():
+                async with httpx.AsyncClient(timeout=120.0) as sclient:
+                    async with sclient.stream(
+                        "POST", f"{base_url}/v1/chat/completions", json=body
+                    ) as resp:
+                        async for chunk in resp.aiter_lines():
+                            if chunk:
+                                yield chunk + "\n"
+            return StreamingResponse(event_stream(), media_type="text/event-stream")
+
+        try:
+            resp = await client.post("/v1/chat/completions", json=body)
+            return JSONResponse(resp.json(), status_code=resp.status_code)
+        except Exception as e:
+            logger.error(f"[PregoPal] Chat completion proxy error: {e}")
+            return JSONResponse({"error": str(e)}, status_code=502)
+
+    @web_app.post("/v1/audio/speech")
+    async def audio_speech(request: Request):
+        """TTS: text -> speech WAV"""
+        body = await request.json()
+        client = web_app.state.llama_client
+        try:
+            resp = await client.post("/v1/audio/speech", json=body)
+            return StreamingResponse(
+                resp.aiter_bytes(),
+                media_type=resp.headers.get("content-type", "audio/wav"),
+            )
+        except Exception as e:
+            logger.error(f"[PregoPal] TTS error: {e}")
+            return JSONResponse({"error": str(e)}, status_code=502)
+
+    @web_app.post("/v1/audio/speech/stream")
+    async def audio_speech_stream(request: Request):
+        """Streaming TTS"""
+        body = await request.json()
+        try:
+            async with httpx.AsyncClient(timeout=120.0) as sclient:
+                async with sclient.stream(
+                    "POST", f"{base_url}/v1/audio/speech/stream", json=body
+                ) as resp:
+                    async def audio_stream():
+                        async for chunk in resp.aiter_bytes():
+                            yield chunk
+                    return StreamingResponse(
+                        audio_stream(),
+                        media_type=resp.headers.get("content-type", "audio/wav"),
+                    )
+        except Exception as e:
+            logger.error(f"[PregoPal] Stream TTS error: {e}")
+            return JSONResponse({"error": str(e)}, status_code=502)
+
+    @web_app.post("/v1/audio/transcriptions")
+    async def audio_transcriptions(request: Request):
+        """STT: speech -> text"""
+        body = await request.json()
+        client = web_app.state.llama_client
+        try:
+            resp = await client.post("/v1/audio/transcriptions", json=body)
+            return JSONResponse(resp.json(), status_code=resp.status_code)
+        except Exception as e:
+            logger.error(f"[PregoPal] STT error: {e}")
+            return JSONResponse({"error": str(e)}, status_code=502)
+
+    @web_app.post("/v1/embeddings")
+    async def embeddings(request: Request):
+        body = await request.json()
+        client = web_app.state.llama_client
+        try:
+            resp = await client.post("/v1/embeddings", json=body)
+            return JSONResponse(resp.json(), status_code=resp.status_code)
+        except Exception as e:
+            logger.error(f"[PregoPal] Embeddings proxy error: {e}")
+            return JSONResponse({"error": str(e)}, status_code=502)
+
+    @web_app.get("/health")
+    async def health():
+        try:
+            client = web_app.state.llama_client
+            ls_resp = await client.get("/health")
+            ls_status = ls_resp.json()
+        except Exception as e:
+            ls_status = {"error": str(e)}
+        return {
+            "status": "ok",
+            "model": "MiniCPM-o-4_5",
+            "engine": "llama.cpp-omni",
+            "cuda": True,
+            "vision": os.path.isfile(paths["vision"]),
+            "audio": os.path.isfile(paths["audio"]),
+            "tts_base_lm": os.path.isfile(paths["tts_base_lm"]),
+            "tts_acoustic": os.path.isfile(paths["tts_acoustic"]),
+            "token2wav_dir": os.path.isdir(paths["token2wav_dir"]),
+            "llama_server_status": ls_status,
+        }
+
+    @web_app.get("/v1/models")
+    async def list_models():
+        try:
+            client = web_app.state.llama_client
+            resp = await client.get("/v1/models")
+            return JSONResponse(resp.json(), status_code=resp.status_code)
+        except Exception:
+            return JSONResponse({
+                "object": "list",
+                "data": [{
+                    "id": "MiniCPM-o-4_5",
+                    "object": "model",
+                    "created": 1,
+                    "owned_by": "prego-pal",
+                }],
+            })
+
+    @web_app.get("/")
+    async def root():
+        return {
+            "service": "PregoPal MiniCPM-o-4_5 Omni API",
+            "version": "3.0.0",
+            "model": MAIN_GGUF,
+            "engine": "llama.cpp-omni (OpenBMB)",
+            "endpoints": {
+                "chat": "POST /v1/chat/completions (text+multimodal, streaming)",
+                "tts": "POST /v1/audio/speech (text->speech)",
+                "tts_stream": "POST /v1/audio/speech/stream (streaming TTS)",
+                "stt": "POST /v1/audio/transcriptions (speech->text)",
+                "embeddings": "POST /v1/embeddings",
+                "models": "GET /v1/models",
+                "health": "GET /health",
+            },
+        }
+
+    return web_app
+
+
+# ============================================================================
+# 4. DIAGNOSE VOLUME
+# ============================================================================
+
+@app.function(
+    image=_omni_image,
+    volumes={MODEL_DIR: model_volume},
+    timeout=120,
+)
+def diagnose_volume():
+    """Check model file integrity in Modal Volume."""
+    print(f"\n{'='*60}")
+    print(f"[Diagnose] {MODEL_SUBDIR}")
+    print(f"{'='*60}")
+    for root, dirs, files in os.walk(MODEL_SUBDIR):
+        level = root.replace(MODEL_SUBDIR, "").count(os.sep)
+        indent = " " * 2 * level
+        print(f"{indent}{os.path.basename(root)}/")
+        subindent = " " * 2 * (level + 1)
+        for file in sorted(files):
+            fpath = os.path.join(root, file)
+            size = os.path.getsize(fpath)
+            print(f"{subindent}{file}  ({size:,} bytes = {size/1024**3:.2f} GB)")
+
+    paths = get_model_paths(MODEL_SUBDIR)
+    all_ok = True
+    for key, path in paths.items():
+        if key == "token2wav_dir":
+            ok = os.path.isdir(path)
+        else:
+            ok = os.path.isfile(path)
+        status = "OK" if ok else "MISSING"
+        if not ok:
+            all_ok = False
+        print(f"  [{status}] {key}: {path}")
+
+    if all_ok:
+        print(f"\n[OK] All model files found! Ready to deploy.")
+    else:
+        print(f"\n[FAIL] Some files missing. Check uploads.")
+
+    main_path = paths["main"]
+    if os.path.isfile(main_path):
+        with open(main_path, "rb") as f:
+            magic = f.read(4)
+        if magic == b"GGUF":
+            print("[OK] Main model is valid GGUF")
+        else:
+            print(f"[WARN] Main model NOT valid GGUF (magic={magic.hex()})")
+
+
+# ============================================================================
+# 5. TEST INFERENCE (standalone - not via ASGI)
+# ============================================================================
+
+@app.function(
+    image=_omni_image,
+    volumes={MODEL_DIR: model_volume},
+    gpu="T4",
+    timeout=600,
+)
+def test_inference():
+    """Test llama-server text+multimodal inference on Modal T4."""
+    import subprocess
+    import time
+    import httpx
+
+    print("[PregoPal] ========== TEST INFERENCE (llama-server) ==========")
+
+    cmd = [
+        "/llama.cpp-omni/build/bin/llama-server",
+        "-m", os.path.join(MODEL_SUBDIR, MAIN_GGUF),
+        "--mmproj", os.path.join(MODEL_SUBDIR, VISION_MMPROJ),
+        "--mmproj", os.path.join(MODEL_SUBDIR, AUDIO_MMPROJ),
+        "--host", "127.0.0.1",
+        "--port", "8081",
+        "-ngl", "99",
+        "-c", "4096",
+        "--no-mmap",
+    ]
+
+    print("[PregoPal] Starting llama-server...")
+    server_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+    base_url = "http://127.0.0.1:8081"
+    ready = False
+    for i in range(30):
+        time.sleep(2)
+        try:
+            r = httpx.get(f"{base_url}/health", timeout=5.0)
+            if r.status_code == 200:
+                ready = True
+                print(f"[PregoPal] llama-server ready (attempt {i+1})")
+                break
+        except Exception:
+            print(f"[PregoPal] Waiting (attempt {i+1})...")
+
+    if not ready:
+        stderr_tail = []
+        for _ in range(10):
+            line = server_proc.stderr.readline()
+            if line:
+                stderr_tail.append(line.strip())
+        print(f"[PregoPal] Timed out waiting for server.\nstderr:\n" + "\n".join(stderr_tail))
+        server_proc.terminate()
+        return
+
+    client = httpx.Client(base_url=base_url, timeout=120.0)
+
+    try:
+        # Test 1: Chinese
+        print("\n[Test 1] Chinese...")
+        t0 = time.time()
+        resp = client.post("/v1/chat/completions", json={
+            "messages": [{"role": "user", "content": "Say hello in Chinese, max 10 chars"}],
+            "max_tokens": 30, "temperature": 0.1,
+        })
+        t1 = time.time()
+        content = resp.json().get("choices", [{}])[0].get("message", {}).get("content", "")
+        print(f"Response ({t1-t0:.1f}s): {content} (status={resp.status_code})")
+
+        # Test 2: English
+        print("\n[Test 2] English...")
+        t0 = time.time()
+        resp = client.post("/v1/chat/completions", json={
+            "messages": [{"role": "user", "content": "What is the capital of France? Answer in 5 words."}],
+            "max_tokens": 30, "temperature": 0.1,
+        })
+        t1 = time.time()
+        content = resp.json().get("choices", [{}])[0].get("message", {}).get("content", "")
+        print(f"Response ({t1-t0:.1f}s): {content} (status={resp.status_code})")
+
+        # Test 3: Health
+        print("\n[Test 3] Health...")
+        resp = client.get("/health")
+        info = resp.json()
+        print(f"Health: model={info.get('model')}, cuda={info.get('cuda')}, "
+              f"vision={info.get('vision')}, audio={info.get('audio')}")
+
+        print(f"\n{'='*50}")
+        print("[OK] All tests passed!")
+        print(f"{'='*50}")
+
+    except Exception as e:
+        print(f"[PregoPal] Test error: {e}")
+        raise
+    finally:
+        server_proc.terminate()
+        server_proc.wait(timeout=10)
+
+
+# ============================================================================
+# 6. LOCAL ENTRY POINT
+# ============================================================================
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "test_inference":
+            test_inference.local()
+        elif sys.argv[1] == "diagnose_volume":
+            diagnose_volume.local()

modal_deploy/deploy_omni_bak_v1.py

@@ -0,0 +1,556 @@
+"""
+PregoPal x MiniCPM-o-4_5 - Modal deploy (llama.cpp-omni full-duplex voice upgrade)
+
+Architecture:
+  FastAPI (ASGI) <-> llama-server (OpenBMB/llama.cpp-omni subprocess)
+                         |
+                  Modal Volume: GGUF models (vision + audio + TTS)
+
+Usage:
+    pip install modal
+    modal token new
+    modal deploy modal_deploy.deploy_omni
+
+Test:
+    modal run -m modal_deploy.deploy_omni::test_inference
+    modal run -m modal_deploy.deploy_omni::diagnose_volume
+
+API:
+    POST /v1/chat/completions   - OpenAI compatible (text + multimodal, streaming)
+    POST /v1/audio/speech       - TTS: text -> voice WAV
+    POST /v1/audio/transcriptions - STT: voice -> text
+    POST /v1/embeddings         - Embeddings
+    GET  /health                - Health check (audio/vision/TTS status)
+    GET  /v1/models             - Model list
+"""
+
+import os
+import modal
+from modal import Image, App, Volume, asgi_app
+
+# ============================================================================
+# 1. IMAGE - Build OpenBMB/llama.cpp-omni from source
+#    Source is copied from local llamacpp_omni/ (repo no longer public on GitHub)
+# ============================================================================
+
+_omni_image = (
+    Image.debian_slim(python_version="3.11")
+    .apt_install(
+        "curl",
+    )
+    # Install CUDA Toolkit for compiling llama.cpp CUDA kernels
+    .run_commands(
+        "curl -L -o /tmp/cuda-keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb",
+        "dpkg -i /tmp/cuda-keyring.deb",
+        "apt-get update",
+        "apt-get install -y cuda-toolkit-12-4 cuda-compiler-12-4",
+    )
+    .apt_install(
+        "curl",
+        "git",
+        "build-essential",
+        "cmake",
+        "libcurl4-openssl-dev",
+        "libsndfile1",
+        "libasound2-dev",
+        "pkg-config",
+    )
+    .pip_install(
+        "fastapi",
+        "uvicorn[standard]",
+        "httpx",
+        "numpy",
+        "Pillow",
+        "soundfile",
+    )
+    # Copy local llamacpp_omni source into image (repo no longer public)
+    .add_local_dir(
+        os.path.join(os.path.dirname(os.path.abspath(__file__)), "llamacpp_omni"),
+        "/llama.cpp-omni",
+        copy=True,
+    )
+    .run_commands(
+        "cd /llama.cpp-omni && cmake -B build "
+        "-DGGML_CUDA=ON "
+        "-DLLAMA_CURL=ON "
+        "-DLLAMA_BUILD_SERVER=ON "
+        "-DLLAMA_BUILD_TESTS=OFF "
+        "-DLLAMA_BUILD_EXAMPLES=OFF "
+        "-DCMAKE_BUILD_TYPE=Release "
+        "-DCMAKE_CUDA_COMPILER=/usr/local/cuda-12/bin/nvcc",
+        "cd /llama.cpp-omni && cmake --build build --config Release -j $(nproc) "
+        "--target llama-server llama-mtmd-cli",
+        "ls -lh /llama.cpp-omni/build/bin/llama-server /llama.cpp-omni/build/bin/llama-mtmd-cli",
+    )
+)
+
+# ============================================================================
+# 2. CONSTANTS
+# ============================================================================
+
+MODEL_DIR = "/models"
+MODEL_SUBDIR = f"{MODEL_DIR}/MiniCPM-o-4_5-gguf"
+
+MAIN_GGUF = "MiniCPM-o-4_5-Q4_K_M.gguf"
+VISION_MMPROJ = "vision/MiniCPM-o-4_5-vision-F16.gguf"
+AUDIO_MMPROJ = "audio/MiniCPM-o-4_5-audio-F16.gguf"
+TTS_BASE_LM = "tts/MiniCPM-o-4_5-tts-F16.gguf"
+TTS_ACOUSTIC = "tts/MiniCPM-o-4_5-projector-F16.gguf"
+TOKEN2WAV_DIR = "token2wav-gguf"
+
+LLAMA_SERVER_PORT = 8081
+
+model_volume = Volume.from_name("minicpm-o-4_5-models", create_if_missing=True)
+app = App("prego-pal-minicpm-omni")
+
+
+def get_model_paths(base_dir: str) -> dict:
+    paths = {
+        "main": os.path.join(base_dir, MAIN_GGUF),
+        "vision": os.path.join(base_dir, VISION_MMPROJ),
+        "audio": os.path.join(base_dir, AUDIO_MMPROJ),
+        "tts_base_lm": os.path.join(base_dir, TTS_BASE_LM),
+        "tts_acoustic": os.path.join(base_dir, TTS_ACOUSTIC),
+        "token2wav_dir": os.path.join(base_dir, TOKEN2WAV_DIR),
+    }
+    for key, path in paths.items():
+        if key == "token2wav_dir":
+            exists = os.path.isdir(path)
+        else:
+            exists = os.path.isfile(path)
+        print(f"[PregoPal] {key}: {path} (exists={exists})")
+    return paths
+
+
+# ============================================================================
+# 3. ASGI APP - FastAPI lifespan + llama-server subprocess
+# ============================================================================
+
+@app.function(
+    image=_omni_image,
+    volumes={MODEL_DIR: model_volume},
+    gpu="T4",
+    timeout=1200,
+    scaledown_window=300,
+)
+@modal.concurrent(max_inputs=10)
+@asgi_app()
+def serve():
+    """
+    FastAPI ASGI app. Launches llama-server subprocess in lifespan.
+    serve() is sync; async logic lives in lifespan context manager.
+    """
+    import asyncio
+    import json
+    import logging
+    import subprocess
+    from contextlib import asynccontextmanager
+    from fastapi import FastAPI, Request
+    from fastapi.responses import StreamingResponse, JSONResponse
+    from fastapi.middleware.cors import CORSMiddleware
+    import httpx
+
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger("prego-pal-omni")
+
+    paths = get_model_paths(MODEL_SUBDIR)
+
+    # Build llama-server command
+    llama_server_bin = "/llama.cpp-omni/build/bin/llama-server"
+    if not os.path.isfile(llama_server_bin):
+        llama_server_bin = "/llama.cpp-omni/build/bin/Release/llama-server"
+
+    cmd = [
+        llama_server_bin,
+        "-m", paths["main"],
+        "--mmproj", paths["vision"],
+        "--mmproj", paths["audio"],
+        "--voxcpm2-base-lm", paths["tts_base_lm"],
+        "--voxcpm2-acoustic", paths["tts_acoustic"],
+        "--host", "127.0.0.1",
+        "--port", str(LLAMA_SERVER_PORT),
+        "-ngl", "99",
+        "-c", "8192",
+        "--no-mmap",
+        "--jinja",
+    ]
+
+    # Check token2wav directory
+    t2w_ok = os.path.isdir(paths["token2wav_dir"])
+    if t2w_ok:
+        t2w_files = os.listdir(paths["token2wav_dir"])
+        logger.info(f"[PregoPal] token2wav files ({len(t2w_files)}): {t2w_files}")
+    else:
+        logger.warning("[PregoPal] token2wav dir NOT FOUND - TTS disabled")
+
+    @asynccontextmanager
+    async def lifespan(web_app: FastAPI):
+        """Async lifecycle: start llama-server subprocess, cleanup on shutdown."""
+        logger.info("[PregoPal] Starting llama-server...")
+        server_process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+
+        # Poll /health until ready (max 90s)
+        base_url = f"http://127.0.0.1:{LLAMA_SERVER_PORT}"
+        ready = False
+        for i in range(45):
+            await asyncio.sleep(2)
+            try:
+                async with httpx.AsyncClient(timeout=5.0) as client:
+                    r = await client.get(f"{base_url}/health")
+                    if r.status_code == 200:
+                        ready = True
+                        logger.info(f"[PregoPal] llama-server ready (attempt {i+1})")
+                        break
+            except Exception:
+                if i > 0 and i % 5 == 0:
+                    logger.info(f"[PregoPal] Waiting for llama-server (attempt {i+1})...")
+
+        if not ready:
+            stderr_lines = []
+            try:
+                for _ in range(20):
+                    line = server_process.stderr.readline()
+                    if line:
+                        stderr_lines.append(line.strip())
+            except Exception:
+                pass
+            logger.error("[PregoPal] llama-server failed to start.\n"
+                          + "\n".join(stderr_lines[-10:]))
+            server_process.terminate()
+            raise RuntimeError("llama-server failed to start within 90s")
+
+        web_app.state.llama_base_url = base_url
+        web_app.state.llama_client = httpx.AsyncClient(base_url=base_url, timeout=120.0)
+
+        yield
+
+        logger.info("[PregoPal] Shutting down llama-server...")
+        server_process.terminate()
+        server_process.wait(timeout=30)
+        await web_app.state.llama_client.aclose()
+        logger.info("[PregoPal] Shutdown complete")
+
+    web_app = FastAPI(
+        title="PregoPal MiniCPM-o-4_5 Omni API",
+        lifespan=lifespan,
+    )
+    web_app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    base_url = f"http://127.0.0.1:{LLAMA_SERVER_PORT}"
+
+    # ---- Proxy Endpoints ----
+
+    @web_app.post("/v1/chat/completions")
+    async def chat_completions(request: Request):
+        body = await request.json()
+        stream = body.get("stream", False)
+        client = web_app.state.llama_client
+
+        if stream:
+            async def event_stream():
+                async with httpx.AsyncClient(timeout=120.0) as sclient:
+                    async with sclient.stream(
+                        "POST", f"{base_url}/v1/chat/completions", json=body
+                    ) as resp:
+                        async for chunk in resp.aiter_lines():
+                            if chunk:
+                                yield chunk + "\n"
+            return StreamingResponse(event_stream(), media_type="text/event-stream")
+
+        try:
+            resp = await client.post("/v1/chat/completions", json=body)
+            return JSONResponse(resp.json(), status_code=resp.status_code)
+        except Exception as e:
+            logger.error(f"[PregoPal] Chat completion proxy error: {e}")
+            return JSONResponse({"error": str(e)}, status_code=502)
+
+    @web_app.post("/v1/audio/speech")
+    async def audio_speech(request: Request):
+        """TTS: text -> speech WAV"""
+        body = await request.json()
+        client = web_app.state.llama_client
+        try:
+            resp = await client.post("/v1/audio/speech", json=body)
+            return StreamingResponse(
+                resp.aiter_bytes(),
+                media_type=resp.headers.get("content-type", "audio/wav"),
+            )
+        except Exception as e:
+            logger.error(f"[PregoPal] TTS error: {e}")
+            return JSONResponse({"error": str(e)}, status_code=502)
+
+    @web_app.post("/v1/audio/speech/stream")
+    async def audio_speech_stream(request: Request):
+        """Streaming TTS"""
+        body = await request.json()
+        try:
+            async with httpx.AsyncClient(timeout=120.0) as sclient:
+                async with sclient.stream(
+                    "POST", f"{base_url}/v1/audio/speech/stream", json=body
+                ) as resp:
+                    async def audio_stream():
+                        async for chunk in resp.aiter_bytes():
+                            yield chunk
+                    return StreamingResponse(
+                        audio_stream(),
+                        media_type=resp.headers.get("content-type", "audio/wav"),
+                    )
+        except Exception as e:
+            logger.error(f"[PregoPal] Stream TTS error: {e}")
+            return JSONResponse({"error": str(e)}, status_code=502)
+
+    @web_app.post("/v1/audio/transcriptions")
+    async def audio_transcriptions(request: Request):
+        """STT: speech -> text"""
+        body = await request.json()
+        client = web_app.state.llama_client
+        try:
+            resp = await client.post("/v1/audio/transcriptions", json=body)
+            return JSONResponse(resp.json(), status_code=resp.status_code)
+        except Exception as e:
+            logger.error(f"[PregoPal] STT error: {e}")
+            return JSONResponse({"error": str(e)}, status_code=502)
+
+    @web_app.post("/v1/embeddings")
+    async def embeddings(request: Request):
+        body = await request.json()
+        client = web_app.state.llama_client
+        try:
+            resp = await client.post("/v1/embeddings", json=body)
+            return JSONResponse(resp.json(), status_code=resp.status_code)
+        except Exception as e:
+            logger.error(f"[PregoPal] Embeddings proxy error: {e}")
+            return JSONResponse({"error": str(e)}, status_code=502)
+
+    @web_app.get("/health")
+    async def health():
+        try:
+            client = web_app.state.llama_client
+            ls_resp = await client.get("/health")
+            ls_status = ls_resp.json()
+        except Exception as e:
+            ls_status = {"error": str(e)}
+        return {
+            "status": "ok",
+            "model": "MiniCPM-o-4_5",
+            "engine": "llama.cpp-omni",
+            "cuda": True,
+            "vision": os.path.isfile(paths["vision"]),
+            "audio": os.path.isfile(paths["audio"]),
+            "tts_base_lm": os.path.isfile(paths["tts_base_lm"]),
+            "tts_acoustic": os.path.isfile(paths["tts_acoustic"]),
+            "token2wav_dir": os.path.isdir(paths["token2wav_dir"]),
+            "llama_server_status": ls_status,
+        }
+
+    @web_app.get("/v1/models")
+    async def list_models():
+        try:
+            client = web_app.state.llama_client
+            resp = await client.get("/v1/models")
+            return JSONResponse(resp.json(), status_code=resp.status_code)
+        except Exception:
+            return JSONResponse({
+                "object": "list",
+                "data": [{
+                    "id": "MiniCPM-o-4_5",
+                    "object": "model",
+                    "created": 1,
+                    "owned_by": "prego-pal",
+                }],
+            })
+
+    @web_app.get("/")
+    async def root():
+        return {
+            "service": "PregoPal MiniCPM-o-4_5 Omni API",
+            "version": "3.0.0",
+            "model": MAIN_GGUF,
+            "engine": "llama.cpp-omni (OpenBMB)",
+            "endpoints": {
+                "chat": "POST /v1/chat/completions (text+multimodal, streaming)",
+                "tts": "POST /v1/audio/speech (text->speech)",
+                "tts_stream": "POST /v1/audio/speech/stream (streaming TTS)",
+                "stt": "POST /v1/audio/transcriptions (speech->text)",
+                "embeddings": "POST /v1/embeddings",
+                "models": "GET /v1/models",
+                "health": "GET /health",
+            },
+        }
+
+    return web_app
+
+
+# ============================================================================
+# 4. DIAGNOSE VOLUME
+# ============================================================================
+
+@app.function(
+    image=_omni_image,
+    volumes={MODEL_DIR: model_volume},
+    timeout=120,
+)
+def diagnose_volume():
+    """Check model file integrity in Modal Volume."""
+    print(f"\n{'='*60}")
+    print(f"[Diagnose] {MODEL_SUBDIR}")
+    print(f"{'='*60}")
+    for root, dirs, files in os.walk(MODEL_SUBDIR):
+        level = root.replace(MODEL_SUBDIR, "").count(os.sep)
+        indent = " " * 2 * level
+        print(f"{indent}{os.path.basename(root)}/")
+        subindent = " " * 2 * (level + 1)
+        for file in sorted(files):
+            fpath = os.path.join(root, file)
+            size = os.path.getsize(fpath)
+            print(f"{subindent}{file}  ({size:,} bytes = {size/1024**3:.2f} GB)")
+
+    paths = get_model_paths(MODEL_SUBDIR)
+    all_ok = True
+    for key, path in paths.items():
+        if key == "token2wav_dir":
+            ok = os.path.isdir(path)
+        else:
+            ok = os.path.isfile(path)
+        status = "OK" if ok else "MISSING"
+        if not ok:
+            all_ok = False
+        print(f"  [{status}] {key}: {path}")
+
+    if all_ok:
+        print(f"\n[OK] All model files found! Ready to deploy.")
+    else:
+        print(f"\n[FAIL] Some files missing. Check uploads.")
+
+    main_path = paths["main"]
+    if os.path.isfile(main_path):
+        with open(main_path, "rb") as f:
+            magic = f.read(4)
+        if magic == b"GGUF":
+            print("[OK] Main model is valid GGUF")
+        else:
+            print(f"[WARN] Main model NOT valid GGUF (magic={magic.hex()})")
+
+
+# ============================================================================
+# 5. TEST INFERENCE (standalone - not via ASGI)
+# ============================================================================
+
+@app.function(
+    image=_omni_image,
+    volumes={MODEL_DIR: model_volume},
+    gpu="T4",
+    timeout=600,
+)
+def test_inference():
+    """Test llama-server text+multimodal inference on Modal T4."""
+    import subprocess
+    import time
+    import httpx
+
+    print("[PregoPal] ========== TEST INFERENCE (llama-server) ==========")
+
+    cmd = [
+        "/llama.cpp-omni/build/bin/llama-server",
+        "-m", os.path.join(MODEL_SUBDIR, MAIN_GGUF),
+        "--mmproj", os.path.join(MODEL_SUBDIR, VISION_MMPROJ),
+        "--mmproj", os.path.join(MODEL_SUBDIR, AUDIO_MMPROJ),
+        "--host", "127.0.0.1",
+        "--port", "8081",
+        "-ngl", "99",
+        "-c", "4096",
+        "--no-mmap",
+    ]
+
+    print("[PregoPal] Starting llama-server...")
+    server_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+    base_url = "http://127.0.0.1:8081"
+    ready = False
+    for i in range(30):
+        time.sleep(2)
+        try:
+            r = httpx.get(f"{base_url}/health", timeout=5.0)
+            if r.status_code == 200:
+                ready = True
+                print(f"[PregoPal] llama-server ready (attempt {i+1})")
+                break
+        except Exception:
+            print(f"[PregoPal] Waiting (attempt {i+1})...")
+
+    if not ready:
+        stderr_tail = []
+        for _ in range(10):
+            line = server_proc.stderr.readline()
+            if line:
+                stderr_tail.append(line.strip())
+        print(f"[PregoPal] Timed out waiting for server.\nstderr:\n" + "\n".join(stderr_tail))
+        server_proc.terminate()
+        return
+
+    client = httpx.Client(base_url=base_url, timeout=120.0)
+
+    try:
+        # Test 1: Chinese
+        print("\n[Test 1] Chinese...")
+        t0 = time.time()
+        resp = client.post("/v1/chat/completions", json={
+            "messages": [{"role": "user", "content": "Say hello in Chinese, max 10 chars"}],
+            "max_tokens": 30, "temperature": 0.1,
+        })
+        t1 = time.time()
+        content = resp.json().get("choices", [{}])[0].get("message", {}).get("content", "")
+        print(f"Response ({t1-t0:.1f}s): {content} (status={resp.status_code})")
+
+        # Test 2: English
+        print("\n[Test 2] English...")
+        t0 = time.time()
+        resp = client.post("/v1/chat/completions", json={
+            "messages": [{"role": "user", "content": "What is the capital of France? Answer in 5 words."}],
+            "max_tokens": 30, "temperature": 0.1,
+        })
+        t1 = time.time()
+        content = resp.json().get("choices", [{}])[0].get("message", {}).get("content", "")
+        print(f"Response ({t1-t0:.1f}s): {content} (status={resp.status_code})")
+
+        # Test 3: Health
+        print("\n[Test 3] Health...")
+        resp = client.get("/health")
+        info = resp.json()
+        print(f"Health: model={info.get('model')}, cuda={info.get('cuda')}, "
+              f"vision={info.get('vision')}, audio={info.get('audio')}")
+
+        print(f"\n{'='*50}")
+        print("[OK] All tests passed!")
+        print(f"{'='*50}")
+
+    except Exception as e:
+        print(f"[PregoPal] Test error: {e}")
+        raise
+    finally:
+        server_proc.terminate()
+        server_proc.wait(timeout=10)
+
+
+# ============================================================================
+# 6. LOCAL ENTRY POINT
+# ============================================================================
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "test_inference":
+            test_inference.local()
+        elif sys.argv[1] == "diagnose_volume":
+            diagnose_volume.local()

modal_deploy/llamacpp_omni

@@ -0,0 +1 @@
+Subproject commit da241979b95b13e747f7c3f8d6821930e0263d33