poolside

/

Laguna-S-2.1-NVFP4

like 159

Follow

Poolside 1.29k

# Laguna S 2.1 NVFP4 — DGX Spark / Portainer Stack v4
# v4 removes shell variables that Portainer Compose would interpolate.
#
# Import in Portainer:
#   Stacks -> Add stack -> Upload -> select this file -> Deploy the stack
#
# This stack intentionally does NOT use the broken prebuilt vLLM ARM64 image.
# It starts from NVIDIA CUDA 13.0.2, creates a persistent Python 3.12 virtual
# environment, installs vLLM 0.25.1 + the Poolside-recommended FlashInfer
# packages, and then serves the already-downloaded local Laguna model.
#
# First-launch safety settings:
#   - 131,072-token context
#   - 1 concurrent sequence
#   - 70% vLLM memory budget
#   - eager execution (no CUDA graphs / torch.compile)
#   - no speculative decoding
#
# After the model is proven stable, those settings can be increased manually.

services:
  laguna-vllm:
    image: nvidia/cuda:13.0.2-devel-ubuntu24.04
    container_name: laguna-vllm
    hostname: laguna-vllm

    # Keep this disabled for the first validation. After a successful launch,
    # change to "unless-stopped" if automatic restart is desired.
    restart: "no"

    ipc: host
    stop_grace_period: 2m

    ports:
      - "8006:8006"

    environment:
      DEBIAN_FRONTEND: noninteractive
      NVIDIA_VISIBLE_DEVICES: all
      NVIDIA_DRIVER_CAPABILITIES: compute,utility
      CUTE_DSL_ARCH: sm_121a
      MAX_JOBS: "4"
      VIRTUAL_ENV: /opt/vllm025
      PATH: /opt/vllm025/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
      PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
      HF_HOME: /root/.cache/huggingface
      VLLM_CACHE_ROOT: /root/.cache/vllm
      TORCH_HOME: /root/.cache/torch

    volumes:
      # The model is already downloaded on the DGX Spark.
      - /home/darkmatter2222/models:/home/darkmatter2222/models:ro

      # Persist the installed environment and compilation/download caches.
      - laguna_vllm025_env:/opt/vllm025
      - laguna_huggingface_cache:/root/.cache/huggingface
      - laguna_vllm_cache:/root/.cache/vllm
      - laguna_flashinfer_cache:/root/.cache/flashinfer
      - laguna_torch_cache:/root/.cache/torch

    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities:
                - gpu

    ulimits:
      memlock:
        soft: -1
        hard: -1
      stack:
        soft: 67108864
        hard: 67108864

    entrypoint:
      - /bin/bash
      - -lc
      - |
        set -Eeuo pipefail
  
        echo "============================================================"
        echo " Laguna S 2.1 NVFP4 bootstrap"
        echo "============================================================"
        echo "CUDA:"
        nvcc --version || true
        nvidia-smi || true
  
        echo
        echo "[1/4] Installing operating-system prerequisites..."
        apt-get update
        apt-get install -y --no-install-recommends \
          ca-certificates \
          curl \
          git \
          build-essential \
          ninja-build \
          pkg-config \
          libnuma-dev \
          python3.12 \
          python3.12-dev \
          python3.12-venv
        rm -rf /var/lib/apt/lists/*
  
        echo
        echo "[2/4] Installing uv when necessary..."
        if [ ! -x /root/.local/bin/uv ]; then
          curl -LsSf https://astral.sh/uv/install.sh | sh
        fi
        /root/.local/bin/uv --version
  
        echo
        echo "[3/4] Creating/updating persistent vLLM 0.25.1 environment..."
        if [ ! -x /opt/vllm025/bin/python ]; then
          /root/.local/bin/uv venv /opt/vllm025 -p python3.12
        fi
  
        if ! /opt/vllm025/bin/python -c \
          'import vllm,sys; sys.exit(0 if vllm.__version__ == "0.25.1" else 1)' \
          >/dev/null 2>&1; then
          /root/.local/bin/uv pip install \
            -p /opt/vllm025 \
            "vllm==0.25.1" \
            --torch-backend=cu130
        fi
  
        if ! /opt/vllm025/bin/python -c \
          'import flashinfer' >/dev/null 2>&1; then
          /root/.local/bin/uv pip install \
            -p /opt/vllm025 \
            "flashinfer-python==0.6.15.dev20260712" \
            "flashinfer-cubin==0.6.15.dev20260712" \
            "flashinfer-jit-cache==0.6.15.dev20260712" \
            --extra-index-url https://flashinfer.ai/whl/nightly/ \
            --extra-index-url https://flashinfer.ai/whl/nightly/cu130/ \
            --index-strategy unsafe-best-match
        fi
  
        echo
        echo "[4/4] Verifying runtime..."
        /opt/vllm025/bin/python - <<'PY'
        import torch
        import vllm
        import flashinfer
  
        print("Python:", __import__("sys").version)
        print("vLLM:", vllm.__version__)
        print("Torch:", torch.__version__)
        print("CUDA available:", torch.cuda.is_available())
        print("CUDA runtime:", torch.version.cuda)
        print("FlashInfer:", getattr(flashinfer, "__version__", "installed"))
  
        if vllm.__version__ != "0.25.1":
            raise RuntimeError(f"Expected vLLM 0.25.1, found {vllm.__version__}")
        if not torch.cuda.is_available():
            raise RuntimeError("CUDA is not available inside the container")
        PY
  
        if [ ! -f /home/darkmatter2222/models/Laguna-S-2.1-NVFP4/config.json ]; then
          echo "ERROR: Model was not found at /home/darkmatter2222/models/Laguna-S-2.1-NVFP4"
          echo "Check the host bind mount and model directory name."
          exit 10
        fi
  
        echo
        echo "Starting Laguna S 2.1 on port 8006..."
        exec /opt/vllm025/bin/vllm serve /home/darkmatter2222/models/Laguna-S-2.1-NVFP4 \
          --served-model-name laguna-s-2.1 \
          --host 0.0.0.0 \
          --port 8006 \
          --trust-remote-code \
          --dtype auto \
          --safetensors-load-strategy lazy \
          --gpu-memory-utilization 0.70 \
          --max-model-len 131072 \
          --max-num-seqs 1 \
          --max-num-batched-tokens 4096 \
          --enable-chunked-prefill \
          --enable-auto-tool-choice \
          --tool-call-parser poolside_v1 \
          --reasoning-parser poolside_v1 \
          --override-generation-config '{"temperature":0.7,"top_p":0.95}' \
          --kv-cache-dtype fp8 \
          --enforce-eager
  
    healthcheck:
      test:
        - CMD-SHELL
        - >-
          /opt/vllm025/bin/python -c
          "import urllib.request;
          urllib.request.urlopen('http://127.0.0.1:8006/health', timeout=5)"
      interval: 30s
      timeout: 10s
      retries: 10
      start_period: 30m

volumes:
  laguna_vllm025_env:
  laguna_huggingface_cache:
  laguna_vllm_cache:
  laguna_flashinfer_cache:
  laguna_torch_cache: