deploy/docker-compose.hy3.yml

services:
  vllm-hy3:
    image: ${HY3_VLLM_IMAGE:-vllm/vllm-openai:v0.23.0-aarch64-cu129}
    network_mode: host
    ipc: host
    shm_size: "64gb"
    # hard cap so a marlin repack overcommit gets the CONTAINER OOM-killed, not the box
    mem_limit: "112g"
    ulimits:
      memlock: -1
      stack: 67108864
    gpus: all
    devices:
      - /dev/infiniband:/dev/infiniband
    volumes:
      - ${HF_CACHE:-${HOME}/.cache/huggingface}:/root/.cache/huggingface
      - ${HY3_MODEL_DIR:-/home/jon/models/Hy3-preview-NVFP4}:/models/hy3:ro
    environment:
      HF_HOME: /root/.cache/huggingface
      VLLM_CACHE_ROOT: /root/.cache/huggingface/vllm-cache
      # proven NVFP4-on-GB10 path (ASUS box): stock cu129 image + marlin
      # Path 1: force the purpose-built sm_121 CuTe-DSL MoE kernel (vLLM PR #40082)
      # via --moe-backend flashinfer_cutedsl below. Do NOT set
      # VLLM_USE_FLASHINFER_MOE_FP4=0 (that removes the flashinfer backends incl cutedsl).
      VLLM_HOST_IP: "${VLLM_HOST_IP:-}"
      VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
      NCCL_NET: "${NCCL_NET:-IB}"
      NCCL_IB_DISABLE: "${NCCL_IB_DISABLE:-0}"
      NCCL_IB_HCA: "${NCCL_IB_HCA}"
      NCCL_SOCKET_IFNAME: "${NCCL_SOCKET_IFNAME}"
      # vLLM 0.23.0's multinode Gloo coord group binds localhost unless told the
      # fabric iface (older dspark image didn't need this).
      GLOO_SOCKET_IFNAME: "${NCCL_SOCKET_IFNAME}"
      NCCL_IB_GID_INDEX: "${NCCL_IB_GID_INDEX:-}"
      NCCL_CROSS_NIC: "${NCCL_CROSS_NIC:-1}"
      NCCL_CUMEM_ENABLE: "${NCCL_CUMEM_ENABLE:-0}"
      NCCL_IGNORE_CPU_AFFINITY: "${NCCL_IGNORE_CPU_AFFINITY:-1}"
      NCCL_DEBUG: "${NCCL_DEBUG:-WARN}"
      NCCL_NVLS_ENABLE: "${NCCL_NVLS_ENABLE:-0}"
      NODE_RANK: "${NODE_RANK}"
      HEADLESS: "${HEADLESS:-}"
      MASTER_ADDR: "${MASTER_ADDR}"
      MASTER_PORT: "${MASTER_PORT:-25000}"
    entrypoint:
      - bash
      - -lc
    command:
      - >
        exec vllm serve /models/hy3
        --served-model-name ${SERVED_MODEL_NAME:-hy3-nvfp4} hy3-preview-nvfp4 deepseek-v4-flash-dspark
        --host ${VLLM_HOST:-0.0.0.0}
        --port 8888
        --trust-remote-code
        --tensor-parallel-size 2
        --pipeline-parallel-size 1
        --moe-backend marlin
        --kv-cache-dtype fp8
        --max-model-len ${MAX_MODEL_LEN:-65536}
        --max-num-seqs ${MAX_NUM_SEQS:-4}
        --max-num-batched-tokens ${MAX_NUM_BATCHED_TOKENS:-8192}
        --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION:-0.90}
        --enforce-eager
        --enable-prefix-caching
        --tool-call-parser hy_v3
        --reasoning-parser hy_v3
        --enable-auto-tool-choice
        --distributed-executor-backend mp
        --nnodes 2
        --node-rank ${NODE_RANK}
        --master-addr ${MASTER_ADDR}
        --master-port ${MASTER_PORT:-25000}
        ${HEADLESS:+--headless}