File size: 2,847 Bytes
370d886 386d766 370d886 386d766 370d886 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | services:
vllm-hy3:
image: ${HY3_VLLM_IMAGE:-vllm/vllm-openai:v0.23.0-aarch64-cu129}
network_mode: host
ipc: host
shm_size: "64gb"
# hard cap so a marlin repack overcommit gets the CONTAINER OOM-killed, not the box
mem_limit: "112g"
ulimits:
memlock: -1
stack: 67108864
gpus: all
devices:
- /dev/infiniband:/dev/infiniband
volumes:
- ${HF_CACHE:-${HOME}/.cache/huggingface}:/root/.cache/huggingface
- ${HY3_MODEL_DIR:-/home/jon/models/Hy3-preview-NVFP4}:/models/hy3:ro
environment:
HF_HOME: /root/.cache/huggingface
VLLM_CACHE_ROOT: /root/.cache/huggingface/vllm-cache
# proven NVFP4-on-GB10 path (ASUS box): stock cu129 image + marlin
# Path 1: force the purpose-built sm_121 CuTe-DSL MoE kernel (vLLM PR #40082)
# via --moe-backend flashinfer_cutedsl below. Do NOT set
# VLLM_USE_FLASHINFER_MOE_FP4=0 (that removes the flashinfer backends incl cutedsl).
VLLM_HOST_IP: "${VLLM_HOST_IP:-}"
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
NCCL_NET: "${NCCL_NET:-IB}"
NCCL_IB_DISABLE: "${NCCL_IB_DISABLE:-0}"
NCCL_IB_HCA: "${NCCL_IB_HCA}"
NCCL_SOCKET_IFNAME: "${NCCL_SOCKET_IFNAME}"
# vLLM 0.23.0's multinode Gloo coord group binds localhost unless told the
# fabric iface (older dspark image didn't need this).
GLOO_SOCKET_IFNAME: "${NCCL_SOCKET_IFNAME}"
NCCL_IB_GID_INDEX: "${NCCL_IB_GID_INDEX:-}"
NCCL_CROSS_NIC: "${NCCL_CROSS_NIC:-1}"
NCCL_CUMEM_ENABLE: "${NCCL_CUMEM_ENABLE:-0}"
NCCL_IGNORE_CPU_AFFINITY: "${NCCL_IGNORE_CPU_AFFINITY:-1}"
NCCL_DEBUG: "${NCCL_DEBUG:-WARN}"
NCCL_NVLS_ENABLE: "${NCCL_NVLS_ENABLE:-0}"
NODE_RANK: "${NODE_RANK}"
HEADLESS: "${HEADLESS:-}"
MASTER_ADDR: "${MASTER_ADDR}"
MASTER_PORT: "${MASTER_PORT:-25000}"
entrypoint:
- bash
- -lc
command:
- >
exec vllm serve /models/hy3
--served-model-name ${SERVED_MODEL_NAME:-hy3-nvfp4} hy3-preview-nvfp4 deepseek-v4-flash-dspark
--host ${VLLM_HOST:-0.0.0.0}
--port 8888
--trust-remote-code
--tensor-parallel-size 2
--pipeline-parallel-size 1
--moe-backend marlin
--kv-cache-dtype fp8
--max-model-len ${MAX_MODEL_LEN:-65536}
--max-num-seqs ${MAX_NUM_SEQS:-4}
--max-num-batched-tokens ${MAX_NUM_BATCHED_TOKENS:-8192}
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION:-0.90}
--enforce-eager
--enable-prefix-caching
--tool-call-parser hy_v3
--reasoning-parser hy_v3
--enable-auto-tool-choice
--distributed-executor-backend mp
--nnodes 2
--node-rank ${NODE_RANK}
--master-addr ${MASTER_ADDR}
--master-port ${MASTER_PORT:-25000}
${HEADLESS:+--headless}
|