models/deploy.sh

#!/usr/bin/env bash
set -euo pipefail

: "${BASE_MODEL_PATH:=/models/Qwen3.5-35B-A3B}"
: "${SERVED_MODEL_NAME:=Qwen3.5-35B-A3B}"
: "${VLLM_HOST:=0.0.0.0}"
: "${VLLM_PORT:=8001}"
: "${CUDA_VISIBLE_DEVICES:=0,1,2,3,4,5,6,7}"

export CUDA_VISIBLE_DEVICES

python -m vllm.entrypoints.openai.api_server \
  --model "${BASE_MODEL_PATH}" \
  --served-model-name "${SERVED_MODEL_NAME}" \
  --tensor-parallel-size 8 \
  --enable-expert-parallel \
  --gpu-memory-utilization 0.90 \
  --host "${VLLM_HOST}" \
  --port "${VLLM_PORT}" \
  --max-num-seqs 64 \
  --max-model-len 262144 \
  --enable-chunked-prefill \
  --enable-auto-tool-choice \
  --tool-call-parser qwen3_coder