File size: 801 Bytes
994182c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | #!/usr/bin/env bash
set -euo pipefail
MODEL_NAME="${MODEL_NAME:-Qwen/Qwen3.6-27B}"
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-qwen36-base}"
HOST="${HOST:-0.0.0.0}"
PORT="${PORT:-8000}"
DTYPE="${DTYPE:-bfloat16}"
TP_SIZE="${TP_SIZE:-1}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}"
VLLM_EXTRA_ARGS="${VLLM_EXTRA_ARGS:---trust-remote-code}"
echo "Serving base model with vLLM:"
echo " model: $MODEL_NAME"
echo " served name: $SERVED_MODEL_NAME"
echo " endpoint: http://$HOST:$PORT/v1"
echo " tensor para: $TP_SIZE"
echo
python -m vllm.entrypoints.openai.api_server \
--model "$MODEL_NAME" \
--served-model-name "$SERVED_MODEL_NAME" \
--host "$HOST" \
--port "$PORT" \
--dtype "$DTYPE" \
--tensor-parallel-size "$TP_SIZE" \
--max-model-len "$MAX_MODEL_LEN" \
$VLLM_EXTRA_ARGS
|