File size: 2,931 Bytes
54dd322 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 | # coding=utf-8
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export VLLM_ENGINE_ITERATION_TIMEOUT_S=1000
timestamp=$(date +"%Y%m%d_%H%M%S")
export LOCAL_CKPT_DIR=${LOAD_CKPT_DIR:=../}
export LOCAL_CHAT_TEMPLATE_PATH=${CHAT_TEMPLATE_PATH:=$SCRIPT_DIR/chat_template_74B_1124.jinja}
if [[ ! -d ${OUTPUT_TEXT_DIR} ]]; then
export OUTPUT_TEXT_DIR=$PROJECT_DIR/outputs
mkdir -p ${OUTPUT_TEXT_DIR}
fi
apply_patch_to_package() {
local PACKAGE_NAME=$1
local PATCH_FILE=$2
local PACKAGE_DIR
echo "apply patch ${PATCH_FILE} to ${PACKAGE_NAME}..."
PACKAGE_DIR=$(python -m pip show "$PACKAGE_NAME" | grep 'Editable project location' | awk -F': ' '{print $2}')
(
cd "$PACKAGE_DIR" || exit 1
if git apply --reverse --check "$PATCH_FILE" >/dev/null 2>&1; then
echo "Patch for ${PACKAGE_NAME} already applied. Skipping."
return 0
fi
if git apply --check "$PATCH_FILE" >/dev/null 2>&1; then
git apply "$PATCH_FILE"
if [ $? -ne 0 ]; then
echo "Failed to apply patch to ${PACKAGE_NAME} due to git apply error."
exit 1
fi
else
echo "Failed to apply patch to ${PACKAGE_NAME}. Patch file is either incorrect or conflicts exist."
exit 1
fi
) || exit 1
}
apply_patch_to_package "vllm-ascend" "$SCRIPT_DIR/vllm-ascend_v0.11.0rc0.patch"
apply_patch_to_package "vllm" "$SCRIPT_DIR/vllm_v0.11.0rc3.patch"
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/omni_custom_ops/bin/set_env.bash
# 指定 HOST=127.0.0.1（本地主机）表示服务器只能从主设备访问。
# 指定 HOST=0.0.0.0 允许从同一网络上的其他设备甚至从互联网访问 vLLM 服务器，前提是网络配置正确（例如，防火墙规则、端口转发）。
HOST=xxx.xxx.xxx.xxx
# 拉起模型服务
python $SCRIPT_DIR/vllm_register.py \
--model $LOCAL_CKPT_DIR \
--served-model-name ${SERVED_MODEL_NAME:=pangu_7b} \
--tensor-parallel-size ${TENSOR_PARALLEL_SIZE:=8} \
--trust-remote-code \
--host ${HOST} \
--port ${PORT:=8000} \
--max-num-seqs ${MAX_NUM_SEQS:=256} \
--max-model-len ${MAX_MODEL_LEN:=131072} \
--tokenizer-mode "slow" \
--dtype bfloat16 \
--enable-log-requests \
--distributed-executor-backend mp \
--gpu-memory-utilization 0.9 \
--max-num-batched-tokens ${MAX_NUM_BATCHED_TOKENS:=4096} \
--no-enable-prefix-caching \
--reasoning-parser pangu \
--chat_template ${LOCAL_CHAT_TEMPLATE_PATH} \
--enable-auto-tool-choice \
--tool-call-parser pangu \
--trust-request-chat-template \
--enable_chunked_prefill \
--additional_config '{"ascend_scheduler_config":{"enabled":true,"enable_chunked_prefill":true,"chunked_prefill_enabled":true}}' | tee $OUTPUT_TEXT_DIR/inference.log |