| name: stable_test | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| repo_org: | |
| required: false | |
| description: 'Tested repository organization name. Default is InternLM' | |
| type: string | |
| default: 'InternLM/lmdeploy' | |
| repo_ref: | |
| required: false | |
| description: 'Set branch or tag or commit id. Default is "main"' | |
| type: string | |
| default: 'main' | |
| offline_mode: | |
| required: true | |
| description: 'Whether start a offline mode, if true, you should prepare code and whl package by yourself' | |
| type: boolean | |
| default: false | |
| schedule: | |
| - cron: '00 8 * * 1' | |
| env: | |
| HOST_PIP_CACHE_DIR: /nvme/github-actions/pip-cache | |
| HOST_LOCALTIME: /usr/share/zoneinfo/Asia/Shanghai | |
| OUTPUT_FOLDER: cuda11.8_dist_${{ github.run_id }} | |
| REPORT_DIR: /nvme/qa_test_models/stable_reports/${{ github.run_id }} | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| COMPASS_DATA_CACHE: /nvme/qa_test_models/dataset | |
| jobs: | |
| linux-build: | |
| if: ${{github.event_name == 'schedule' || (!cancelled() && !inputs.offline_mode)}} | |
| strategy: | |
| matrix: | |
| pyver: [py310] | |
| runs-on: ubuntu-latest | |
| env: | |
| PYTHON_VERSION: ${{ matrix.pyver }} | |
| PLAT_NAME: manylinux2014_x86_64 | |
| DOCKER_TAG: cuda11.8 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ github.event.inputs.repo_org || 'InternLM/lmdeploy' }} | |
| ref: ${{github.event.inputs.repo_ref || 'main'}} | |
| - name: Build | |
| run: | | |
| echo ${PYTHON_VERSION} | |
| echo ${PLAT_NAME} | |
| echo ${DOCKER_TAG} | |
| echo ${OUTPUT_FOLDER} | |
| echo ${GITHUB_RUN_ID} | |
| # remove -it | |
| sed -i 's/docker run --rm -it/docker run --rm/g' builder/manywheel/build_wheel.sh | |
| bash builder/manywheel/build_wheel.sh ${PYTHON_VERSION} ${PLAT_NAME} ${DOCKER_TAG} ${OUTPUT_FOLDER} | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| if-no-files-found: error | |
| path: builder/manywheel/${{ env.OUTPUT_FOLDER }} | |
| retention-days: 1 | |
| name: my-artifact-${{ github.run_id }}-${{ matrix.pyver }} | |
| benchmark: | |
| needs: linux-build | |
| if: ${{github.event_name == 'schedule' || !cancelled()}} | |
| runs-on: [self-hosted, lmdeploy-stable] | |
| timeout-minutes: 10080 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| model: ['internlm/internlm2_5-20b-chat'] | |
| container: | |
| image: openmmlab/lmdeploy:latest-cu11 | |
| options: "--gpus=all --ipc=host --user root -e PIP_CACHE_DIR=/root/.cache/pip -e NVIDIA_DISABLE_REQUIRE=1 -e NO_PROXY=localhost,127.0.0.1 -e no_proxy=localhost,127.0.0.1 --pull never" | |
| volumes: | |
| - /nvme/github-actions/pip-cache:/root/.cache/pip | |
| - /nvme/github-actions/packages:/root/packages | |
| - /nvme/qa_test_models:/nvme/qa_test_models | |
| - /mnt/187:/mnt/187 | |
| - /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro | |
| steps: | |
| - name: Clone repository | |
| uses: actions/checkout@v3 | |
| if: ${{github.event_name == 'schedule' || !inputs.offline_mode}} | |
| with: | |
| repository: ${{ github.event.inputs.repo_org || 'InternLM/lmdeploy' }} | |
| ref: ${{github.event.inputs.repo_ref || 'main'}} | |
| - name: Copy repository - offline | |
| if: ${{inputs.offline_mode}} | |
| run: cp -r /nvme/qa_test_models/offline_pkg/lmdeploy/. . | |
| - name: Download Artifacts | |
| if: ${{github.event_name == 'schedule' || !inputs.offline_mode}} | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: my-artifact-${{ github.run_id }}-py310 | |
| - name: Install lmdeploy - dependency | |
| run: | | |
| # manually install flash attn | |
| # the install packeage from. https://github.com/Dao-AILab/flash-attention/releases | |
| python3 -m pip install /root/packages/flash_attn-2.6.3+cu118torch2.3cxx11abiFALSE-cp310-cp310-linux_x86_64.whl | |
| python3 -m pip install /root/packages/xformers-0.0.27+cu118-cp310-cp310-manylinux2014_x86_64.whl --no-deps | |
| python3 -m pip install -r /nvme/qa_test_models/offline_pkg/requirements.txt | |
| - name: Install lmdeploy | |
| if: ${{github.event_name == 'schedule' || !inputs.offline_mode}} | |
| run: | | |
| python3 -m pip install lmdeploy-*.whl --no-deps | |
| python3 -m pip install -r requirements/test.txt | |
| - name: Install lmdeploy - offline | |
| if: ${{inputs.offline_mode}} | |
| run: | | |
| python3 -m pip install /nvme/qa_test_models/offline_pkg/py310/lmdeploy-*.whl --no-deps | |
| python3 -m pip install -r requirements/test.txt | |
| - name: Install opencompass | |
| run: | | |
| git clone --depth=1 https://github.com/open-compass/opencompass.git | |
| cd opencompass | |
| python3 -m pip install -e . | |
| cd .. | |
| - name: Check env | |
| run: | | |
| python3 -m pip list | |
| lmdeploy check_env | |
| - name: Start restful api turbomind | |
| run: | | |
| mkdir ${{env.REPORT_DIR}} -p | |
| CUDA_VISIBLE_DEVICES=6,7 lmdeploy serve api_server /nvme/qa_test_models/${{matrix.model}} --tp 2 --max-batch-size 256 --cache-max-entry-count 0.9 --server-port 23344 > ${{env.REPORT_DIR}}/restful.log 2>&1 & | |
| echo "restful_pid=$!" >> "$GITHUB_ENV" | |
| sleep 120s | |
| - name: Run OC result | |
| continue-on-error: true | |
| run: | | |
| ln -s /nvme/qa_test_models/dataset/data . | |
| opencompass .github/scripts/eval_stable_object_config.py --reuse --dump-eval-details --work-dir ${{env.REPORT_DIR}}-object-1 | |
| opencompass .github/scripts/eval_stable_subject_config.py --reuse --dump-eval-details --work-dir ${{env.REPORT_DIR}}-subject-1 | |
| opencompass .github/scripts/eval_stable_object_config.py --reuse --dump-eval-details --work-dir ${{env.REPORT_DIR}}-object-2 | |
| opencompass .github/scripts/eval_stable_subject_config.py --reuse --dump-eval-details --work-dir ${{env.REPORT_DIR}}-subject-2 | |
| opencompass .github/scripts/eval_stable_object_config.py --reuse --dump-eval-details --work-dir ${{env.REPORT_DIR}}-object-3 | |
| opencompass .github/scripts/eval_stable_subject_config.py --reuse --dump-eval-details --work-dir ${{env.REPORT_DIR}}-subject-3 | |
| - name: Test lmdeploy - restful api | |
| run: | | |
| python3 benchmark/profile_restful_api.py --backend lmdeploy --base-url http://localhost:23344 --dataset-path /nvme/qa_test_models/datasets/ShareGPT_V3_unfiltered_cleaned_split.json --num-prompts 10000 --output-file ${{env.REPORT_DIR}}/stable.jsonl > ${{env.REPORT_DIR}}/stable.log | |
| python3 /nvme/qa_test_models/offline_pkg/profile_restful_api_internal.py localhost:23344 /nvme/qa_test_models/${{matrix.model}} /nvme/qa_test_models/datasets/Mixed.json --stream-output True --num-prompts 100000 --csv ${{env.REPORT_DIR}}/stable-internal-1.csv > ${{env.REPORT_DIR}}/stable-internal-1.log | |
| python3 /nvme/qa_test_models/offline_pkg/profile_restful_api_internal.py localhost:23344 /nvme/qa_test_models/${{matrix.model}} /nvme/qa_test_models/datasets/Mixed.json --stream-output True --num-prompts 100000 --csv ${{env.REPORT_DIR}}/stable-internal-2.csv > ${{env.REPORT_DIR}}/stable-internal-2.log | |
| python3 /nvme/qa_test_models/offline_pkg/profile_restful_api_internal.py localhost:23344 /nvme/qa_test_models/${{matrix.model}} /nvme/qa_test_models/datasets/Mixed.json --stream-output True --num-prompts 100000 --csv ${{env.REPORT_DIR}}/stable-internal-3.csv > ${{env.REPORT_DIR}}/stable-internal-3.log | |
| python3 /nvme/qa_test_models/offline_pkg/profile_restful_api_internal.py localhost:23344 /nvme/qa_test_models/${{matrix.model}} /nvme/qa_test_models/datasets/Mixed.json --stream-output True --num-prompts 100000 --csv ${{env.REPORT_DIR}}/stable-internal-2.csv > ${{env.REPORT_DIR}}/stable-internal-4.log | |
| python3 /nvme/qa_test_models/offline_pkg/profile_restful_api_internal.py localhost:23344 /nvme/qa_test_models/${{matrix.model}} /nvme/qa_test_models/datasets/Mixed.json --stream-output True --num-prompts 100000 --csv ${{env.REPORT_DIR}}/stable-internal-3.csv > ${{env.REPORT_DIR}}/stable-internal-5.log | |
| - name: Attach result | |
| if: always() | |
| run: | | |
| python3 .github/scripts/action_tools.py generate_csv_from_profile_result ${{env.REPORT_DIR}}/stable.jsonl ${{env.REPORT_DIR}}/stable.csv | |
| python3 .github/scripts/action_tools.py add_summary ${{env.REPORT_DIR}}/stable.csv | |
| python3 .github/scripts/action_tools.py add_summary ${{env.REPORT_DIR}}/stable-internal-1.csv | |
| python3 .github/scripts/action_tools.py add_summary ${{env.REPORT_DIR}}/stable-internal-2.csv | |
| python3 .github/scripts/action_tools.py add_summary ${{env.REPORT_DIR}}/stable-internal-3.csv | |
| python3 .github/scripts/action_tools.py add_summary ${{env.REPORT_DIR}}/stable-internal-4.csv | |
| python3 .github/scripts/action_tools.py add_summary ${{env.REPORT_DIR}}/stable-internal-5.csv | |
| - name: Kill api server | |
| if: always() | |
| run: | | |
| kill -15 "$restful_pid" | |
| - name: Clear workfile | |
| if: always() | |
| run: | | |
| chmod -R 777 $REPORT_DIR | |
| export workdir=$(pwd) | |
| cd .. | |
| rm -rf $workdir | |
| mkdir $workdir | |
| chmod -R 777 $workdir | |