lmdeploy/.github/workflows/benchmark.yml

name: benchmark_test

on:
  workflow_dispatch:
    inputs:
      repo_org:
        required: false
        description: 'Tested repository organization name. Default is InternLM'
        type: string
        default: 'InternLM/lmdeploy'
      repo_ref:
        required: false
        description: 'Set branch or tag or commit id. Default is "main"'
        type: string
        default: 'main'
      benchmark_type:
        required: true
        description: 'Set benchmark type. Default is "["longtext", "throughput", "api_server", "prefixcache"]"'
        type: string
        default: "['apiserver', 'mllm_apiserver', 'throughput', 'longtext', 'prefixcache']"
      backend:
        required: true
        description: 'Set backend filter. Default is "["turbomind", "pytorch"]"'
        type: string
        default: "['turbomind', 'pytorch']"
      offline_mode:
        required: true
        description: 'Whether start a offline mode, if true, you should prepare code and whl package by yourself'
        type: boolean
        default: false
      docker_tag:
        required: true
        description: 'Docker tag'
        type: string
        default: 'latest-cu12.8'
      result_tag:
        required: true
        description: 'result_tag if is not none, benchmark results will be uploaded to feishu'
        type: string
        default: "default"

env:
  HOST_PIP_CACHE_DIR: /nvme/github-actions/pip-cache
  HOST_LOCALTIME: /usr/share/zoneinfo/Asia/Shanghai
  OUTPUT_FOLDER: cuda12.8_dist_${{ github.run_id }}
  REPORT_DIR: /nvme/qa_test_models/benchmark_report/${{ inputs.repo_ref }}_${{ github.run_id }}
  ALLURE_REPORT_DIR: /nvme/qa_test_models/benchmark_report/allure_report/${{ inputs.repo_ref }}_${{ github.run_id }}
  TEST_CODE_PATH: /nvme/qa_test_models/test_pkg/lmdeploy/${{ inputs.repo_ref }}_${{ github.run_id }}
  OFFLINE_CODE_PATH: /nvme/qa_test_models/offline_pkg/lmdeploy
  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
  RUN_ID: ${{ inputs.repo_ref }}_${{ github.run_id }}

jobs:
  linux-build:
    if: ${{github.event_name == 'schedule' || (!cancelled() && !inputs.offline_mode)}}
    strategy:
      matrix:
        pyver: [py312]
    runs-on: ubuntu-latest
    env:
      PYTHON_VERSION: ${{ matrix.pyver }}
      PLAT_NAME: manylinux2014_x86_64
      DOCKER_TAG: cuda12.8
    steps:
      - name: Free disk space
        uses: jlumbroso/free-disk-space@main
        with:
          # This might remove tools that are actually needed, if set to "true" but frees about 6 GB
          tool-cache: false
          docker-images: false
          # All of these default to true, but feel free to set to "false" if necessary for your workflow
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          swap-storage: false
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          repository: ${{ github.event.inputs.repo_org || 'InternLM/lmdeploy' }}
          ref: ${{github.event.inputs.repo_ref || 'main'}}
      - name: Build
        run: |
          echo ${PYTHON_VERSION}
          echo ${PLAT_NAME}
          echo ${DOCKER_TAG}
          echo ${OUTPUT_FOLDER}
          echo ${GITHUB_RUN_ID}
          # remove -it
          sed -i 's/docker run --rm -it/docker run --rm/g' builder/manywheel/build_wheel.sh
          bash builder/manywheel/build_wheel.sh ${PYTHON_VERSION} ${PLAT_NAME} ${DOCKER_TAG} ${OUTPUT_FOLDER}
      - name: Upload Artifacts
        uses: actions/upload-artifact@v4
        with:
          if-no-files-found: error
          path: builder/manywheel/${{ env.OUTPUT_FOLDER }}
          retention-days: 1
          name: my-artifact-${{ github.run_id }}-${{ matrix.pyver }}

  download_pkgs:
    needs: linux-build
    if: ${{!cancelled()}}
    runs-on: [self-hosted, linux-a100]
    timeout-minutes: 50
    container:
      image: openmmlab/lmdeploy:${{ inputs.docker_tag || 'latest-cu12.8' }}
      options: "--gpus=all --ipc=host --user root -e PIP_CACHE_DIR=/root/.cache/pip -e NVIDIA_DISABLE_REQUIRE=1 --pull never"
      volumes:
        - /nvme/qa_test_models:/nvme/qa_test_models
        - /mnt/121:/mnt/121
        - /mnt/104:/mnt/104
        - /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro
    steps:
      - name: Clone repository
        uses: actions/checkout@v2
        if: ${{github.event_name == 'schedule' || !inputs.offline_mode}}
        with:
          repository: ${{ github.event.inputs.repo_org || 'InternLM/lmdeploy' }}
          ref: ${{github.event.inputs.repo_ref || 'main'}}
      - name: Copy repository
        if: ${{github.event_name == 'schedule' || !inputs.offline_mode}}
        run: rm -rf ${{env.TEST_CODE_PATH}} && mkdir ${{env.TEST_CODE_PATH}} && chmod 777 ${{env.TEST_CODE_PATH}} && cp -r . ${{env.TEST_CODE_PATH}}
      - name: Copy repository - offline
        if: ${{inputs.offline_mode}}
        run: rm -rf ${{env.TEST_CODE_PATH}} && mkdir ${{env.TEST_CODE_PATH}} && chmod 777 ${{env.TEST_CODE_PATH}} && cp -r ${{env.OFFLINE_CODE_PATH}}/. ${{env.TEST_CODE_PATH}}
      - name: Download Artifacts
        if: ${{github.event_name == 'schedule' || !inputs.offline_mode}}
        uses: actions/download-artifact@v4
        with:
          name: my-artifact-${{ github.run_id }}-py312
      - name: Copy Artifacts
        if: ${{github.event_name == 'schedule' || !inputs.offline_mode}}
        run: rm ${{env.TEST_CODE_PATH}}/lmdeploy-*.whl -f && cp lmdeploy-*.whl ${{env.TEST_CODE_PATH}}
      - name: Copy Artifacts - offline
        if: ${{inputs.offline_mode}}
        run: rm ${{env.TEST_CODE_PATH}}/lmdeploy-*.whl -f && cp ${{env.OFFLINE_CODE_PATH}}/lmdeploy-*.whl ${{env.TEST_CODE_PATH}}
      - name: Mark as start
        run: |
          chmod -R 777 ${{env.TEST_CODE_PATH}}
          mkdir ${{env.REPORT_DIR}} -p
          echo "starttime=$(date +%s)" > ${{env.REPORT_DIR}}/status.txt

  benchmark:
    needs: download_pkgs
    if: ${{github.event_name == 'schedule' || !cancelled()}}
    runs-on: [self-hosted, linux-a100]
    strategy:
      fail-fast: false
      matrix:
        benchmark_type: ${{fromJSON(github.event.inputs.benchmark_type)}}
        gpu_num: ['gpu_num_1', 'gpu_num_2', 'gpu_num_4', 'gpu_num_8']
        transformers: ["", "legacy"]
        include:
          - n: 8
            gpu_num: gpu_num_1
          - n: 4
            gpu_num: gpu_num_2
          - n: 2
            gpu_num: gpu_num_4
          - n: 1
            gpu_num: gpu_num_8
    env:
      TEST_ENV: ${{ matrix.transformers }}
    timeout-minutes: 480
    container:
      image: openmmlab/lmdeploy:${{ inputs.docker_tag || 'latest-cu12.8' }}
      options: "--gpus=all --ipc=host --user root -e PIP_CACHE_DIR=/root/.cache/pip -e NVIDIA_DISABLE_REQUIRE=1 --pull never"
      volumes:
        - /nvme/github-actions/pip-cache:/root/.cache/pip
        - /nvme/github-actions/packages:/root/packages
        - /nvme/qa_test_models:/nvme/qa_test_models
        - /nvme/huggingface_hub:/nvme/huggingface_hub
        - /mnt/121:/mnt/121
        - /mnt/104:/mnt/104
        - /mnt/bigdisk:/mnt/bigdisk
        - /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro
    steps:
      - name: Copy repository and Artifacts
        run: |
          cp -r ${{env.TEST_CODE_PATH}}/. .
          mkdir ${{env.REPORT_DIR}} -p
          echo "starttime=$(date +%s)" > ${{env.REPORT_DIR}}/status.txt
      - name: Install lmdeploy - dependency
        run: |
          python3 -m pip install -r /nvme/qa_test_models/offline_pkg/requirements.txt
      - name: Install lmdeploy
        run: |
          python3 -m pip uninstall lmdeploy -y && python3 -m pip install lmdeploy-*.whl --no-deps
          python3 -m pip install -r requirements/test.txt
      - name: Downgrade transformers
        if: ${{matrix.transformers == 'legacy'}}
        run: |
          pip install transformers==4.57.6
      - name: Check env
        run: |
          python3 -m pip list
          lmdeploy check_env
      - name: Run other benchmark - all
        if: contains(fromJson(github.event.inputs.backend), 'turbomind') && contains(fromJson(github.event.inputs.backend), 'pytorch')
        run: |
            pytest autotest/benchmark/test_${{matrix.benchmark_type}}_performance.py -n ${{matrix.n}} -m '${{matrix.gpu_num}} and not pr_test and not function' --alluredir=${{env.ALLURE_REPORT_DIR}}
      - name: Run other benchmark - turbomind
        if: contains(fromJson(github.event.inputs.backend), 'turbomind') && !contains(fromJson(github.event.inputs.backend), 'pytorch')
        run: |
            pytest autotest/benchmark/test_${{matrix.benchmark_type}}_performance.py -n ${{matrix.n}} -m '${{matrix.gpu_num}} and not pr_test and not function and turbomind' --alluredir=${{env.ALLURE_REPORT_DIR}}
      - name: Run other benchmark - pytorch
        if: contains(fromJson(github.event.inputs.backend), 'pytorch') && !contains(fromJson(github.event.inputs.backend), 'turbomind')
        run: |
            pytest autotest/benchmark/test_${{matrix.benchmark_type}}_performance.py -n ${{matrix.n}} -m '${{matrix.gpu_num}} and not pr_test and not function and pytorch' --alluredir=${{env.ALLURE_REPORT_DIR}}
      - name: Generate result
        if: always()
        run: |
            cd /nvme/qa_test_models/feishu_upload
            python3 test_benchmark.py --root ${{env.REPORT_DIR}} --output ${{env.REPORT_DIR}}/${{inputs.result_tag}}.txt --hardware A100 --infer-version ${{inputs.result_tag}}
      - name: Async result
        if: always() && inputs.result_tag != 'default'
        env:
          FEISHU_APP_ID: ${{secrets.FEISHU_APP_ID}}
          FEISHU_APP_SECRET: ${{secrets.FEISHU_APP_SECRET}}
          FEISHU_TABLE_TOKEN: ${{secrets.FEISHU_TABLE_TOKEN}}
          FEISHU_TABLE_ID: ${{secrets.BENCHMARK_FEISHU_TABLE_ID}}
        run: |
            cd /nvme/qa_test_models/feishu_upload
            python3 main.py --skip-duplicates ${{env.REPORT_DIR}}/${{inputs.result_tag}}.txt --config config-benchmark.py
      - name: Clear workfile
        if: always()
        run: |
          echo "status=done" >> ${{env.REPORT_DIR}}/status.txt
          export workdir=$(pwd)
          cd ..
          rm -rf $workdir
          mkdir $workdir
          chmod -R 777 $workdir