diffusers_src/.github/workflows/benchmark.yml

name: Benchmarking tests

on:
  workflow_dispatch:
  schedule:
    - cron: "30 1 1,15 * *" # every 2 weeks on the 1st and the 15th of every month at 1:30 AM

permissions:
  contents: read

env:
  DIFFUSERS_IS_CI: yes
  HF_XET_HIGH_PERFORMANCE: 1
  HF_HOME: /mnt/cache
  OMP_NUM_THREADS: 8
  MKL_NUM_THREADS: 8
  BASE_PATH: benchmark_outputs

jobs:
  torch_models_cuda_benchmark_tests:
    env:
      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_BENCHMARK }}
    name: Torch Core Models CUDA Benchmarking Tests
    strategy:
      fail-fast: false
      max-parallel: 1
    runs-on:
      group: aws-g6e-4xlarge
    container:
      image: diffusers/diffusers-pytorch-cuda
      options: --shm-size "16gb" --ipc host --gpus all
    steps:
      - name: Checkout diffusers
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          fetch-depth: 2
      - name: NVIDIA-SMI
        run: |
          nvidia-smi
      - name: Install dependencies
        run: |
          apt update
          apt install -y libpq-dev postgresql-client
          uv pip install -e ".[quality]"
          uv pip install -r benchmarks/requirements.txt
      - name: Environment
        run: |
          diffusers-cli env
      - name: Diffusers Benchmarking
        env:
          HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
        run: |
          cd benchmarks && python run_all.py

      - name: Push results to the Hub
        env: 
          HF_TOKEN: ${{ secrets.DIFFUSERS_BOT_TOKEN }}
        run: |
          cd benchmarks && python push_results.py
          mkdir $BASE_PATH && cp *.csv $BASE_PATH

      - name: Test suite reports artifacts
        if: ${{ always() }}
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6
        with:
          name: benchmark_test_reports
          path: benchmarks/${{ env.BASE_PATH }}

      - name: Report success status
        if: ${{ success() }}
        run: |
          pip install requests && python utils/notify_benchmarking_status.py --status=success

      - name: Report failure status
        if: ${{ failure() }}
        run: |
          pip install requests && python utils/notify_benchmarking_status.py --status=failure