scripts/yans/lm-evaluation-harness/.github/workflows/new_tasks.yml

name: Tasks Modified

on:
  push:
    branches:
      - 'main'
  pull_request:
    branches:
      - 'main'
  workflow_dispatch:
# comment/edit out the above to stop/change the triggers
jobs:
  changed_files:
    runs-on: ubuntu-latest  # windows-latest || macos-latest
    timeout-minutes: 120
    name: Scan for changed tasks
    steps:
      - name: checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 2  # OR "2" -> To retrieve the preceding commit.

      # Uses the tj-actions/changed-files action to check for changes.
      # Outputs provided here: https://github.com/tj-actions/changed-files#outputs
      # The `files_yaml` input optionally takes a yaml string to specify filters,
      # and prepends the filter name to the standard output names.
      - name: Check task folders
        id: changed-tasks
        uses: tj-actions/changed-files@v44.5.2
        with:
          # tasks checks the tasks folder and api checks the api folder for changes
          files_yaml: |
            tasks:
              - lm_eval/tasks/**
            api:
              - lm_eval/api/**
          write_output_files: true

    # The next step is optional; the files are written to the workspace by default (above).
    # so it's just for debugging
      - name: Run Tests
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
        run: |
          echo .github/outputs/tasks_all_changed_and_modified_files.txt >> 'GITHUB_ENV'
          echo "One or more test file(s) has changed."
          echo "List of all the files that have changed: ${{ steps.changed-tasks.outputs.tasks_all_modified_files }}"

      - name: Set up Python 3.9
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
        uses: actions/setup-python@v4
        with:
          python-version: 3.9
          cache: 'pip'
          cache-dependency-path: setup.py
      - name: Install dependencies
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
        run: |
            python -m pip install --upgrade pip
            pip install -e '.[dev,ifeval]' --extra-index-url https://download.pytorch.org/whl/cpu
    #   Install optional git dependencies
    #       pip install bleurt@https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt
    #       if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
      - name: Test with pytest
        # if new tasks are added, run tests on them
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true'
        run: python -m pytest tests/test_tasks.py -s -vv
        # if api is modified, run tests on it
      - name: Test more tasks with pytest
        env:
          API: true
        if: steps.changed-tasks.outputs.api_any_modified == 'true'
        run: python -m pytest tests/test_tasks.py -s -vv