lm-evaluation-harness/.github/workflows/unit_tests.yml

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
# just comment out unwanted steps to turn off the test.
name: Unit Tests

on:
  push:
    branches:
      - 'main'
  pull_request:
    branches:
      - 'main'
  workflow_dispatch:
# Jobs run concurrently and steps run sequentially within a job.
# jobs: linter and cpu_tests. Add more jobs/steps as required.
jobs:
  linter:
    name: Linters
    runs-on: ubuntu-latest
    timeout-minutes: 5

    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Set up Python 3.9
      uses: actions/setup-python@v5
      with:
        python-version: 3.9
        cache: pip
        cache-dependency-path: pyproject.toml
    - name: Pre-Commit
      env:
        SKIP: "no-commit-to-branch,mypy"

      uses: pre-commit/action@v3.0.1
#       # mypy turned off for now
#    - name: Lint with mypy
#      run: mypy . --ignore-missing-imports --check-untyped-defs --explicit-package-bases --warn-unreachable
# Job 2
  testcpu:
    name: CPU Tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12" ]
    timeout-minutes: 30
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        cache: pip
        cache-dependency-path: pyproject.toml
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e '.[dev,sentencepiece,api]' --extra-index-url https://download.pytorch.org/whl/cpu
#         Install optional git dependencies
#                pip install bleurt@https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt
#        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
    - name: Test with pytest
      run: python -m pytest --showlocals -s -vv -n=auto --ignore=tests/models/test_neuralmagic.py --ignore=tests/models/test_openvino.py
    - name: Archive artifacts
      uses: actions/upload-artifact@v4
      with:
        name: output_testcpu${{ matrix.python-version }}
        path: |
          test_logs/*
  testmodels:
    name: External LM Tests
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Set up Python 3.9
      uses: actions/setup-python@v5
      with:
        python-version: 3.9
        cache: pip
        cache-dependency-path: pyproject.toml
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e '.[dev,optimum,deepsparse,sparseml,api]' --extra-index-url https://download.pytorch.org/whl/cpu
        pip install -U transformers peft
    - name: Test with pytest
      run: python -m pytest tests/models --showlocals -s -vv