hf-data-engine/.github/workflows/ci.yml

name: CI/CD Pipeline

on:
  push:
    branches: [ main, develop, claude/* ]
  pull_request:
    branches: [ main, develop ]

jobs:
  code-quality:
    name: Code Quality Checks
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.9'

    - name: Cache dependencies
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements.txt
        pip install black flake8 isort mypy pylint pytest pytest-cov pytest-asyncio

    - name: Run Black (code formatting check)
      run: |
        black --check --diff .

    - name: Run isort (import sorting check)
      run: |
        isort --check-only --diff .

    - name: Run Flake8 (linting)
      run: |
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=100 --statistics

    - name: Run MyPy (type checking)
      run: |
        mypy --install-types --non-interactive --ignore-missing-imports .
      continue-on-error: true  # Don't fail build on type errors initially

    - name: Run Pylint
      run: |
        pylint **/*.py --exit-zero --max-line-length=100
      continue-on-error: true

  test:
    name: Run Tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11']

    steps:
    - uses: actions/checkout@v3

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Cache dependencies
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }}

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements.txt
        pip install pytest pytest-cov pytest-asyncio pytest-timeout

    - name: Run pytest with coverage
      run: |
        pytest tests/ -v --cov=. --cov-report=xml --cov-report=html --cov-report=term

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3
      with:
        file: ./coverage.xml
        flags: unittests
        name: codecov-umbrella
        fail_ci_if_error: false

  security-scan:
    name: Security Scanning
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.9'

    - name: Install security tools
      run: |
        python -m pip install --upgrade pip
        pip install safety bandit

    - name: Run Safety (dependency vulnerability check)
      run: |
        pip install -r requirements.txt
        safety check --json || true

    - name: Run Bandit (security linting)
      run: |
        bandit -r . -f json -o bandit-report.json || true

    - name: Upload security reports
      uses: actions/upload-artifact@v3
      with:
        name: security-reports
        path: |
          bandit-report.json

  docker-build:
    name: Docker Build Test
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v2

    - name: Build Docker image
      run: |
        docker build -t crypto-dt-source:test .

    - name: Test Docker image
      run: |
        docker run --rm crypto-dt-source:test python --version

  integration-tests:
    name: Integration Tests
    runs-on: ubuntu-latest
    needs: [test]

    steps:
    - uses: actions/checkout@v3

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.9'

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements.txt
        pip install pytest pytest-asyncio

    - name: Run integration tests
      run: |
        pytest tests/test_integration.py -v
      env:
        ENABLE_AUTH: false
        LOG_LEVEL: DEBUG

  performance-tests:
    name: Performance Tests
    runs-on: ubuntu-latest
    needs: [test]

    steps:
    - uses: actions/checkout@v3

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.9'

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements.txt
        pip install pytest pytest-benchmark

    - name: Run performance tests
      run: |
        pytest tests/test_performance.py -v --benchmark-only
      continue-on-error: true

  deploy-docs:
    name: Deploy Documentation
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/main'
    needs: [code-quality, test]

    steps:
    - uses: actions/checkout@v3

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.9'

    - name: Install documentation tools
      run: |
        pip install mkdocs mkdocs-material

    - name: Build documentation
      run: |
        # mkdocs build
        echo "Documentation build placeholder"

    - name: Deploy to GitHub Pages
      uses: peaceiris/actions-gh-pages@v3
      if: github.event_name == 'push'
      with:
        github_token: ${{ secrets.GITHUB_TOKEN }}
        publish_dir: ./site
      continue-on-error: true