.github/workflows/test.yml

name: Test Suite

on:
  push:
    branches:
      - main
      - newUI
  pull_request:
    branches:
      - main
      - newUI

jobs:
  test:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'

    - name: Install uv
      uses: astral-sh/setup-uv@v4

    - name: Install dependencies
      run: |
        # uv is 10-100x faster than pip for package resolution and installation
        uv pip install --system -r requirements_minimal.txt llamafirewall

        # Fix NeMo GuardRails + OpenAI compatibility issue
        uv pip install --system nemoguardrails presidio-analyzer presidio-anonymizer

        # Force compatible versions AFTER all installs (later installs can upgrade deps)
        # nemoguardrails pulls langchain 1.x which is incompatible with 0.2.x sub-packages
        uv pip install --system --reinstall --no-deps \
          "openai==1.54.0" \
          "langchain==0.2.16" \
          "langchain-core==0.2.43" \
          "langchain-community==0.2.16" \
          "langchain-openai==0.1.25"

        # llamafirewall needs HfFolder (removed in huggingface_hub 1.x)
        # Let uv resolve the full dependency tree (transformers, tokenizers, etc.)
        uv pip install --system \
          "huggingface_hub==0.26.2" \
          "transformers==4.46.3"
        uv pip install --system tiktoken "httpx<0.28.0"

        # Verify versions and compatibility
        echo "Verifying package versions..."
        python -c "import openai; print(f'✓ OpenAI: {openai.__version__}')"
        python -c "import langchain; print(f'✓ langchain: {langchain.__version__}')"
        python -c "import langchain_core; print(f'✓ langchain-core: {langchain_core.__version__}')"
        python -c "import langchain_community; print(f'✓ langchain-community: {langchain_community.__version__}')"
        python -c "import langchain_openai; print('✓ langchain-openai: importable')"
        python -c "import httpx; print(f'✓ httpx: {httpx.__version__}')"
        python -c "import tiktoken; print(f'✓ tiktoken: {tiktoken.__version__}')"
        python -c "import openai; assert openai.__version__ < '1.58.0', 'OpenAI version too new!'"
        python -c "import nemoguardrails; print('✓ NeMo GuardRails importable')"
        python -c "import llamafirewall; print('✓ LlamaFirewall importable')"
        python -c "from langchain_openai import ChatOpenAI; print('✓ ChatOpenAI loadable')"
        echo "✅ All package versions compatible"

    - name: Run DataDisclosureGuard false positive test
      id: test_disclosure_fix
      continue-on-error: true
      run: |
        set +e
        python test_data_disclosure_fix.py 2>&1 | tee /tmp/test_output.txt
        EXIT_CODE=${PIPESTATUS[0]}
        set -e
        COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1)
        if [ -n "$COUNTS" ]; then
          echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT
          echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT
        else
          echo "passed=0" >> $GITHUB_OUTPUT
          echo "total=1" >> $GITHUB_OUTPUT
        fi
        exit $EXIT_CODE

    - name: Run DataDisclosureGuard alignment test
      id: test_alignment_fix
      continue-on-error: true
      env:
        TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
      run: |
        set +e
        python test_alignment_fix.py 2>&1 | tee /tmp/test_output.txt
        EXIT_CODE=${PIPESTATUS[0]}
        set -e
        COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1)
        if [ -n "$COUNTS" ]; then
          echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT
          echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT
        else
          echo "passed=0" >> $GITHUB_OUTPUT
          echo "total=1" >> $GITHUB_OUTPUT
        fi
        exit $EXIT_CODE

    - name: Run user-provided notification contact test
      id: test_notification_contact
      continue-on-error: true
      run: |
        set +e
        python test_user_provided_notification_contact.py 2>&1 | tee /tmp/test_output.txt
        EXIT_CODE=${PIPESTATUS[0]}
        set -e
        COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1)
        if [ -n "$COUNTS" ]; then
          echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT
          echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT
        else
          echo "passed=0" >> $GITHUB_OUTPUT
          echo "total=1" >> $GITHUB_OUTPUT
        fi
        exit $EXIT_CODE

    - name: Run deviation/bias detection test
      id: test_deviations
      continue-on-error: true
      run: |
        set +e
        python test_deviations.py 2>&1 | tee /tmp/test_output.txt
        EXIT_CODE=${PIPESTATUS[0]}
        set -e
        COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1)
        if [ -n "$COUNTS" ]; then
          echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT
          echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT
        else
          echo "passed=0" >> $GITHUB_OUTPUT
          echo "total=1" >> $GITHUB_OUTPUT
        fi
        exit $EXIT_CODE

    - name: Run PromptGuard scanner test
      id: test_prompt_guard
      continue-on-error: true
      run: |
        set +e
        python test_prompt_guard_scanner.py 2>&1 | tee /tmp/test_output.txt
        EXIT_CODE=${PIPESTATUS[0]}
        set -e
        COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1)
        if [ -n "$COUNTS" ]; then
          echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT
          echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT
        else
          echo "passed=0" >> $GITHUB_OUTPUT
          echo "total=1" >> $GITHUB_OUTPUT
        fi
        exit $EXIT_CODE

    - name: Run AlignmentCheck dual dimensions test
      id: test_alignment_dual_dimensions
      continue-on-error: true
      env:
        TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
      run: |
        set +e
        python test_alignment_dual_dimensions.py 2>&1 | tee /tmp/test_output.txt
        EXIT_CODE=${PIPESTATUS[0]}
        set -e
        COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1)
        if [ -n "$COUNTS" ]; then
          echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT
          echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT
        else
          echo "passed=0" >> $GITHUB_OUTPUT
          echo "total=1" >> $GITHUB_OUTPUT
        fi
        exit $EXIT_CODE

    - name: Run AlignmentCheck vs FactChecker separation test
      id: test_alignment_vs_factchecker
      continue-on-error: true
      env:
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      run: |
        set +e
        python test_alignment_vs_factchecker.py 2>&1 | tee /tmp/test_output.txt
        EXIT_CODE=${PIPESTATUS[0]}
        set -e
        COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1)
        if [ -n "$COUNTS" ]; then
          echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT
          echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT
        else
          echo "passed=0" >> $GITHUB_OUTPUT
          echo "total=1" >> $GITHUB_OUTPUT
        fi
        exit $EXIT_CODE

    - name: Run AlignmentCheck scanner test
      id: test_alignment_check
      continue-on-error: true
      env:
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      run: |
        set +e
        python test_alignment_check.py 2>&1 | tee /tmp/test_output.txt
        EXIT_CODE=${PIPESTATUS[0]}
        set -e
        COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1)
        if [ -n "$COUNTS" ]; then
          echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT
          echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT
        else
          echo "passed=0" >> $GITHUB_OUTPUT
          echo "total=1" >> $GITHUB_OUTPUT
        fi
        exit $EXIT_CODE

    - name: Run Native LlamaFirewall scanner test
      id: test_native_llamafirewall
      continue-on-error: true
      env:
        TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
      run: |
        set +e
        python test_native_llamafirewall_scanner.py 2>&1 | tee /tmp/test_output.txt
        EXIT_CODE=${PIPESTATUS[0]}
        set -e
        COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1)
        if [ -n "$COUNTS" ]; then
          echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT
          echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT
        else
          echo "passed=0" >> $GITHUB_OUTPUT
          echo "total=1" >> $GITHUB_OUTPUT
        fi
        exit $EXIT_CODE

    - name: Run FactsChecker scanner test
      id: test_facts_checker
      continue-on-error: true
      env:
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      run: |
        set +e
        python test_facts_checker_scanner.py 2>&1 | tee /tmp/test_output.txt
        EXIT_CODE=${PIPESTATUS[0]}
        set -e
        COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1)
        if [ -n "$COUNTS" ]; then
          echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT
          echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT
        else
          echo "passed=0" >> $GITHUB_OUTPUT
          echo "total=1" >> $GITHUB_OUTPUT
        fi
        exit $EXIT_CODE

    - name: Collect test results
      id: results
      if: always()
      run: |
        TOTAL_PASSED=0
        TOTAL_SCENARIOS=0
        FAILED_FILES=""

        # Helper: add counts from a test step
        add_counts() {
          local file_name="$1"
          local step_passed="$2"
          local step_total="$3"
          local step_outcome="$4"

          # Default to 0 if empty (step was skipped)
          step_passed=${step_passed:-0}
          step_total=${step_total:-0}

          TOTAL_PASSED=$((TOTAL_PASSED + step_passed))
          TOTAL_SCENARIOS=$((TOTAL_SCENARIOS + step_total))

          if [ "$step_outcome" != "success" ]; then
            local step_failed=$((step_total - step_passed))
            FAILED_FILES="${FAILED_FILES}• ${file_name} (${step_passed}/${step_total} scenarios passed)\n"
          fi
        }

        # Collect counts from each test step
        add_counts "test_data_disclosure_fix.py" "${{ steps.test_disclosure_fix.outputs.passed }}" "${{ steps.test_disclosure_fix.outputs.total }}" "${{ steps.test_disclosure_fix.outcome }}"
        add_counts "test_alignment_fix.py" "${{ steps.test_alignment_fix.outputs.passed }}" "${{ steps.test_alignment_fix.outputs.total }}" "${{ steps.test_alignment_fix.outcome }}"
        add_counts "test_user_provided_notification_contact.py" "${{ steps.test_notification_contact.outputs.passed }}" "${{ steps.test_notification_contact.outputs.total }}" "${{ steps.test_notification_contact.outcome }}"
        add_counts "test_deviations.py" "${{ steps.test_deviations.outputs.passed }}" "${{ steps.test_deviations.outputs.total }}" "${{ steps.test_deviations.outcome }}"
        add_counts "test_prompt_guard_scanner.py" "${{ steps.test_prompt_guard.outputs.passed }}" "${{ steps.test_prompt_guard.outputs.total }}" "${{ steps.test_prompt_guard.outcome }}"
        add_counts "test_alignment_dual_dimensions.py" "${{ steps.test_alignment_dual_dimensions.outputs.passed }}" "${{ steps.test_alignment_dual_dimensions.outputs.total }}" "${{ steps.test_alignment_dual_dimensions.outcome }}"
        add_counts "test_alignment_vs_factchecker.py" "${{ steps.test_alignment_vs_factchecker.outputs.passed }}" "${{ steps.test_alignment_vs_factchecker.outputs.total }}" "${{ steps.test_alignment_vs_factchecker.outcome }}"
        add_counts "test_alignment_check.py" "${{ steps.test_alignment_check.outputs.passed }}" "${{ steps.test_alignment_check.outputs.total }}" "${{ steps.test_alignment_check.outcome }}"
        add_counts "test_native_llamafirewall_scanner.py" "${{ steps.test_native_llamafirewall.outputs.passed }}" "${{ steps.test_native_llamafirewall.outputs.total }}" "${{ steps.test_native_llamafirewall.outcome }}"
        add_counts "test_facts_checker_scanner.py" "${{ steps.test_facts_checker.outputs.passed }}" "${{ steps.test_facts_checker.outputs.total }}" "${{ steps.test_facts_checker.outcome }}"

        TOTAL_FAILED=$((TOTAL_SCENARIOS - TOTAL_PASSED))

        if [ -z "$FAILED_FILES" ]; then
          FAILED_FILES="None"
        fi

        echo "passed=$TOTAL_PASSED" >> $GITHUB_OUTPUT
        echo "failed=$TOTAL_FAILED" >> $GITHUB_OUTPUT
        echo "total=$TOTAL_SCENARIOS" >> $GITHUB_OUTPUT
        echo "failed_tests=$FAILED_FILES" >> $GITHUB_OUTPUT

        echo "Results: $TOTAL_PASSED/$TOTAL_SCENARIOS test scenarios passed ($TOTAL_FAILED failed)"
        if [ $TOTAL_FAILED -gt 0 ]; then
          echo "❌ Failed test files:"
          echo -e "$FAILED_FILES"
          exit 1
        fi

    - name: Notify Slack on failure
      if: failure()
      uses: slackapi/slack-github-action@v1.26.0
      with:
        payload: |
          {
            "text": "❌ Test Suite Failed on main branch",
            "blocks": [
              {
                "type": "header",
                "text": {
                  "type": "plain_text",
                  "text": "❌ AI Agent Guards - Test Failure",
                  "emoji": true
                }
              },
              {
                "type": "section",
                "fields": [
                  {
                    "type": "mrkdwn",
                    "text": "*Repository:*\n${{ github.repository }}"
                  },
                  {
                    "type": "mrkdwn",
                    "text": "*Branch:*\n${{ github.ref_name }}"
                  },
                  {
                    "type": "mrkdwn",
                    "text": "*Commit:*\n<${{ github.event.head_commit.url }}|${{ github.sha }}>"
                  },
                  {
                    "type": "mrkdwn",
                    "text": "*Author:*\n${{ github.event.head_commit.author.name }}"
                  }
                ]
              },
              {
                "type": "section",
                "fields": [
                  {
                    "type": "mrkdwn",
                    "text": "*Scenarios Passed:* ${{ steps.results.outputs.passed }}/${{ steps.results.outputs.total }}"
                  },
                  {
                    "type": "mrkdwn",
                    "text": "*Scenarios Failed:* ${{ steps.results.outputs.failed }}/${{ steps.results.outputs.total }}"
                  }
                ]
              },
              {
                "type": "section",
                "text": {
                  "type": "mrkdwn",
                  "text": "*All Tests:*\n• test_data_disclosure_fix.py: ${{ steps.test_disclosure_fix.outcome }}\n• test_alignment_fix.py: ${{ steps.test_alignment_fix.outcome }}\n• test_user_provided_notification_contact.py: ${{ steps.test_notification_contact.outcome }}\n• test_deviations.py: ${{ steps.test_deviations.outcome }}\n• test_prompt_guard_scanner.py: ${{ steps.test_prompt_guard.outcome }}\n• test_alignment_dual_dimensions.py: ${{ steps.test_alignment_dual_dimensions.outcome }}\n• test_alignment_vs_factchecker.py: ${{ steps.test_alignment_vs_factchecker.outcome }}\n• test_alignment_check.py: ${{ steps.test_alignment_check.outcome }}\n• test_native_llamafirewall_scanner.py: ${{ steps.test_native_llamafirewall.outcome }}\n• test_facts_checker_scanner.py: ${{ steps.test_facts_checker.outcome }}"
                }
              },
              {
                "type": "actions",
                "elements": [
                  {
                    "type": "button",
                    "text": {
                      "type": "plain_text",
                      "text": "View Workflow Run",
                      "emoji": true
                    },
                    "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                  },
                  {
                    "type": "button",
                    "text": {
                      "type": "plain_text",
                      "text": "View Commit",
                      "emoji": true
                    },
                    "url": "${{ github.event.head_commit.url }}"
                  }
                ]
              }
            ]
          }
      env:
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK