name: Test Suite on: push: branches: - main - newUI pull_request: branches: - main - newUI jobs: test: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.11' - name: Install uv uses: astral-sh/setup-uv@v4 - name: Install dependencies run: | # uv is 10-100x faster than pip for package resolution and installation uv pip install --system -r requirements_minimal.txt llamafirewall # Fix NeMo GuardRails + OpenAI compatibility issue uv pip install --system nemoguardrails presidio-analyzer presidio-anonymizer # Force compatible versions AFTER all installs (later installs can upgrade deps) # nemoguardrails pulls langchain 1.x which is incompatible with 0.2.x sub-packages uv pip install --system --reinstall --no-deps \ "openai==1.54.0" \ "langchain==0.2.16" \ "langchain-core==0.2.43" \ "langchain-community==0.2.16" \ "langchain-openai==0.1.25" # llamafirewall needs HfFolder (removed in huggingface_hub 1.x) # Let uv resolve the full dependency tree (transformers, tokenizers, etc.) uv pip install --system \ "huggingface_hub==0.26.2" \ "transformers==4.46.3" uv pip install --system tiktoken "httpx<0.28.0" # Verify versions and compatibility echo "Verifying package versions..." python -c "import openai; print(f'✓ OpenAI: {openai.__version__}')" python -c "import langchain; print(f'✓ langchain: {langchain.__version__}')" python -c "import langchain_core; print(f'✓ langchain-core: {langchain_core.__version__}')" python -c "import langchain_community; print(f'✓ langchain-community: {langchain_community.__version__}')" python -c "import langchain_openai; print('✓ langchain-openai: importable')" python -c "import httpx; print(f'✓ httpx: {httpx.__version__}')" python -c "import tiktoken; print(f'✓ tiktoken: {tiktoken.__version__}')" python -c "import openai; assert openai.__version__ < '1.58.0', 'OpenAI version too new!'" python -c "import nemoguardrails; print('✓ NeMo GuardRails importable')" python -c "import llamafirewall; print('✓ LlamaFirewall importable')" python -c "from langchain_openai import ChatOpenAI; print('✓ ChatOpenAI loadable')" echo "✅ All package versions compatible" - name: Run DataDisclosureGuard false positive test id: test_disclosure_fix continue-on-error: true run: | set +e python test_data_disclosure_fix.py 2>&1 | tee /tmp/test_output.txt EXIT_CODE=${PIPESTATUS[0]} set -e COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1) if [ -n "$COUNTS" ]; then echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT else echo "passed=0" >> $GITHUB_OUTPUT echo "total=1" >> $GITHUB_OUTPUT fi exit $EXIT_CODE - name: Run DataDisclosureGuard alignment test id: test_alignment_fix continue-on-error: true env: TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} run: | set +e python test_alignment_fix.py 2>&1 | tee /tmp/test_output.txt EXIT_CODE=${PIPESTATUS[0]} set -e COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1) if [ -n "$COUNTS" ]; then echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT else echo "passed=0" >> $GITHUB_OUTPUT echo "total=1" >> $GITHUB_OUTPUT fi exit $EXIT_CODE - name: Run user-provided notification contact test id: test_notification_contact continue-on-error: true run: | set +e python test_user_provided_notification_contact.py 2>&1 | tee /tmp/test_output.txt EXIT_CODE=${PIPESTATUS[0]} set -e COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1) if [ -n "$COUNTS" ]; then echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT else echo "passed=0" >> $GITHUB_OUTPUT echo "total=1" >> $GITHUB_OUTPUT fi exit $EXIT_CODE - name: Run deviation/bias detection test id: test_deviations continue-on-error: true run: | set +e python test_deviations.py 2>&1 | tee /tmp/test_output.txt EXIT_CODE=${PIPESTATUS[0]} set -e COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1) if [ -n "$COUNTS" ]; then echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT else echo "passed=0" >> $GITHUB_OUTPUT echo "total=1" >> $GITHUB_OUTPUT fi exit $EXIT_CODE - name: Run PromptGuard scanner test id: test_prompt_guard continue-on-error: true run: | set +e python test_prompt_guard_scanner.py 2>&1 | tee /tmp/test_output.txt EXIT_CODE=${PIPESTATUS[0]} set -e COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1) if [ -n "$COUNTS" ]; then echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT else echo "passed=0" >> $GITHUB_OUTPUT echo "total=1" >> $GITHUB_OUTPUT fi exit $EXIT_CODE - name: Run AlignmentCheck dual dimensions test id: test_alignment_dual_dimensions continue-on-error: true env: TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} run: | set +e python test_alignment_dual_dimensions.py 2>&1 | tee /tmp/test_output.txt EXIT_CODE=${PIPESTATUS[0]} set -e COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1) if [ -n "$COUNTS" ]; then echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT else echo "passed=0" >> $GITHUB_OUTPUT echo "total=1" >> $GITHUB_OUTPUT fi exit $EXIT_CODE - name: Run AlignmentCheck vs FactChecker separation test id: test_alignment_vs_factchecker continue-on-error: true env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | set +e python test_alignment_vs_factchecker.py 2>&1 | tee /tmp/test_output.txt EXIT_CODE=${PIPESTATUS[0]} set -e COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1) if [ -n "$COUNTS" ]; then echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT else echo "passed=0" >> $GITHUB_OUTPUT echo "total=1" >> $GITHUB_OUTPUT fi exit $EXIT_CODE - name: Run AlignmentCheck scanner test id: test_alignment_check continue-on-error: true env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | set +e python test_alignment_check.py 2>&1 | tee /tmp/test_output.txt EXIT_CODE=${PIPESTATUS[0]} set -e COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1) if [ -n "$COUNTS" ]; then echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT else echo "passed=0" >> $GITHUB_OUTPUT echo "total=1" >> $GITHUB_OUTPUT fi exit $EXIT_CODE - name: Run Native LlamaFirewall scanner test id: test_native_llamafirewall continue-on-error: true env: TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} run: | set +e python test_native_llamafirewall_scanner.py 2>&1 | tee /tmp/test_output.txt EXIT_CODE=${PIPESTATUS[0]} set -e COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1) if [ -n "$COUNTS" ]; then echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT else echo "passed=0" >> $GITHUB_OUTPUT echo "total=1" >> $GITHUB_OUTPUT fi exit $EXIT_CODE - name: Run FactsChecker scanner test id: test_facts_checker continue-on-error: true env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | set +e python test_facts_checker_scanner.py 2>&1 | tee /tmp/test_output.txt EXIT_CODE=${PIPESTATUS[0]} set -e COUNTS=$(grep -o 'TEST_COUNTS:[0-9]*/[0-9]*' /tmp/test_output.txt | tail -1) if [ -n "$COUNTS" ]; then echo "passed=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f1)" >> $GITHUB_OUTPUT echo "total=$(echo $COUNTS | cut -d: -f2 | cut -d/ -f2)" >> $GITHUB_OUTPUT else echo "passed=0" >> $GITHUB_OUTPUT echo "total=1" >> $GITHUB_OUTPUT fi exit $EXIT_CODE - name: Collect test results id: results if: always() run: | TOTAL_PASSED=0 TOTAL_SCENARIOS=0 FAILED_FILES="" # Helper: add counts from a test step add_counts() { local file_name="$1" local step_passed="$2" local step_total="$3" local step_outcome="$4" # Default to 0 if empty (step was skipped) step_passed=${step_passed:-0} step_total=${step_total:-0} TOTAL_PASSED=$((TOTAL_PASSED + step_passed)) TOTAL_SCENARIOS=$((TOTAL_SCENARIOS + step_total)) if [ "$step_outcome" != "success" ]; then local step_failed=$((step_total - step_passed)) FAILED_FILES="${FAILED_FILES}• ${file_name} (${step_passed}/${step_total} scenarios passed)\n" fi } # Collect counts from each test step add_counts "test_data_disclosure_fix.py" "${{ steps.test_disclosure_fix.outputs.passed }}" "${{ steps.test_disclosure_fix.outputs.total }}" "${{ steps.test_disclosure_fix.outcome }}" add_counts "test_alignment_fix.py" "${{ steps.test_alignment_fix.outputs.passed }}" "${{ steps.test_alignment_fix.outputs.total }}" "${{ steps.test_alignment_fix.outcome }}" add_counts "test_user_provided_notification_contact.py" "${{ steps.test_notification_contact.outputs.passed }}" "${{ steps.test_notification_contact.outputs.total }}" "${{ steps.test_notification_contact.outcome }}" add_counts "test_deviations.py" "${{ steps.test_deviations.outputs.passed }}" "${{ steps.test_deviations.outputs.total }}" "${{ steps.test_deviations.outcome }}" add_counts "test_prompt_guard_scanner.py" "${{ steps.test_prompt_guard.outputs.passed }}" "${{ steps.test_prompt_guard.outputs.total }}" "${{ steps.test_prompt_guard.outcome }}" add_counts "test_alignment_dual_dimensions.py" "${{ steps.test_alignment_dual_dimensions.outputs.passed }}" "${{ steps.test_alignment_dual_dimensions.outputs.total }}" "${{ steps.test_alignment_dual_dimensions.outcome }}" add_counts "test_alignment_vs_factchecker.py" "${{ steps.test_alignment_vs_factchecker.outputs.passed }}" "${{ steps.test_alignment_vs_factchecker.outputs.total }}" "${{ steps.test_alignment_vs_factchecker.outcome }}" add_counts "test_alignment_check.py" "${{ steps.test_alignment_check.outputs.passed }}" "${{ steps.test_alignment_check.outputs.total }}" "${{ steps.test_alignment_check.outcome }}" add_counts "test_native_llamafirewall_scanner.py" "${{ steps.test_native_llamafirewall.outputs.passed }}" "${{ steps.test_native_llamafirewall.outputs.total }}" "${{ steps.test_native_llamafirewall.outcome }}" add_counts "test_facts_checker_scanner.py" "${{ steps.test_facts_checker.outputs.passed }}" "${{ steps.test_facts_checker.outputs.total }}" "${{ steps.test_facts_checker.outcome }}" TOTAL_FAILED=$((TOTAL_SCENARIOS - TOTAL_PASSED)) if [ -z "$FAILED_FILES" ]; then FAILED_FILES="None" fi echo "passed=$TOTAL_PASSED" >> $GITHUB_OUTPUT echo "failed=$TOTAL_FAILED" >> $GITHUB_OUTPUT echo "total=$TOTAL_SCENARIOS" >> $GITHUB_OUTPUT echo "failed_tests=$FAILED_FILES" >> $GITHUB_OUTPUT echo "Results: $TOTAL_PASSED/$TOTAL_SCENARIOS test scenarios passed ($TOTAL_FAILED failed)" if [ $TOTAL_FAILED -gt 0 ]; then echo "❌ Failed test files:" echo -e "$FAILED_FILES" exit 1 fi - name: Notify Slack on failure if: failure() uses: slackapi/slack-github-action@v1.26.0 with: payload: | { "text": "❌ Test Suite Failed on main branch", "blocks": [ { "type": "header", "text": { "type": "plain_text", "text": "❌ AI Agent Guards - Test Failure", "emoji": true } }, { "type": "section", "fields": [ { "type": "mrkdwn", "text": "*Repository:*\n${{ github.repository }}" }, { "type": "mrkdwn", "text": "*Branch:*\n${{ github.ref_name }}" }, { "type": "mrkdwn", "text": "*Commit:*\n<${{ github.event.head_commit.url }}|${{ github.sha }}>" }, { "type": "mrkdwn", "text": "*Author:*\n${{ github.event.head_commit.author.name }}" } ] }, { "type": "section", "fields": [ { "type": "mrkdwn", "text": "*Scenarios Passed:* ${{ steps.results.outputs.passed }}/${{ steps.results.outputs.total }}" }, { "type": "mrkdwn", "text": "*Scenarios Failed:* ${{ steps.results.outputs.failed }}/${{ steps.results.outputs.total }}" } ] }, { "type": "section", "text": { "type": "mrkdwn", "text": "*All Tests:*\n• test_data_disclosure_fix.py: ${{ steps.test_disclosure_fix.outcome }}\n• test_alignment_fix.py: ${{ steps.test_alignment_fix.outcome }}\n• test_user_provided_notification_contact.py: ${{ steps.test_notification_contact.outcome }}\n• test_deviations.py: ${{ steps.test_deviations.outcome }}\n• test_prompt_guard_scanner.py: ${{ steps.test_prompt_guard.outcome }}\n• test_alignment_dual_dimensions.py: ${{ steps.test_alignment_dual_dimensions.outcome }}\n• test_alignment_vs_factchecker.py: ${{ steps.test_alignment_vs_factchecker.outcome }}\n• test_alignment_check.py: ${{ steps.test_alignment_check.outcome }}\n• test_native_llamafirewall_scanner.py: ${{ steps.test_native_llamafirewall.outcome }}\n• test_facts_checker_scanner.py: ${{ steps.test_facts_checker.outcome }}" } }, { "type": "actions", "elements": [ { "type": "button", "text": { "type": "plain_text", "text": "View Workflow Run", "emoji": true }, "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" }, { "type": "button", "text": { "type": "plain_text", "text": "View Commit", "emoji": true }, "url": "${{ github.event.head_commit.url }}" } ] } ] } env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK