more tests

.github/workflows/test.yml

@@ -58,6 +58,13 @@ jobs:
         python test_deviations.py
         echo "status=$?" >> $GITHUB_OUTPUT
 
+    - name: Run PromptGuard scanner test
+      id: test_prompt_guard
+      continue-on-error: true
+      run: |
+        python test_prompt_guard_scanner.py
+        echo "status=$?" >> $GITHUB_OUTPUT
+
     - name: Run AlignmentCheck dual dimensions test (optional)
       id: test_alignment_dual_dimensions
       continue-on-error: true
@@ -122,6 +129,49 @@ jobs:
           exit $TEST_RESULT
         fi
 
+    - name: Run Native LlamaFirewall scanner test (optional)
+      id: test_native_llamafirewall
+      continue-on-error: true
+      env:
+        TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+      run: |
+        if [ -z "$TOGETHER_API_KEY" ]; then
+          echo "⏭️ Skipping Native LlamaFirewall test (TOGETHER_API_KEY not configured)"
+          echo "outcome=skipped" >> $GITHUB_OUTPUT
+          exit 0
+        fi
+        echo "Running Native LlamaFirewall scanner test..."
+        python test_native_llamafirewall_scanner.py
+        TEST_RESULT=$?
+        if [ $TEST_RESULT -eq 0 ]; then
+          echo "outcome=success" >> $GITHUB_OUTPUT
+        else
+          echo "outcome=failure" >> $GITHUB_OUTPUT
+          exit $TEST_RESULT
+        fi
+
+    - name: Run FactsChecker scanner test (optional)
+      id: test_facts_checker
+      continue-on-error: true
+      env:
+        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      run: |
+        if [ -z "$OPENAI_API_KEY" ]; then
+          echo "⏭️ Skipping FactsChecker test (OPENAI_API_KEY not configured)"
+          echo "   To enable: Add OPENAI_API_KEY to GitHub repository secrets"
+          echo "outcome=skipped" >> $GITHUB_OUTPUT
+          exit 0
+        fi
+        echo "Running FactsChecker scanner test (7 tests including temporal awareness)..."
+        python test_facts_checker_scanner.py
+        TEST_RESULT=$?
+        if [ $TEST_RESULT -eq 0 ]; then
+          echo "outcome=success" >> $GITHUB_OUTPUT
+        else
+          echo "outcome=failure" >> $GITHUB_OUTPUT
+          exit $TEST_RESULT
+        fi
+
     - name: Collect test results
       id: results
       if: always()
@@ -129,7 +179,7 @@ jobs:
         FAILED=0
         PASSED=0
         SKIPPED=0
-        TOTAL=4
+        TOTAL=5
         FAILED_TESTS=""
 
         # Core tests (always run)
@@ -161,6 +211,13 @@ jobs:
           PASSED=$((PASSED + 1))
         fi
 
+        if [ "${{ steps.test_prompt_guard.outcome }}" != "success" ]; then
+          FAILED=$((FAILED + 1))
+          FAILED_TESTS="${FAILED_TESTS}• test_prompt_guard_scanner.py\n"
+        else
+          PASSED=$((PASSED + 1))
+        fi
+
         # Optional tests (AlignmentCheck - requires API key)
         # Test 1: AlignmentCheck dual dimensions
         ALIGNMENT_DUAL_OUTCOME="${{ steps.test_alignment_dual_dimensions.outputs.outcome }}"
@@ -201,6 +258,32 @@ jobs:
           FAILED_TESTS="${FAILED_TESTS}• test_alignment_check.py\n"
         fi
 
+        # Test 4: Native LlamaFirewall scanner test
+        NATIVE_LLAMAFIREWALL_OUTCOME="${{ steps.test_native_llamafirewall.outputs.outcome }}"
+        if [ "$NATIVE_LLAMAFIREWALL_OUTCOME" == "skipped" ]; then
+          SKIPPED=$((SKIPPED + 1))
+        elif [ "$NATIVE_LLAMAFIREWALL_OUTCOME" == "success" ]; then
+          PASSED=$((PASSED + 1))
+          TOTAL=$((TOTAL + 1))
+        elif [ "$NATIVE_LLAMAFIREWALL_OUTCOME" == "failure" ]; then
+          FAILED=$((FAILED + 1))
+          TOTAL=$((TOTAL + 1))
+          FAILED_TESTS="${FAILED_TESTS}• test_native_llamafirewall_scanner.py\n"
+        fi
+
+        # Test 5: FactsChecker scanner test
+        FACTS_CHECKER_OUTCOME="${{ steps.test_facts_checker.outputs.outcome }}"
+        if [ "$FACTS_CHECKER_OUTCOME" == "skipped" ]; then
+          SKIPPED=$((SKIPPED + 1))
+        elif [ "$FACTS_CHECKER_OUTCOME" == "success" ]; then
+          PASSED=$((PASSED + 1))
+          TOTAL=$((TOTAL + 1))
+        elif [ "$FACTS_CHECKER_OUTCOME" == "failure" ]; then
+          FAILED=$((FAILED + 1))
+          TOTAL=$((TOTAL + 1))
+          FAILED_TESTS="${FAILED_TESTS}• test_facts_checker_scanner.py\n"
+        fi
+
         # If no tests failed, set a default message
         if [ -z "$FAILED_TESTS" ]; then
           FAILED_TESTS="None"
@@ -282,7 +365,7 @@ jobs:
                 "type": "section",
                 "text": {
                   "type": "mrkdwn",
-                  "text": "*Core Tests:*\n• test_data_disclosure_fix.py: ${{ steps.test_disclosure_fix.outcome }}\n• test_alignment_fix.py: ${{ steps.test_alignment_fix.outcome }}\n• test_user_provided_notification_contact.py: ${{ steps.test_notification_contact.outcome }}\n• test_deviations.py: ${{ steps.test_deviations.outcome }}\n\n*Optional Tests (require TOGETHER_API_KEY):*\n• test_alignment_dual_dimensions.py: ${{ steps.test_alignment_dual_dimensions.outputs.outcome }}\n• test_alignment_vs_factchecker.py: ${{ steps.test_alignment_vs_factchecker.outputs.outcome }}\n• test_alignment_check.py: ${{ steps.test_alignment_check.outputs.outcome }}"
+                  "text": "*Core Tests (5):*\n• test_data_disclosure_fix.py: ${{ steps.test_disclosure_fix.outcome }}\n• test_alignment_fix.py: ${{ steps.test_alignment_fix.outcome }}\n• test_user_provided_notification_contact.py: ${{ steps.test_notification_contact.outcome }}\n• test_deviations.py: ${{ steps.test_deviations.outcome }}\n• test_prompt_guard_scanner.py: ${{ steps.test_prompt_guard.outcome }}\n\n*Optional Tests (require API keys):*\n• test_alignment_dual_dimensions.py: ${{ steps.test_alignment_dual_dimensions.outputs.outcome }} (TOGETHER_API_KEY)\n• test_alignment_vs_factchecker.py: ${{ steps.test_alignment_vs_factchecker.outputs.outcome }} (TOGETHER_API_KEY)\n• test_alignment_check.py: ${{ steps.test_alignment_check.outputs.outcome }} (TOGETHER_API_KEY)\n• test_native_llamafirewall_scanner.py: ${{ steps.test_native_llamafirewall.outputs.outcome }} (TOGETHER_API_KEY)\n• test_facts_checker_scanner.py: ${{ steps.test_facts_checker.outputs.outcome }} (OPENAI_API_KEY)"
                 }
               },
               {

GITHUB_ACTIONS_TESTS.md

@@ -0,0 +1,124 @@
+# GitHub Actions Test Suite
+
+## Overview
+The test suite runs automatically on every push to `main` and on pull requests.
+
+## Test Count: 10 Tests Total
+
+### Core Tests (5) - Always Run
+These tests run without requiring API keys:
+1. ✅ `test_data_disclosure_fix.py` - DataDisclosureGuard false positive test
+2. ✅ `test_alignment_fix.py` - DataDisclosureGuard alignment test
+3. ✅ `test_user_provided_notification_contact.py` - User-provided notification contact test
+4. ✅ `test_deviations.py` - Deviation/bias detection test
+5. ✅ `test_prompt_guard_scanner.py` - PromptGuard scanner pattern-based detection (no API key needed)
+
+### Optional Tests (5) - Require API Keys
+These tests only run when the required API keys are configured in GitHub Secrets:
+
+**Requires TOGETHER_API_KEY:**
+6. ⚙️ `test_alignment_dual_dimensions.py` - AlignmentCheck dual dimensions test
+7. ⚙️ `test_alignment_vs_factchecker.py` - AlignmentCheck vs FactChecker separation test
+8. ⚙️ `test_alignment_check.py` - AlignmentCheck scanner test
+9. ⚙️ `test_native_llamafirewall_scanner.py` - Native LlamaFirewall scanner test
+
+**Requires OPENAI_API_KEY:**
+10. ⚙️ `test_facts_checker_scanner.py` - FactsChecker scanner test (7 subtests including temporal awareness)
+
+## Required GitHub Secrets
+
+To run all 10 tests, configure these secrets in your repository:
+
+### 1. TOGETHER_API_KEY ⚠️ REQUIRED FOR 4 TESTS
+- Used by: AlignmentCheck tests (6, 7, 8) and Native LlamaFirewall test (9)
+- Provider: [Together AI](https://api.together.xyz/)
+- Purpose: Powers the AlignmentCheck scanner for behavioral drift detection
+
+### 2. OPENAI_API_KEY ⚠️ REQUIRED FOR 1 TEST
+- Used by: FactsChecker test (10)
+- Provider: [OpenAI](https://platform.openai.com/)
+- Purpose: Powers NeMo GuardRails fact-checking with GPT-4o-mini
+
+## How to Add Secrets
+
+1. Go to your GitHub repository
+2. Navigate to: **Settings → Secrets and variables → Actions**
+3. Click **"New repository secret"**
+4. Add each secret:
+   - Name: `TOGETHER_API_KEY`
+   - Value: Your Together AI API key
+   - Click "Add secret"
+
+   - Name: `OPENAI_API_KEY`
+   - Value: Your OpenAI API key
+   - Click "Add secret"
+
+## Test Results
+
+The workflow will:
+- ✅ **Pass**: If all running tests succeed (skipped tests don't cause failure)
+- ❌ **Fail**: If any running test fails
+- ⏭️ **Skip**: Optional tests without required API keys
+
+### Slack Notifications
+
+Both success and failure notifications are sent to Slack with:
+- Total tests: X/10
+- Tests passed, failed, and skipped
+- Detailed per-test status with outcomes
+- Links to workflow run and commit
+
+## Adding New Tests
+
+When adding a new test file, follow these steps:
+
+1. **Create the test file** (e.g., `test_new_scanner.py`)
+
+2. **Update `.github/workflows/test.yml`:**
+   - Add test execution step (with or without API key check)
+   - Update "Collect test results" section to count the new test
+   - Update Slack notification to list the new test
+   - Update `TOTAL=X` count if it's a core test
+
+3. **Update this document:**
+   - Add test to the appropriate section (Core or Optional)
+   - Document any new required secrets
+   - Update the test count in the title
+
+4. **Notify the team:**
+   - If new secrets are required, inform repository admins
+   - Document which provider and why the secret is needed
+
+## Example: Adding a Test That Requires a New Secret
+
+If you add a test requiring `ANTHROPIC_API_KEY`:
+
+```yaml
+- name: Run Claude scanner test (optional)
+  id: test_claude_scanner
+  continue-on-error: true
+  env:
+    ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+  run: |
+    if [ -z "$ANTHROPIC_API_KEY" ]; then
+      echo "⏭️ Skipping Claude scanner test (ANTHROPIC_API_KEY not configured)"
+      echo "outcome=skipped" >> $GITHUB_OUTPUT
+      exit 0
+    fi
+    python test_claude_scanner.py
+    TEST_RESULT=$?
+    if [ $TEST_RESULT -eq 0 ]; then
+      echo "outcome=success" >> $GITHUB_OUTPUT
+    else
+      echo "outcome=failure" >> $GITHUB_OUTPUT
+      exit $TEST_RESULT
+    fi
+```
+
+Then notify: "⚠️ New test added requiring `ANTHROPIC_API_KEY` from Claude AI"
+
+---
+
+**Last Updated:** 2026-02-07
+**Total Tests:** 10 (5 core + 5 optional)
+**Required Secrets:** 2 (TOGETHER_API_KEY, OPENAI_API_KEY)