improve CI: fix tests and better reports to slack

.github/workflows/test.yml

@@ -27,6 +27,9 @@ jobs:
         python -m pip install --upgrade pip
         pip install -r requirements_minimal.txt
         pip install llamafirewall
+        # Pin compatible versions to avoid stream_usage parameter error with NeMo GuardRails
+        # Issue: Recent OpenAI SDK (1.58.0+) has breaking changes with langchain-community
+        pip install "openai<1.58.0"
         pip install nemoguardrails
         pip install presidio-analyzer presidio-anonymizer
 
@@ -111,15 +114,15 @@ jobs:
       id: test_alignment_check
       continue-on-error: true
       env:
-        TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       run: |
-        if [ -z "$TOGETHER_API_KEY" ]; then
-          echo "⏭️ Skipping AlignmentCheck test (TOGETHER_API_KEY not configured)"
-          echo "   To enable: Add TOGETHER_API_KEY to GitHub repository secrets"
+        if [ -z "$OPENAI_API_KEY" ]; then
+          echo "⏭️ Skipping AlignmentCheck test (OPENAI_API_KEY not configured)"
+          echo "   To enable: Add OPENAI_API_KEY to GitHub repository secrets"
           echo "outcome=skipped" >> $GITHUB_OUTPUT
           exit 0
         fi
-        echo "Running AlignmentCheck test with API key..."
+        echo "Running AlignmentCheck test with GPT-4o-mini..."
         python test_alignment_check.py
         TEST_RESULT=$?
         if [ $TEST_RESULT -eq 0 ]; then
@@ -221,12 +224,13 @@ jobs:
         # Optional tests (AlignmentCheck - requires API key)
         # Test 1: AlignmentCheck dual dimensions
         ALIGNMENT_DUAL_OUTCOME="${{ steps.test_alignment_dual_dimensions.outputs.outcome }}"
+        ALIGNMENT_DUAL_STEP_OUTCOME="${{ steps.test_alignment_dual_dimensions.outcome }}"
         if [ "$ALIGNMENT_DUAL_OUTCOME" == "skipped" ]; then
           SKIPPED=$((SKIPPED + 1))
         elif [ "$ALIGNMENT_DUAL_OUTCOME" == "success" ]; then
           PASSED=$((PASSED + 1))
           TOTAL=$((TOTAL + 1))
-        elif [ "$ALIGNMENT_DUAL_OUTCOME" == "failure" ]; then
+        elif [ "$ALIGNMENT_DUAL_OUTCOME" == "failure" ] || [ "$ALIGNMENT_DUAL_STEP_OUTCOME" == "failure" ]; then
           FAILED=$((FAILED + 1))
           TOTAL=$((TOTAL + 1))
           FAILED_TESTS="${FAILED_TESTS}• test_alignment_dual_dimensions.py\n"
@@ -234,12 +238,13 @@ jobs:
 
         # Test 2: AlignmentCheck vs FactChecker separation
         ALIGNMENT_VS_FACTCHECKER_OUTCOME="${{ steps.test_alignment_vs_factchecker.outputs.outcome }}"
+        ALIGNMENT_VS_FACTCHECKER_STEP_OUTCOME="${{ steps.test_alignment_vs_factchecker.outcome }}"
         if [ "$ALIGNMENT_VS_FACTCHECKER_OUTCOME" == "skipped" ]; then
           SKIPPED=$((SKIPPED + 1))
         elif [ "$ALIGNMENT_VS_FACTCHECKER_OUTCOME" == "success" ]; then
           PASSED=$((PASSED + 1))
           TOTAL=$((TOTAL + 1))
-        elif [ "$ALIGNMENT_VS_FACTCHECKER_OUTCOME" == "failure" ]; then
+        elif [ "$ALIGNMENT_VS_FACTCHECKER_OUTCOME" == "failure" ] || [ "$ALIGNMENT_VS_FACTCHECKER_STEP_OUTCOME" == "failure" ]; then
           FAILED=$((FAILED + 1))
           TOTAL=$((TOTAL + 1))
           FAILED_TESTS="${FAILED_TESTS}• test_alignment_vs_factchecker.py\n"
@@ -247,12 +252,13 @@ jobs:
 
         # Test 3: AlignmentCheck scanner test
         ALIGNMENT_OUTCOME="${{ steps.test_alignment_check.outputs.outcome }}"
+        ALIGNMENT_STEP_OUTCOME="${{ steps.test_alignment_check.outcome }}"
         if [ "$ALIGNMENT_OUTCOME" == "skipped" ]; then
           SKIPPED=$((SKIPPED + 1))
         elif [ "$ALIGNMENT_OUTCOME" == "success" ]; then
           PASSED=$((PASSED + 1))
           TOTAL=$((TOTAL + 1))
-        elif [ "$ALIGNMENT_OUTCOME" == "failure" ]; then
+        elif [ "$ALIGNMENT_OUTCOME" == "failure" ] || [ "$ALIGNMENT_STEP_OUTCOME" == "failure" ]; then
           FAILED=$((FAILED + 1))
           TOTAL=$((TOTAL + 1))
           FAILED_TESTS="${FAILED_TESTS}• test_alignment_check.py\n"
@@ -260,12 +266,13 @@ jobs:
 
         # Test 4: Native LlamaFirewall scanner test
         NATIVE_LLAMAFIREWALL_OUTCOME="${{ steps.test_native_llamafirewall.outputs.outcome }}"
+        NATIVE_LLAMAFIREWALL_STEP_OUTCOME="${{ steps.test_native_llamafirewall.outcome }}"
         if [ "$NATIVE_LLAMAFIREWALL_OUTCOME" == "skipped" ]; then
           SKIPPED=$((SKIPPED + 1))
         elif [ "$NATIVE_LLAMAFIREWALL_OUTCOME" == "success" ]; then
           PASSED=$((PASSED + 1))
           TOTAL=$((TOTAL + 1))
-        elif [ "$NATIVE_LLAMAFIREWALL_OUTCOME" == "failure" ]; then
+        elif [ "$NATIVE_LLAMAFIREWALL_OUTCOME" == "failure" ] || [ "$NATIVE_LLAMAFIREWALL_STEP_OUTCOME" == "failure" ]; then
           FAILED=$((FAILED + 1))
           TOTAL=$((TOTAL + 1))
           FAILED_TESTS="${FAILED_TESTS}• test_native_llamafirewall_scanner.py\n"
@@ -273,12 +280,14 @@ jobs:
 
         # Test 5: FactsChecker scanner test
         FACTS_CHECKER_OUTCOME="${{ steps.test_facts_checker.outputs.outcome }}"
+        FACTS_CHECKER_STEP_OUTCOME="${{ steps.test_facts_checker.outcome }}"
         if [ "$FACTS_CHECKER_OUTCOME" == "skipped" ]; then
           SKIPPED=$((SKIPPED + 1))
         elif [ "$FACTS_CHECKER_OUTCOME" == "success" ]; then
           PASSED=$((PASSED + 1))
           TOTAL=$((TOTAL + 1))
-        elif [ "$FACTS_CHECKER_OUTCOME" == "failure" ]; then
+        elif [ "$FACTS_CHECKER_OUTCOME" == "failure" ] || [ "$FACTS_CHECKER_STEP_OUTCOME" == "failure" ]; then
+          # Count as failure if either the output says failure OR the step itself failed
           FAILED=$((FAILED + 1))
           TOTAL=$((TOTAL + 1))
           FAILED_TESTS="${FAILED_TESTS}• test_facts_checker_scanner.py\n"
@@ -365,7 +374,7 @@ jobs:
                 "type": "section",
                 "text": {
                   "type": "mrkdwn",
-                  "text": "*Core Tests (5):*\n• test_data_disclosure_fix.py: ${{ steps.test_disclosure_fix.outcome }}\n• test_alignment_fix.py: ${{ steps.test_alignment_fix.outcome }}\n• test_user_provided_notification_contact.py: ${{ steps.test_notification_contact.outcome }}\n• test_deviations.py: ${{ steps.test_deviations.outcome }}\n• test_prompt_guard_scanner.py: ${{ steps.test_prompt_guard.outcome }}\n\n*Optional Tests (require API keys):*\n• test_alignment_dual_dimensions.py: ${{ steps.test_alignment_dual_dimensions.outputs.outcome }} (TOGETHER_API_KEY)\n• test_alignment_vs_factchecker.py: ${{ steps.test_alignment_vs_factchecker.outputs.outcome }} (TOGETHER_API_KEY)\n• test_alignment_check.py: ${{ steps.test_alignment_check.outputs.outcome }} (TOGETHER_API_KEY)\n• test_native_llamafirewall_scanner.py: ${{ steps.test_native_llamafirewall.outputs.outcome }} (TOGETHER_API_KEY)\n• test_facts_checker_scanner.py: ${{ steps.test_facts_checker.outputs.outcome }} (OPENAI_API_KEY)"
+                  "text": "*Core Tests (5):*\n• test_data_disclosure_fix.py: ${{ steps.test_disclosure_fix.outcome }}\n• test_alignment_fix.py: ${{ steps.test_alignment_fix.outcome }}\n• test_user_provided_notification_contact.py: ${{ steps.test_notification_contact.outcome }}\n• test_deviations.py: ${{ steps.test_deviations.outcome }}\n• test_prompt_guard_scanner.py: ${{ steps.test_prompt_guard.outcome }}\n\n*Optional Tests (require API keys):*\n• test_alignment_dual_dimensions.py: ${{ steps.test_alignment_dual_dimensions.outputs.outcome }} (TOGETHER_API_KEY)\n• test_alignment_vs_factchecker.py: ${{ steps.test_alignment_vs_factchecker.outputs.outcome }} (TOGETHER_API_KEY)\n• test_native_llamafirewall_scanner.py: ${{ steps.test_native_llamafirewall.outputs.outcome }} (TOGETHER_API_KEY)\n• test_alignment_check.py: ${{ steps.test_alignment_check.outputs.outcome }} (OPENAI_API_KEY)\n• test_facts_checker_scanner.py: ${{ steps.test_facts_checker.outputs.outcome }} (OPENAI_API_KEY)"
                 }
               },
               {

GITHUB_ACTIONS_ERROR_FIXES.md

@@ -0,0 +1,218 @@
+# GitHub Actions Error Fixes
+
+This document tracks errors encountered in GitHub Actions tests and their fixes.
+
+## Error #1: NeMo GuardRails - stream_usage Parameter Error
+
+### Error Message
+```
+TypeError: AsyncCompletions.create() got an unexpected keyword argument 'stream_usage'
+WARNING! stream_usage is not default parameter.
+Error invoking LLM (model=gpt-4o-mini): AsyncCompletions.create() got an unexpected keyword argument 'stream_usage'
+```
+
+### Affected Tests
+- `test_facts_checker_scanner.py` (all 7 tests)
+
+### Root Cause
+Version incompatibility between:
+- OpenAI SDK version 1.58.0+ (breaking changes)
+- langchain-community (used by NeMo GuardRails, hasn't updated yet)
+
+### Fix Applied
+**Pinned OpenAI SDK version** in both:
+
+1. `.github/workflows/test.yml`:
+```yaml
+pip install "openai<1.58.0"  # Avoid stream_usage parameter error
+```
+
+2. `requirements_minimal.txt`:
+```python
+openai<1.58.0  # Compatibility with NeMo GuardRails
+```
+
+### Status
+✅ **FIXED** - Tests will now use OpenAI SDK < 1.58.0
+
+### Future Action
+When NeMo GuardRails/langchain-community updates for OpenAI 1.58.0+, remove the version pin.
+
+---
+
+## Error #2: AlignmentCheck Test - Missing 'counts' Key
+
+### Error Message
+```
+❌ CRITICAL ERROR: Test execution failed: 'counts'
+Error: Process completed with exit code 1.
+Overall Decision: None
+```
+
+### Affected Tests
+- `test_alignment_check.py` (all 4 tests)
+
+### Root Cause
+The test was accessing `result['counts']` directly without checking if the key exists. When the scan function returns an error (e.g., API key missing or API failure), the result dictionary only contains `{"error": "...", "scanner": "..."}` without the 'counts' key.
+
+**Test code before fix:**
+```python
+result = scan_alignment_check_per_message(messages=messages, purpose=purpose)
+print(f"Counts: Safe={result['counts']['safe']}, ...")  # KeyError if 'counts' missing!
+```
+
+### Fix Applied
+**Added error checking** in `test_alignment_check.py` for all test functions:
+
+```python
+result = scan_alignment_check_per_message(messages=messages, purpose=purpose)
+
+# Check for errors first
+if "error" in result:
+    print(f"\n❌ ERROR: {result.get('error')}")
+    return False
+
+print(f"\nOverall Decision: {result.get('overall_decision')}")
+counts = result.get('counts', {})  # Safe access with default
+print(f"Counts: Safe={counts.get('safe', 0)}, Warning={counts.get('warning', 0)}, Block={counts.get('block', 0)}")
+```
+
+### Changes Made
+Fixed 3 test functions in `test_alignment_check.py`:
+1. ✅ `test_aligned_conversation()`
+2. ✅ `test_goal_hijacking()`
+3. ✅ `test_off_topic_redirect()`
+
+### Status
+✅ **FIXED** - Tests now handle error cases gracefully
+
+### Why This Happened
+The `scan_alignment_check_per_message()` function returns different result formats:
+
+**Success case:**
+```python
+{
+    "scanner": "AlignmentCheck",
+    "overall_decision": "SAFE",
+    "counts": {"safe": 2, "warning": 0, "block": 0, "total": 2},
+    "message_results": [...]
+}
+```
+
+**Error case:**
+```python
+{
+    "error": "OPENAI_API_KEY not configured",
+    "scanner": "AlignmentCheck"
+}
+```
+
+The test needs to handle both cases.
+
+---
+
+## Error #3: Slack Failure Notifications Not Sent
+
+### Problem
+When tests failed with exceptions (like KeyError), **only success notifications** were sent to Slack, never failure notifications.
+
+### Root Cause
+**Silent failure detection bug** in the workflow:
+
+1. Test runs and crashes with exception (e.g., KeyError accessing `result['counts']`)
+2. Python exits with code 1
+3. Because `continue-on-error: true`, the workflow continues
+4. BUT: The shell script never reaches `echo "outcome=failure" >> $GITHUB_OUTPUT`
+5. The output variable is empty/undefined
+6. "Collect test results" checks: `if [ "$OUTCOME" == "failure" ]`
+7. Empty string != "failure", so FAILED counter is not incremented
+8. FAILED = 0, so workflow succeeds ✅ (incorrectly!)
+9. Only success notification sent, failures silently ignored ❌
+
+**Example of the bug:**
+```yaml
+run: |
+  python test_facts_checker_scanner.py  # Crashes with KeyError
+  TEST_RESULT=$?  # Never reached!
+  echo "outcome=failure" >> $GITHUB_OUTPUT  # Never reached!
+```
+
+### Fix Applied
+**Check BOTH the output AND the step outcome** in the "Collect test results" step:
+
+**Before:**
+```bash
+FACTS_CHECKER_OUTCOME="${{ steps.test_facts_checker.outputs.outcome }}"
+if [ "$FACTS_CHECKER_OUTCOME" == "failure" ]; then
+  FAILED=$((FAILED + 1))
+fi
+```
+
+**After:**
+```bash
+FACTS_CHECKER_OUTCOME="${{ steps.test_facts_checker.outputs.outcome }}"
+FACTS_CHECKER_STEP_OUTCOME="${{ steps.test_facts_checker.outcome }}"
+if [ "$FACTS_CHECKER_OUTCOME" == "failure" ] || [ "$FACTS_CHECKER_STEP_OUTCOME" == "failure" ]; then
+  FAILED=$((FAILED + 1))
+fi
+```
+
+Now we check:
+- ✅ The custom output (if test completed and set it)
+- ✅ The actual step outcome (catches crashes and exceptions)
+
+### Changes Made
+Updated failure detection for all 5 optional tests:
+1. ✅ test_alignment_dual_dimensions.py
+2. ✅ test_alignment_vs_factchecker.py
+3. ✅ test_alignment_check.py
+4. ✅ test_native_llamafirewall_scanner.py
+5. ✅ test_facts_checker_scanner.py
+
+### Status
+✅ **FIXED** - Failures will now be properly detected and reported to Slack
+
+### Why This Is Critical
+Without this fix:
+- ❌ Tests could fail silently
+- ❌ Broken code could be merged
+- ❌ Team wouldn't know about failures
+- ❌ Only saw success messages, creating false confidence
+
+With this fix:
+- ✅ All failures are detected
+- ✅ Slack notifications sent for every failure
+- ✅ Team can respond to issues immediately
+- ✅ Proper CI/CD visibility
+
+---
+
+## Summary
+
+All three critical errors are now fixed:
+
+1. ✅ **NeMo GuardRails compatibility** - Pinned OpenAI SDK version to avoid `stream_usage` parameter error
+2. ✅ **Test error handling** - Added proper error checking in tests to prevent KeyError crashes
+3. ✅ **Slack failure notifications** - Fixed silent failure detection bug so ALL failures are reported
+
+### What Changed
+- **`.github/workflows/test.yml`**:
+  - Added OpenAI SDK version pin
+  - Fixed failure detection to check both output and step outcome
+- **`test_alignment_check.py`**:
+  - Added error handling for missing 'counts' key
+- **`requirements_minimal.txt`**:
+  - Added OpenAI SDK version constraint
+
+### Expected Behavior Now
+✅ Tests will run with compatible dependencies
+✅ Tests will handle errors gracefully
+✅ **ALL failures will send Slack notifications**
+✅ **ALL successes will send Slack notifications**
+✅ Full visibility into CI/CD pipeline health
+
+**Next run should succeed AND properly report any issues!** 🎉
+
+---
+
+**Last Updated:** 2026-02-07

GITHUB_ACTIONS_TESTS.md

@@ -19,25 +19,38 @@ These tests only run when the required API keys are configured in GitHub Secrets
 **Requires TOGETHER_API_KEY:**
 6. ⚙️ `test_alignment_dual_dimensions.py` - AlignmentCheck dual dimensions test
 7. ⚙️ `test_alignment_vs_factchecker.py` - AlignmentCheck vs FactChecker separation test
-8. ⚙️ `test_alignment_check.py` - AlignmentCheck scanner test
-9. ⚙️ `test_native_llamafirewall_scanner.py` - Native LlamaFirewall scanner test
+8. ⚙️ `test_native_llamafirewall_scanner.py` - Native LlamaFirewall scanner test
 
 **Requires OPENAI_API_KEY:**
+9. ⚙️ `test_alignment_check.py` - AlignmentCheck scanner test (GPT-4o-mini fallback implementation)
 10. ⚙️ `test_facts_checker_scanner.py` - FactsChecker scanner test (7 subtests including temporal awareness)
 
+## Known Issues
+
+### NeMo GuardRails - OpenAI SDK Compatibility
+
+**Issue:** NeMo GuardRails may fail with:
+```
+TypeError: AsyncCompletions.create() got an unexpected keyword argument 'stream_usage'
+```
+
+**Fix:** The workflow pins `openai<1.58.0` to avoid this issue. See `NEMO_GUARDRAILS_FIX.md` for details.
+
+**Status:** ✅ FIXED (version constraint applied in workflow)
+
 ## Required GitHub Secrets
 
 To run all 10 tests, configure these secrets in your repository:
 
-### 1. TOGETHER_API_KEY ⚠️ REQUIRED FOR 4 TESTS
-- Used by: AlignmentCheck tests (6, 7, 8) and Native LlamaFirewall test (9)
+### 1. TOGETHER_API_KEY ⚠️ REQUIRED FOR 3 TESTS
+- Used by: AlignmentCheck tests (6, 7) and Native LlamaFirewall test (8)
 - Provider: [Together AI](https://api.together.xyz/)
-- Purpose: Powers the AlignmentCheck scanner for behavioral drift detection
+- Purpose: Powers native LlamaFirewall for behavioral drift detection
 
-### 2. OPENAI_API_KEY ⚠️ REQUIRED FOR 1 TEST
-- Used by: FactsChecker test (10)
+### 2. OPENAI_API_KEY ⚠️ REQUIRED FOR 2 TESTS
+- Used by: AlignmentCheck GPT-4o-mini fallback test (9) and FactsChecker test (10)
 - Provider: [OpenAI](https://platform.openai.com/)
-- Purpose: Powers NeMo GuardRails fact-checking with GPT-4o-mini
+- Purpose: Powers GPT-4o-mini based tests (AlignmentCheck fallback and NeMo GuardRails fact-checking)
 
 ## How to Add Secrets
 

NEMO_GUARDRAILS_FIX.md

@@ -0,0 +1,114 @@
+# NeMo GuardRails - OpenAI SDK Compatibility Fix
+
+## Problem
+
+When running FactsChecker tests in GitHub Actions, you may see this error:
+
+```
+TypeError: AsyncCompletions.create() got an unexpected keyword argument 'stream_usage'
+```
+
+Full error trace:
+```
+WARNING! stream_usage is not default parameter.
+stream_usage was transferred to model_kwargs.
+Please confirm that stream_usage is what you intended.
+Error invoking LLM (model=gpt-4o-mini): AsyncCompletions.create() got an unexpected keyword argument 'stream_usage'
+```
+
+## Root Cause
+
+**Version incompatibility** between:
+- `openai` (OpenAI Python SDK) version **1.58.0 or newer**
+- `langchain-community` (used internally by NeMo GuardRails)
+
+The OpenAI SDK version 1.58.0+ introduced breaking changes to the API, and `langchain-community` (which NeMo GuardRails depends on) hasn't been updated to handle these changes yet.
+
+Specifically:
+1. NeMo GuardRails uses `langchain-community` to call OpenAI's API
+2. `langchain-community` tries to pass a `stream_usage` parameter
+3. OpenAI SDK 1.58.0+ doesn't accept this parameter
+4. Result: TypeError
+
+## Solution
+
+### Fix #1: Pin OpenAI SDK Version (APPLIED)
+
+Constrain the OpenAI SDK to versions before 1.58.0:
+
+```bash
+pip install "openai<1.58.0"
+```
+
+This is applied in `.github/workflows/test.yml`:
+
+```yaml
+- name: Install dependencies
+  run: |
+    pip install "openai<1.58.0"  # Avoid stream_usage parameter error
+    pip install nemoguardrails
+```
+
+### Fix #2: Wait for Upstream Updates
+
+Alternatively, wait for:
+- NeMo GuardRails to update to use newer langchain versions
+- OR langchain-community to be updated for OpenAI SDK 1.58.0+
+
+Monitor these repositories:
+- https://github.com/NVIDIA/NeMo-Guardrails/issues
+- https://github.com/langchain-ai/langchain/issues
+
+### Fix #3: Use Alternative LLM Provider
+
+If you don't want to pin versions, switch NeMo GuardRails to use a different LLM provider in `nemo_config/config.yml`:
+
+```yaml
+# Example: Use Anthropic Claude instead
+models:
+  - type: main
+    engine: anthropic
+    model: claude-3-5-sonnet-20241022
+```
+
+Then use `ANTHROPIC_API_KEY` instead of `OPENAI_API_KEY`.
+
+## Verification
+
+After applying the fix, the FactsChecker test should pass:
+
+```bash
+python test_facts_checker_scanner.py
+```
+
+Expected output:
+```
+✅ PASS: FactsChecker Available
+✅ PASS: Self-Contradiction Detection
+✅ PASS: RAG Ungroundedness - Fabricated API
+...
+✅ ALL TESTS PASSED (7/7)
+```
+
+## Related Issues
+
+- OpenAI SDK breaking changes: https://github.com/openai/openai-python/releases
+- NeMo GuardRails compatibility: https://github.com/NVIDIA/NeMo-Guardrails/issues
+- Langchain OpenAI integration: https://github.com/langchain-ai/langchain/issues
+
+## Timeline
+
+- **Issue Discovered:** 2026-02-07
+- **Fix Applied:** 2026-02-07
+- **Status:** FIXED (pinned openai<1.58.0)
+
+## Future Action
+
+When NeMo GuardRails or langchain-community release updates that are compatible with OpenAI SDK 1.58.0+, remove the version constraint:
+
+```diff
+- pip install "openai<1.58.0"
++ pip install openai
+```
+
+Test thoroughly before removing the constraint!

requirements_minimal.txt

@@ -9,4 +9,10 @@ python-dotenv>=1.0.0
 
 # Core libraries without heavy dependencies
 pydantic>=2.0.0
-requests>=2.28.0  # Required for AlignmentCheck API calls
+requests>=2.28.0  # Required for AlignmentCheck API calls
+
+# OpenAI SDK version constraint (compatibility with NeMo GuardRails)
+# Issue: OpenAI SDK 1.58.0+ has breaking changes that cause stream_usage parameter errors
+# with langchain-community (used by NeMo GuardRails). Pin to earlier version until fixed.
+# See NEMO_GUARDRAILS_FIX.md for details.
+openai<1.58.0

test_alignment_check.py

@@ -28,8 +28,14 @@ def test_aligned_conversation():
 
     result = scan_alignment_check_per_message(messages=messages, purpose=purpose)
 
+    # Check for errors first
+    if "error" in result:
+        print(f"\n❌ ERROR: {result.get('error')}")
+        return False
+
     print(f"\nOverall Decision: {result.get('overall_decision')}")
-    print(f"Counts: Safe={result['counts']['safe']}, Warning={result['counts']['warning']}, Block={result['counts']['block']}")
+    counts = result.get('counts', {})
+    print(f"Counts: Safe={counts.get('safe', 0)}, Warning={counts.get('warning', 0)}, Block={counts.get('block', 0)}")
 
     # Verify expected result
     if result.get('overall_decision') == 'SAFE':
@@ -60,8 +66,14 @@ def test_goal_hijacking():
 
     result = scan_alignment_check_per_message(messages=messages, purpose=purpose)
 
+    # Check for errors first
+    if "error" in result:
+        print(f"\n❌ ERROR: {result.get('error')}")
+        return False
+
     print(f"\nOverall Decision: {result.get('overall_decision')}")
-    print(f"Counts: Safe={result['counts']['safe']}, Warning={result['counts']['warning']}, Block={result['counts']['block']}")
+    counts = result.get('counts', {})
+    print(f"Counts: Safe={counts.get('safe', 0)}, Warning={counts.get('warning', 0)}, Block={counts.get('block', 0)}")
 
     # Verify expected result
     if result.get('overall_decision') != 'SAFE':
@@ -92,8 +104,14 @@ def test_off_topic_redirect():
 
     result = scan_alignment_check_per_message(messages=messages, purpose=purpose)
 
+    # Check for errors first
+    if "error" in result:
+        print(f"\n❌ ERROR: {result.get('error')}")
+        return False
+
     print(f"\nOverall Decision: {result.get('overall_decision')}")
-    print(f"Counts: Safe={result['counts']['safe']}, Warning={result['counts']['warning']}, Block={result['counts']['block']}")
+    counts = result.get('counts', {})
+    print(f"Counts: Safe={counts.get('safe', 0)}, Warning={counts.get('warning', 0)}, Block={counts.get('block', 0)}")
 
     # Verify expected result
     if result.get('overall_decision') != 'SAFE':