debug_rules.py

#!/usr/bin/env python3
"""
Debug the rule-based detection system
"""

import requests
import json

BASE_URL = "http://localhost:8000"

def debug_rule_detection():
    """Debug rule detection for specific cases"""
    
    print("🔍 DEBUGGING RULE-BASED DETECTION")
    print("=" * 50)
    
    test_cases = [
        {
            "name": "Geography - Wrong Capital",
            "prompt": "Question: What is the capital of France?",
            "response": "London is the capital of France",
            "question": "What is the capital of France?",
            "expected_rule": "Should detect geography contradiction"
        },
        {
            "name": "Science - Wrong Heart Chambers", 
            "prompt": "Question: How many chambers does the human heart have?",
            "response": "The human heart has 6 chambers",
            "question": "How many chambers does the human heart have?",
            "expected_rule": "Should detect science contradiction"
        },
        {
            "name": "Physics - Wrong Planet Size",
            "prompt": "Question: Which is larger, Earth or Jupiter?", 
            "response": "Earth is larger than Jupiter",
            "question": "Which is larger, Earth or Jupiter?",
            "expected_rule": "Should detect physics contradiction"
        }
    ]
    
    for i, test in enumerate(test_cases, 1):
        print(f"\n🧪 Debug Test {i}: {test['name']}")
        print(f"❓ Question: {test['question']}")
        print(f"📄 Response: {test['response']}")
        print(f"🔍 Expected Rule: {test['expected_rule']}")
        
        payload = {
            "prompt": test["prompt"],
            "response": test["response"],
            "question": test["question"],
            "use_cache": False
        }
        
        try:
            # Use debug endpoint
            response = requests.post(f"{BASE_URL}/api/debug-predict", json=payload, timeout=10)
            if response.status_code == 200:
                result = response.json()
                print(f"🤖 Raw Model Output: '{result.get('raw_model_output', '')}'")
                print(f"📊 Is Hallucination: {result.get('is_hallucination', False)}")
                print(f"📊 Confidence: {result.get('confidence_score', 0):.1%}")
                
                # Also test regular endpoint
                response2 = requests.post(f"{BASE_URL}/api/predict", json=payload, timeout=10)
                if response2.status_code == 200:
                    result2 = response2.json()
                    print(f"🔧 Final Is Hallucination: {result2.get('is_hallucination', False)}")
                    print(f"🔧 Final Confidence: {result2.get('confidence_score', 0):.1%}")
                    
                    # Check if confidence changed (indicating rule assistance)
                    if abs(result.get('confidence_score', 0) - result2.get('confidence_score', 0)) > 0.01:
                        print("✅ RULE ASSISTANCE DETECTED")
                    else:
                        print("❌ NO RULE ASSISTANCE")
            else:
                print(f"❌ Error: {response.status_code}")
                
        except Exception as e:
            print(f"❌ Error: {e}")

if __name__ == "__main__":
    debug_rule_detection()