File size: 3,376 Bytes
5031515 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | #!/usr/bin/env python3
"""
Debug the rule-based detection system
"""
import requests
import json
BASE_URL = "http://localhost:8000"
def debug_rule_detection():
"""Debug rule detection for specific cases"""
print("🔍 DEBUGGING RULE-BASED DETECTION")
print("=" * 50)
test_cases = [
{
"name": "Geography - Wrong Capital",
"prompt": "Question: What is the capital of France?",
"response": "London is the capital of France",
"question": "What is the capital of France?",
"expected_rule": "Should detect geography contradiction"
},
{
"name": "Science - Wrong Heart Chambers",
"prompt": "Question: How many chambers does the human heart have?",
"response": "The human heart has 6 chambers",
"question": "How many chambers does the human heart have?",
"expected_rule": "Should detect science contradiction"
},
{
"name": "Physics - Wrong Planet Size",
"prompt": "Question: Which is larger, Earth or Jupiter?",
"response": "Earth is larger than Jupiter",
"question": "Which is larger, Earth or Jupiter?",
"expected_rule": "Should detect physics contradiction"
}
]
for i, test in enumerate(test_cases, 1):
print(f"\n🧪 Debug Test {i}: {test['name']}")
print(f"❓ Question: {test['question']}")
print(f"📄 Response: {test['response']}")
print(f"🔍 Expected Rule: {test['expected_rule']}")
payload = {
"prompt": test["prompt"],
"response": test["response"],
"question": test["question"],
"use_cache": False
}
try:
# Use debug endpoint
response = requests.post(f"{BASE_URL}/api/debug-predict", json=payload, timeout=10)
if response.status_code == 200:
result = response.json()
print(f"🤖 Raw Model Output: '{result.get('raw_model_output', '')}'")
print(f"📊 Is Hallucination: {result.get('is_hallucination', False)}")
print(f"📊 Confidence: {result.get('confidence_score', 0):.1%}")
# Also test regular endpoint
response2 = requests.post(f"{BASE_URL}/api/predict", json=payload, timeout=10)
if response2.status_code == 200:
result2 = response2.json()
print(f"🔧 Final Is Hallucination: {result2.get('is_hallucination', False)}")
print(f"🔧 Final Confidence: {result2.get('confidence_score', 0):.1%}")
# Check if confidence changed (indicating rule assistance)
if abs(result.get('confidence_score', 0) - result2.get('confidence_score', 0)) > 0.01:
print("✅ RULE ASSISTANCE DETECTED")
else:
print("❌ NO RULE ASSISTANCE")
else:
print(f"❌ Error: {response.status_code}")
except Exception as e:
print(f"❌ Error: {e}")
if __name__ == "__main__":
debug_rule_detection()
|