backend/scripts/extended_privacy_tests.py

"""
Extended Privacy Detection Tests for SAAP Evaluation
Includes: Edge Cases, False Positive Tests, False Negative Tests, Adversarial Cases

Purpose: Address professor's feedback about trivial tests
"""

import sys
import json
from pathlib import Path
from typing import List, Dict, Tuple
from dataclasses import dataclass
from datetime import datetime

# Add parent directory to path
sys.path.insert(0, str(Path(__file__).parent.parent))

from services.privacy_detector import PrivacyDetector, PrivacyLevel, detect_privacy_level

@dataclass
class TestCase:
    """Single test case for privacy detection"""
    id: str
    category: str
    input_text: str
    expected_level: str  # "private", "confidential", "internal", "public"
    description: str
    difficulty: str  # "easy", "medium", "hard", "adversarial"

def create_extended_test_cases() -> List[TestCase]:
    """
    Create comprehensive test cases including:
    - Easy cases (baseline)
    - Medium cases (real-world variations)
    - Hard cases (edge cases, partial matches)
    - Adversarial cases (designed to trick the detector)
    """
    
    test_cases = []
    
    # ==========================================
    # CATEGORY 1: TRUE POSITIVES (should be PRIVATE/CONFIDENTIAL)
    # ==========================================
    
    # --- Easy True Positives (Baseline) ---
    test_cases.extend([
        TestCase("TP-E-01", "IBAN", "Meine IBAN ist DE89370400440532013000", "private", "Standard deutsche IBAN", "easy"),
        TestCase("TP-E-02", "Kreditkarte", "Meine Kreditkartennummer ist 4111111111111111", "private", "Standard Visa-Testformat", "easy"),
        TestCase("TP-E-03", "Medizin", "Der Patient hat Diabetes Typ 2 diagnostiziert", "private", "Explizite Diagnose", "easy"),
        TestCase("TP-E-04", "Gehalt", "Mein Gehalt beträgt 75.000 EUR jährlich", "private", "Explizite Gehaltsangabe", "easy"),
    ])
    
    # --- Medium True Positives (Real-World Variations) ---
    test_cases.extend([
        TestCase("TP-M-01", "IBAN-Text", "Bitte überweisen Sie auf DE44500105175407324931 bis Freitag", "private", "IBAN in Satzkontext", "medium"),
        TestCase("TP-M-02", "Medizin-Verlauf", "Seit der Behandlung mit Metformin sind die Blutzuckerwerte stabil", "private", "Medikament mit Kontext", "medium"),
        TestCase("TP-M-03", "Kreditkarte-Spaces", "4111 1111 1111 1111 ist meine Kartennummer", "private", "Kreditkarte mit Leerzeichen", "medium"),
        TestCase("TP-M-04", "Personal-Kombi", "Meine Adresse ist Hauptstraße 15, 12345 Berlin, geboren am 15.03.1985", "private", "Kombination persönlicher Daten", "medium"),
        TestCase("TP-M-05", "SSN-US", "My social security number is 123-45-6789", "private", "US SSN Format", "medium"),
        TestCase("TP-M-06", "Medizin-Symptom", "Ich habe seit Wochen starke Kopfschmerzen und Übelkeit, mein Arzt vermutet Migräne", "private", "Symptombeschreibung mit Arzt", "medium"),
    ])
    
    # --- Hard True Positives (Edge Cases) ---
    test_cases.extend([
        TestCase("TP-H-01", "IBAN-Keine-Prefix", "DE89370400440532013000 - das ist die Kontoverbindung", "private", "IBAN ohne vorherige Erklärung", "hard"),
        TestCase("TP-H-02", "Medizin-Indirekt", "Nach meinem Krankenhausaufenthalt wegen der Herzoperation brauche ich Reha", "private", "Indirekter medizinischer Kontext", "hard"),
        TestCase("TP-H-03", "Multi-Pattern", "Kontakt: +49 176 12345678, email@test.de, geboren 01.01.1990", "confidential", "Mehrere Muster in einem Text", "hard"),
        TestCase("TP-H-04", "Kontext-Medizin", "Ich bin Patient in der Klinik und nehme täglich meine Medikamente", "private", "Impliziter medizinischer Status", "hard"),
        TestCase("TP-H-05", "Gehalts-Indirekt", "Mein monatliches Nettoeinkommen nach Steuern liegt bei etwa 3.500 Euro", "private", "Gehalt ohne direktes Keyword", "hard"),
    ])
    
    # ==========================================
    # CATEGORY 2: TRUE NEGATIVES (should be PUBLIC)
    # ==========================================
    
    # --- Easy True Negatives (Clearly Non-Sensitive) ---
    test_cases.extend([
        TestCase("TN-E-01", "Tech", "Python ist eine interpretierte Programmiersprache", "public", "Technische Info", "easy"),
        TestCase("TN-E-02", "Wetter", "Das Wetter in Berlin ist heute sonnig mit 22 Grad", "public", "Wetterbericht", "easy"),
        TestCase("TN-E-03", "Frage", "Was ist der Unterschied zwischen REST und GraphQL?", "public", "Technische Frage", "easy"),
        TestCase("TN-E-04", "Allgemein", "Wie funktioniert Machine Learning?", "public", "Allgemeine Wissensfrage", "easy"),
    ])
    
    # --- Medium True Negatives (Contains potential trigger words but non-sensitive) ---
    test_cases.extend([
        TestCase("TN-M-01", "Fiktion-Medizin", "In der Serie 'House' diagnostiziert der Arzt komplexe Krankheiten", "public", "Fiktionaler medizinischer Kontext", "medium"),
        TestCase("TN-M-02", "Beispiel-IBAN", "Das IBAN-Format für Deutschland ist DE + 2 Prüfziffern + 8 BLZ + 10 Kontonummer", "public", "Erklärung ohne echte IBAN", "medium"),
        TestCase("TN-M-03", "Code-Beispiel", "const password = 'hash123'; // Beispiel für Passwortspeicherung", "public", "Code-Snippet mit 'password' String", "medium"),
        TestCase("TN-M-04", "Bildung", "Im Medizinstudium lernt man über Anatomie und Physiologie", "public", "Bildungskontext, nicht Patient", "medium"),
    ])
    
    # --- Hard True Negatives (Designed to test False Positive avoidance) ---
    test_cases.extend([
        TestCase("TN-H-01", "Partiell-IBAN", "DE123 ist keine gültige IBAN - sie muss 22 Zeichen haben", "public", "Ungültiges IBAN-Format", "hard"),
        TestCase("TN-H-02", "Zahlenfolge", "Die Telefonnummer 12345678901234567890 ist zu lang", "public", "Lange Zahl, aber keine Kreditkarte", "hard"),
        TestCase("TN-H-03", "Fiktion-Gehalt", "In dem Roman verdient der Protagonist 100.000 EUR Gehalt", "public", "Fiktionales Gehalt", "hard"),
        TestCase("TN-H-04", "Generisch-Patient", "Der Patient in Zimmer 5 ist ein NPC im Videospiel", "public", "Gaming-Kontext mit 'Patient'", "hard"),
        TestCase("TN-H-05", "Tech-Health", "Das System health check zeigt CPU: healthy, Memory: healthy", "public", "Technischer 'health' Begriff", "hard"),
        TestCase("TN-H-06", "English-Blood", "The new album 'Blood on the Dance Floor' is available now", "public", "Nicht-medizinischer Blood-Kontext", "hard"),
    ])
    
    # ==========================================
    # CATEGORY 3: ADVERSARIAL CASES (Edge of detection)
    # ==========================================
    
    test_cases.extend([
        # Obfuscation attempts
        TestCase("ADV-01", "Obfusc-IBAN", "D E 8 9 3 7 0 4 0 0 4 4 0 5 3 2 0 1 3 0 0 0", "public", "IBAN mit Leerzeichen (Obfuscation)", "adversarial"),
        TestCase("ADV-02", "Leet-Password", "My p@ssw0rd is s3cur3123", "public", "Leet-Speak Passwort-Obfuscation", "adversarial"),
        
        # Contextual ambiguity
        TestCase("ADV-03", "Kontext-Dual", "Ich behandle den Bug im Code, nicht den Patienten", "public", "Dual-Kontext (Code vs Medizin)", "adversarial"),
        TestCase("ADV-04", "Historisch", "Im Jahr 1985 kostete die Diagnose von Computerviren viel Zeit", "public", "Historischer Tech-Kontext mit Medizin-Wort", "adversarial"),
        
        # Near-miss patterns
        TestCase("ADV-05", "Fast-IBAN", "DE8937040044053201300 ist falsch formatiert", "public", "Ungültige IBAN (21 statt 22 Zeichen)", "adversarial"),
        TestCase("ADV-06", "Fast-CC", "411111111111111 ist keine Kreditkarte", "public", "15-stellige Zahl (nicht CC)", "adversarial"),
        
        # Multi-language mixing
        TestCase("ADV-07", "Mixed-Lang", "The patient wurde behandelt successfully", "private", "Deutsch-Englisch Mix mit Patient", "adversarial"),
        
        # Embedding in noise
        TestCase("ADV-08", "Noise-Medical", "Lorem ipsum dolor sit amet Diabetes consectetur adipiscing elit", "private", "Medizinisches Keyword in Lorem Ipsum", "adversarial"),
    ])
    
    # ==========================================
    # CATEGORY 4: BOUNDARY TESTING
    # ==========================================
    
    test_cases.extend([
        # Single keyword boundaries
        TestCase("BND-01", "Single-Health", "health", "public", "Nur 'health' ohne Kontext", "medium"),
        TestCase("BND-02", "Single-Patient", "patient", "private", "Nur 'patient' als Keyword", "medium"),
        TestCase("BND-03", "Single-Gehalt", "gehalt", "private", "Nur 'Gehalt' als Keyword", "medium"),
        
        # Word boundary tests
        TestCase("BND-04", "Word-Boundary", "unhealthy Wealthiness", "public", "Enthält 'health' als Substring, sollte NICHT triggern", "hard"),
        TestCase("BND-05", "Compound-DE", "Gehaltstabelle und Einkommensteuer im Überblick", "private", "Deutsche Komposita mit Gehalt", "hard"),
    ])
    
    return test_cases


def run_extended_tests() -> Dict:
    """Run all extended test cases and collect results"""
    
    test_cases = create_extended_test_cases()
    detector = PrivacyDetector()
    
    results = {
        "timestamp": datetime.now().isoformat(),
        "total_tests": len(test_cases),
        "summary": {
            "TP": 0,  # True Positive (correct PRIVATE/CONFIDENTIAL detection)
            "TN": 0,  # True Negative (correct PUBLIC detection)
            "FP": 0,  # False Positive (incorrectly marked as sensitive)
            "FN": 0   # False Negative (missed sensitive data)
        },
        "by_difficulty": {
            "easy": {"correct": 0, "total": 0},
            "medium": {"correct": 0, "total": 0},
            "hard": {"correct": 0, "total": 0},
            "adversarial": {"correct": 0, "total": 0}
        },
        "test_details": []
    }
    
    for tc in test_cases:
        # Detect privacy level
        detected_level, details = detect_privacy_level(tc.input_text, agent_id=None)
        detected_value = detected_level.value
        
        # Determine if detection is correct
        expected_sensitive = tc.expected_level in ["private", "confidential"]
        detected_sensitive = detected_value in ["private", "confidential"]
        
        is_correct = False
        confusion_type = ""
        
        if expected_sensitive and detected_sensitive:
            results["summary"]["TP"] += 1
            is_correct = True
            confusion_type = "TP"
        elif not expected_sensitive and not detected_sensitive:
            results["summary"]["TN"] += 1
            is_correct = True
            confusion_type = "TN"
        elif not expected_sensitive and detected_sensitive:
            results["summary"]["FP"] += 1
            confusion_type = "FP"
        else:  # expected_sensitive and not detected_sensitive
            results["summary"]["FN"] += 1
            confusion_type = "FN"
        
        # Track by difficulty
        results["by_difficulty"][tc.difficulty]["total"] += 1
        if is_correct:
            results["by_difficulty"][tc.difficulty]["correct"] += 1
        
        # Store details
        results["test_details"].append({
            "id": tc.id,
            "category": tc.category,
            "difficulty": tc.difficulty,
            "input": tc.input_text[:80] + ("..." if len(tc.input_text) > 80 else ""),
            "expected": tc.expected_level,
            "detected": detected_value,
            "correct": is_correct,
            "confusion_type": confusion_type,
            "detection_reason": details.get("reason", "unknown"),
            "keyword_matches": len(details.get("keyword_matches", [])),
            "pattern_matches": len(details.get("pattern_matches", []))
        })
    
    # Calculate metrics
    s = results["summary"]
    total = s["TP"] + s["TN"] + s["FP"] + s["FN"]
    
    # Precision = TP / (TP + FP)
    results["metrics"] = {
        "precision": s["TP"] / (s["TP"] + s["FP"]) if (s["TP"] + s["FP"]) > 0 else 0,
        "recall": s["TP"] / (s["TP"] + s["FN"]) if (s["TP"] + s["FN"]) > 0 else 0,
        "accuracy": (s["TP"] + s["TN"]) / total if total > 0 else 0
    }
    
    # F1 Score
    p, r = results["metrics"]["precision"], results["metrics"]["recall"]
    results["metrics"]["f1_score"] = 2 * p * r / (p + r) if (p + r) > 0 else 0
    
    return results


def print_results(results: Dict):
    """Print results in formatted output"""
    
    print("\n" + "="*70)
    print("  EXTENDED PRIVACY DETECTION TEST RESULTS")
    print("="*70)
    print(f"\nTimestamp: {results['timestamp']}")
    print(f"Total Tests: {results['total_tests']}\n")
    
    # Confusion Matrix
    s = results["summary"]
    print("KONFUSIONSMATRIX:")
    print("-"*50)
    print(f"                       │ Predicted PRIVATE │ Predicted PUBLIC")
    print(f"───────────────────────┼───────────────────┼──────────────────")
    print(f" Actual PRIVATE        │ {s['TP']:>8} (TP)     │ {s['FN']:>8} (FN)")
    print(f" Actual PUBLIC         │ {s['FP']:>8} (FP)     │ {s['TN']:>8} (TN)")
    print("-"*50)
    
    # Metrics
    m = results["metrics"]
    print(f"\nMETRIKEN:")
    print(f"  Precision: {m['precision']:.4f} ({m['precision']*100:.2f}%)")
    print(f"  Recall:    {m['recall']:.4f} ({m['recall']*100:.2f}%)")
    print(f"  F1-Score:  {m['f1_score']:.4f} ({m['f1_score']*100:.2f}%)")
    print(f"  Accuracy:  {m['accuracy']:.4f} ({m['accuracy']*100:.2f}%)")
    
    # By Difficulty
    print(f"\nERGEBNISSE NACH SCHWIERIGKEITSGRAD:")
    for diff, data in results["by_difficulty"].items():
        if data["total"] > 0:
            rate = data["correct"] / data["total"] * 100
            print(f"  {diff.capitalize():12}: {data['correct']}/{data['total']} ({rate:.1f}%)")
    
    # Failed Tests
    failed = [t for t in results["test_details"] if not t["correct"]]
    if failed:
        print(f"\nFEHLGESCHLAGENE TESTS ({len(failed)}):")
        print("-"*70)
        for t in failed:
            print(f"  [{t['id']}] {t['confusion_type']}")
            print(f"    Input: {t['input']}")
            print(f"    Expected: {t['expected']} | Detected: {t['detected']}")
            print(f"    Reason: {t['detection_reason']}")
            print()
    else:
        print("\n✓ Alle Tests bestanden!")
    
    print("="*70)


def generate_markdown_table(results: Dict) -> str:
    """Generate markdown table for thesis documentation"""
    
    lines = []
    lines.append("### Erweiterte Konfusionsmatrix (n={})".format(results["total_tests"]))
    lines.append("")
    lines.append("|                | Predicted PRIVATE/CONF | Predicted PUBLIC/INT |")
    lines.append("|----------------|------------------------|----------------------|")
    lines.append(f"| **Actual PRIVATE/CONF** | {results['summary']['TP']} (TP) | {results['summary']['FN']} (FN) |")
    lines.append(f"| **Actual PUBLIC/INT** | {results['summary']['FP']} (FP) | {results['summary']['TN']} (TN) |")
    lines.append("")
    
    m = results["metrics"]
    lines.append("### Berechnete Metriken")
    lines.append("")
    lines.append("```")
    lines.append(f"Precision = TP / (TP + FP) = {results['summary']['TP']} / ({results['summary']['TP']} + {results['summary']['FP']}) = {m['precision']:.4f} ({m['precision']*100:.2f}%)")
    lines.append(f"Recall    = TP / (TP + FN) = {results['summary']['TP']} / ({results['summary']['TP']} + {results['summary']['FN']}) = {m['recall']:.4f} ({m['recall']*100:.2f}%)")
    lines.append(f"F1-Score  = 2 * (P * R) / (P + R) = {m['f1_score']:.4f} ({m['f1_score']*100:.2f}%)")
    lines.append(f"Accuracy  = (TP + TN) / Total = {m['accuracy']:.4f} ({m['accuracy']*100:.2f}%)")
    lines.append("```")
    lines.append("")
    
    # Difficulty breakdown
    lines.append("### Ergebnisse nach Schwierigkeitsgrad")
    lines.append("")
    lines.append("| Schwierigkeit | Korrekt | Gesamt | Rate |")
    lines.append("|---------------|---------|--------|------|")
    for diff, data in results["by_difficulty"].items():
        if data["total"] > 0:
            rate = data["correct"] / data["total"] * 100
            lines.append(f"| {diff.capitalize()} | {data['correct']} | {data['total']} | {rate:.1f}% |")
    lines.append("")
    
    # Failed tests detail
    failed = [t for t in results["test_details"] if not t["correct"]]
    if failed:
        lines.append("### Fehlgeschlagene Testfälle (Detail)")
        lines.append("")
        lines.append("| Test-ID | Typ | Input (gekürzt) | Erwartet | Erkannt | Grund |")
        lines.append("|---------|-----|-----------------|----------|---------|-------|")
        for t in failed:
            input_short = t['input'][:40] + "..." if len(t['input']) > 40 else t['input']
            lines.append(f"| {t['id']} | {t['confusion_type']} | {input_short} | {t['expected']} | {t['detected']} | {t['detection_reason']} |")
    
    return "\n".join(lines)


if __name__ == "__main__":
    print("Running Extended Privacy Detection Tests...")
    results = run_extended_tests()
    print_results(results)
    
    # Generate markdown for documentation
    md_output = generate_markdown_table(results)
    
    # Save results
    output_path = Path(__file__).parent.parent.parent / "docs" / "extended_privacy_test_results.json"
    with open(output_path, "w", encoding="utf-8") as f:
        json.dump(results, f, indent=2, ensure_ascii=False)
    
    md_path = Path(__file__).parent.parent.parent / "docs" / "extended_privacy_test_results.md"
    with open(md_path, "w", encoding="utf-8") as f:
        f.write(md_output)
    
    print(f"\nResults saved to:")
    print(f"  - {output_path}")
    print(f"  - {md_path}")