""" Extended Privacy Detection Tests for SAAP Evaluation Includes: Edge Cases, False Positive Tests, False Negative Tests, Adversarial Cases Purpose: Address professor's feedback about trivial tests """ import sys import json from pathlib import Path from typing import List, Dict, Tuple from dataclasses import dataclass from datetime import datetime # Add parent directory to path sys.path.insert(0, str(Path(__file__).parent.parent)) from services.privacy_detector import PrivacyDetector, PrivacyLevel, detect_privacy_level @dataclass class TestCase: """Single test case for privacy detection""" id: str category: str input_text: str expected_level: str # "private", "confidential", "internal", "public" description: str difficulty: str # "easy", "medium", "hard", "adversarial" def create_extended_test_cases() -> List[TestCase]: """ Create comprehensive test cases including: - Easy cases (baseline) - Medium cases (real-world variations) - Hard cases (edge cases, partial matches) - Adversarial cases (designed to trick the detector) """ test_cases = [] # ========================================== # CATEGORY 1: TRUE POSITIVES (should be PRIVATE/CONFIDENTIAL) # ========================================== # --- Easy True Positives (Baseline) --- test_cases.extend([ TestCase("TP-E-01", "IBAN", "Meine IBAN ist DE89370400440532013000", "private", "Standard deutsche IBAN", "easy"), TestCase("TP-E-02", "Kreditkarte", "Meine Kreditkartennummer ist 4111111111111111", "private", "Standard Visa-Testformat", "easy"), TestCase("TP-E-03", "Medizin", "Der Patient hat Diabetes Typ 2 diagnostiziert", "private", "Explizite Diagnose", "easy"), TestCase("TP-E-04", "Gehalt", "Mein Gehalt beträgt 75.000 EUR jährlich", "private", "Explizite Gehaltsangabe", "easy"), ]) # --- Medium True Positives (Real-World Variations) --- test_cases.extend([ TestCase("TP-M-01", "IBAN-Text", "Bitte überweisen Sie auf DE44500105175407324931 bis Freitag", "private", "IBAN in Satzkontext", "medium"), TestCase("TP-M-02", "Medizin-Verlauf", "Seit der Behandlung mit Metformin sind die Blutzuckerwerte stabil", "private", "Medikament mit Kontext", "medium"), TestCase("TP-M-03", "Kreditkarte-Spaces", "4111 1111 1111 1111 ist meine Kartennummer", "private", "Kreditkarte mit Leerzeichen", "medium"), TestCase("TP-M-04", "Personal-Kombi", "Meine Adresse ist Hauptstraße 15, 12345 Berlin, geboren am 15.03.1985", "private", "Kombination persönlicher Daten", "medium"), TestCase("TP-M-05", "SSN-US", "My social security number is 123-45-6789", "private", "US SSN Format", "medium"), TestCase("TP-M-06", "Medizin-Symptom", "Ich habe seit Wochen starke Kopfschmerzen und Übelkeit, mein Arzt vermutet Migräne", "private", "Symptombeschreibung mit Arzt", "medium"), ]) # --- Hard True Positives (Edge Cases) --- test_cases.extend([ TestCase("TP-H-01", "IBAN-Keine-Prefix", "DE89370400440532013000 - das ist die Kontoverbindung", "private", "IBAN ohne vorherige Erklärung", "hard"), TestCase("TP-H-02", "Medizin-Indirekt", "Nach meinem Krankenhausaufenthalt wegen der Herzoperation brauche ich Reha", "private", "Indirekter medizinischer Kontext", "hard"), TestCase("TP-H-03", "Multi-Pattern", "Kontakt: +49 176 12345678, email@test.de, geboren 01.01.1990", "confidential", "Mehrere Muster in einem Text", "hard"), TestCase("TP-H-04", "Kontext-Medizin", "Ich bin Patient in der Klinik und nehme täglich meine Medikamente", "private", "Impliziter medizinischer Status", "hard"), TestCase("TP-H-05", "Gehalts-Indirekt", "Mein monatliches Nettoeinkommen nach Steuern liegt bei etwa 3.500 Euro", "private", "Gehalt ohne direktes Keyword", "hard"), ]) # ========================================== # CATEGORY 2: TRUE NEGATIVES (should be PUBLIC) # ========================================== # --- Easy True Negatives (Clearly Non-Sensitive) --- test_cases.extend([ TestCase("TN-E-01", "Tech", "Python ist eine interpretierte Programmiersprache", "public", "Technische Info", "easy"), TestCase("TN-E-02", "Wetter", "Das Wetter in Berlin ist heute sonnig mit 22 Grad", "public", "Wetterbericht", "easy"), TestCase("TN-E-03", "Frage", "Was ist der Unterschied zwischen REST und GraphQL?", "public", "Technische Frage", "easy"), TestCase("TN-E-04", "Allgemein", "Wie funktioniert Machine Learning?", "public", "Allgemeine Wissensfrage", "easy"), ]) # --- Medium True Negatives (Contains potential trigger words but non-sensitive) --- test_cases.extend([ TestCase("TN-M-01", "Fiktion-Medizin", "In der Serie 'House' diagnostiziert der Arzt komplexe Krankheiten", "public", "Fiktionaler medizinischer Kontext", "medium"), TestCase("TN-M-02", "Beispiel-IBAN", "Das IBAN-Format für Deutschland ist DE + 2 Prüfziffern + 8 BLZ + 10 Kontonummer", "public", "Erklärung ohne echte IBAN", "medium"), TestCase("TN-M-03", "Code-Beispiel", "const password = 'hash123'; // Beispiel für Passwortspeicherung", "public", "Code-Snippet mit 'password' String", "medium"), TestCase("TN-M-04", "Bildung", "Im Medizinstudium lernt man über Anatomie und Physiologie", "public", "Bildungskontext, nicht Patient", "medium"), ]) # --- Hard True Negatives (Designed to test False Positive avoidance) --- test_cases.extend([ TestCase("TN-H-01", "Partiell-IBAN", "DE123 ist keine gültige IBAN - sie muss 22 Zeichen haben", "public", "Ungültiges IBAN-Format", "hard"), TestCase("TN-H-02", "Zahlenfolge", "Die Telefonnummer 12345678901234567890 ist zu lang", "public", "Lange Zahl, aber keine Kreditkarte", "hard"), TestCase("TN-H-03", "Fiktion-Gehalt", "In dem Roman verdient der Protagonist 100.000 EUR Gehalt", "public", "Fiktionales Gehalt", "hard"), TestCase("TN-H-04", "Generisch-Patient", "Der Patient in Zimmer 5 ist ein NPC im Videospiel", "public", "Gaming-Kontext mit 'Patient'", "hard"), TestCase("TN-H-05", "Tech-Health", "Das System health check zeigt CPU: healthy, Memory: healthy", "public", "Technischer 'health' Begriff", "hard"), TestCase("TN-H-06", "English-Blood", "The new album 'Blood on the Dance Floor' is available now", "public", "Nicht-medizinischer Blood-Kontext", "hard"), ]) # ========================================== # CATEGORY 3: ADVERSARIAL CASES (Edge of detection) # ========================================== test_cases.extend([ # Obfuscation attempts TestCase("ADV-01", "Obfusc-IBAN", "D E 8 9 3 7 0 4 0 0 4 4 0 5 3 2 0 1 3 0 0 0", "public", "IBAN mit Leerzeichen (Obfuscation)", "adversarial"), TestCase("ADV-02", "Leet-Password", "My p@ssw0rd is s3cur3123", "public", "Leet-Speak Passwort-Obfuscation", "adversarial"), # Contextual ambiguity TestCase("ADV-03", "Kontext-Dual", "Ich behandle den Bug im Code, nicht den Patienten", "public", "Dual-Kontext (Code vs Medizin)", "adversarial"), TestCase("ADV-04", "Historisch", "Im Jahr 1985 kostete die Diagnose von Computerviren viel Zeit", "public", "Historischer Tech-Kontext mit Medizin-Wort", "adversarial"), # Near-miss patterns TestCase("ADV-05", "Fast-IBAN", "DE8937040044053201300 ist falsch formatiert", "public", "Ungültige IBAN (21 statt 22 Zeichen)", "adversarial"), TestCase("ADV-06", "Fast-CC", "411111111111111 ist keine Kreditkarte", "public", "15-stellige Zahl (nicht CC)", "adversarial"), # Multi-language mixing TestCase("ADV-07", "Mixed-Lang", "The patient wurde behandelt successfully", "private", "Deutsch-Englisch Mix mit Patient", "adversarial"), # Embedding in noise TestCase("ADV-08", "Noise-Medical", "Lorem ipsum dolor sit amet Diabetes consectetur adipiscing elit", "private", "Medizinisches Keyword in Lorem Ipsum", "adversarial"), ]) # ========================================== # CATEGORY 4: BOUNDARY TESTING # ========================================== test_cases.extend([ # Single keyword boundaries TestCase("BND-01", "Single-Health", "health", "public", "Nur 'health' ohne Kontext", "medium"), TestCase("BND-02", "Single-Patient", "patient", "private", "Nur 'patient' als Keyword", "medium"), TestCase("BND-03", "Single-Gehalt", "gehalt", "private", "Nur 'Gehalt' als Keyword", "medium"), # Word boundary tests TestCase("BND-04", "Word-Boundary", "unhealthy Wealthiness", "public", "Enthält 'health' als Substring, sollte NICHT triggern", "hard"), TestCase("BND-05", "Compound-DE", "Gehaltstabelle und Einkommensteuer im Überblick", "private", "Deutsche Komposita mit Gehalt", "hard"), ]) return test_cases def run_extended_tests() -> Dict: """Run all extended test cases and collect results""" test_cases = create_extended_test_cases() detector = PrivacyDetector() results = { "timestamp": datetime.now().isoformat(), "total_tests": len(test_cases), "summary": { "TP": 0, # True Positive (correct PRIVATE/CONFIDENTIAL detection) "TN": 0, # True Negative (correct PUBLIC detection) "FP": 0, # False Positive (incorrectly marked as sensitive) "FN": 0 # False Negative (missed sensitive data) }, "by_difficulty": { "easy": {"correct": 0, "total": 0}, "medium": {"correct": 0, "total": 0}, "hard": {"correct": 0, "total": 0}, "adversarial": {"correct": 0, "total": 0} }, "test_details": [] } for tc in test_cases: # Detect privacy level detected_level, details = detect_privacy_level(tc.input_text, agent_id=None) detected_value = detected_level.value # Determine if detection is correct expected_sensitive = tc.expected_level in ["private", "confidential"] detected_sensitive = detected_value in ["private", "confidential"] is_correct = False confusion_type = "" if expected_sensitive and detected_sensitive: results["summary"]["TP"] += 1 is_correct = True confusion_type = "TP" elif not expected_sensitive and not detected_sensitive: results["summary"]["TN"] += 1 is_correct = True confusion_type = "TN" elif not expected_sensitive and detected_sensitive: results["summary"]["FP"] += 1 confusion_type = "FP" else: # expected_sensitive and not detected_sensitive results["summary"]["FN"] += 1 confusion_type = "FN" # Track by difficulty results["by_difficulty"][tc.difficulty]["total"] += 1 if is_correct: results["by_difficulty"][tc.difficulty]["correct"] += 1 # Store details results["test_details"].append({ "id": tc.id, "category": tc.category, "difficulty": tc.difficulty, "input": tc.input_text[:80] + ("..." if len(tc.input_text) > 80 else ""), "expected": tc.expected_level, "detected": detected_value, "correct": is_correct, "confusion_type": confusion_type, "detection_reason": details.get("reason", "unknown"), "keyword_matches": len(details.get("keyword_matches", [])), "pattern_matches": len(details.get("pattern_matches", [])) }) # Calculate metrics s = results["summary"] total = s["TP"] + s["TN"] + s["FP"] + s["FN"] # Precision = TP / (TP + FP) results["metrics"] = { "precision": s["TP"] / (s["TP"] + s["FP"]) if (s["TP"] + s["FP"]) > 0 else 0, "recall": s["TP"] / (s["TP"] + s["FN"]) if (s["TP"] + s["FN"]) > 0 else 0, "accuracy": (s["TP"] + s["TN"]) / total if total > 0 else 0 } # F1 Score p, r = results["metrics"]["precision"], results["metrics"]["recall"] results["metrics"]["f1_score"] = 2 * p * r / (p + r) if (p + r) > 0 else 0 return results def print_results(results: Dict): """Print results in formatted output""" print("\n" + "="*70) print(" EXTENDED PRIVACY DETECTION TEST RESULTS") print("="*70) print(f"\nTimestamp: {results['timestamp']}") print(f"Total Tests: {results['total_tests']}\n") # Confusion Matrix s = results["summary"] print("KONFUSIONSMATRIX:") print("-"*50) print(f" │ Predicted PRIVATE │ Predicted PUBLIC") print(f"───────────────────────┼───────────────────┼──────────────────") print(f" Actual PRIVATE │ {s['TP']:>8} (TP) │ {s['FN']:>8} (FN)") print(f" Actual PUBLIC │ {s['FP']:>8} (FP) │ {s['TN']:>8} (TN)") print("-"*50) # Metrics m = results["metrics"] print(f"\nMETRIKEN:") print(f" Precision: {m['precision']:.4f} ({m['precision']*100:.2f}%)") print(f" Recall: {m['recall']:.4f} ({m['recall']*100:.2f}%)") print(f" F1-Score: {m['f1_score']:.4f} ({m['f1_score']*100:.2f}%)") print(f" Accuracy: {m['accuracy']:.4f} ({m['accuracy']*100:.2f}%)") # By Difficulty print(f"\nERGEBNISSE NACH SCHWIERIGKEITSGRAD:") for diff, data in results["by_difficulty"].items(): if data["total"] > 0: rate = data["correct"] / data["total"] * 100 print(f" {diff.capitalize():12}: {data['correct']}/{data['total']} ({rate:.1f}%)") # Failed Tests failed = [t for t in results["test_details"] if not t["correct"]] if failed: print(f"\nFEHLGESCHLAGENE TESTS ({len(failed)}):") print("-"*70) for t in failed: print(f" [{t['id']}] {t['confusion_type']}") print(f" Input: {t['input']}") print(f" Expected: {t['expected']} | Detected: {t['detected']}") print(f" Reason: {t['detection_reason']}") print() else: print("\n✓ Alle Tests bestanden!") print("="*70) def generate_markdown_table(results: Dict) -> str: """Generate markdown table for thesis documentation""" lines = [] lines.append("### Erweiterte Konfusionsmatrix (n={})".format(results["total_tests"])) lines.append("") lines.append("| | Predicted PRIVATE/CONF | Predicted PUBLIC/INT |") lines.append("|----------------|------------------------|----------------------|") lines.append(f"| **Actual PRIVATE/CONF** | {results['summary']['TP']} (TP) | {results['summary']['FN']} (FN) |") lines.append(f"| **Actual PUBLIC/INT** | {results['summary']['FP']} (FP) | {results['summary']['TN']} (TN) |") lines.append("") m = results["metrics"] lines.append("### Berechnete Metriken") lines.append("") lines.append("```") lines.append(f"Precision = TP / (TP + FP) = {results['summary']['TP']} / ({results['summary']['TP']} + {results['summary']['FP']}) = {m['precision']:.4f} ({m['precision']*100:.2f}%)") lines.append(f"Recall = TP / (TP + FN) = {results['summary']['TP']} / ({results['summary']['TP']} + {results['summary']['FN']}) = {m['recall']:.4f} ({m['recall']*100:.2f}%)") lines.append(f"F1-Score = 2 * (P * R) / (P + R) = {m['f1_score']:.4f} ({m['f1_score']*100:.2f}%)") lines.append(f"Accuracy = (TP + TN) / Total = {m['accuracy']:.4f} ({m['accuracy']*100:.2f}%)") lines.append("```") lines.append("") # Difficulty breakdown lines.append("### Ergebnisse nach Schwierigkeitsgrad") lines.append("") lines.append("| Schwierigkeit | Korrekt | Gesamt | Rate |") lines.append("|---------------|---------|--------|------|") for diff, data in results["by_difficulty"].items(): if data["total"] > 0: rate = data["correct"] / data["total"] * 100 lines.append(f"| {diff.capitalize()} | {data['correct']} | {data['total']} | {rate:.1f}% |") lines.append("") # Failed tests detail failed = [t for t in results["test_details"] if not t["correct"]] if failed: lines.append("### Fehlgeschlagene Testfälle (Detail)") lines.append("") lines.append("| Test-ID | Typ | Input (gekürzt) | Erwartet | Erkannt | Grund |") lines.append("|---------|-----|-----------------|----------|---------|-------|") for t in failed: input_short = t['input'][:40] + "..." if len(t['input']) > 40 else t['input'] lines.append(f"| {t['id']} | {t['confusion_type']} | {input_short} | {t['expected']} | {t['detected']} | {t['detection_reason']} |") return "\n".join(lines) if __name__ == "__main__": print("Running Extended Privacy Detection Tests...") results = run_extended_tests() print_results(results) # Generate markdown for documentation md_output = generate_markdown_table(results) # Save results output_path = Path(__file__).parent.parent.parent / "docs" / "extended_privacy_test_results.json" with open(output_path, "w", encoding="utf-8") as f: json.dump(results, f, indent=2, ensure_ascii=False) md_path = Path(__file__).parent.parent.parent / "docs" / "extended_privacy_test_results.md" with open(md_path, "w", encoding="utf-8") as f: f.write(md_output) print(f"\nResults saved to:") print(f" - {output_path}") print(f" - {md_path}")