backend/scripts/run_real_tests.py

#!/usr/bin/env python3
"""Echte Evaluationstests für alle 4 Hypothesen (H1-H4)"""
import requests
import time
import json
import statistics

BASE_URL = "http://localhost:8000"
results = {"h1": {}, "h2": {}, "h3": {}, "h4": {}}

# ===== H1: Privacy Detection Tests =====
print("=" * 50)
print("H1: Privacy Detection Tests")
print("=" * 50)

privacy_tests = [
    {"msg": "Meine IBAN ist DE89370400440532013000", "expected": "private", "category": "IBAN"},
    {"msg": "Kreditkarte: 4111111111111111", "expected": "private", "category": "Kreditkarte"},
    {"msg": "Patient hat Diabetes Typ 2", "expected": "private", "category": "Medizin"},
    {"msg": "Mein Gehalt beträgt 75000 Euro", "expected": "private", "category": "Finanzen"},
    {"msg": "Was ist Python?", "expected": "public", "category": "Allgemein1"},
    {"msg": "Erkläre Docker", "expected": "public", "category": "Allgemein2"},
]

h1_results = []
for test in privacy_tests:
    try:
        resp = requests.post(f"{BASE_URL}/api/v1/multi-agent/chat", 
            json={"user_message": test["msg"], "privacy_mode": "auto"}, timeout=60)
        data = resp.json()
        detected = data.get("privacy_protection", {}).get("privacy_level", "unknown")
        correct = detected.lower() == test["expected"].lower()
        h1_results.append({"category": test["category"], "expected": test["expected"], 
                          "detected": detected, "correct": correct})
        print(f"  {test['category']}: {detected} (erwartet: {test['expected']}) -> {'✓' if correct else '✗'}")
    except Exception as e:
        print(f"  {test['category']}: FEHLER - {e}")

results["h1"]["tests"] = h1_results
results["h1"]["accuracy"] = sum(1 for r in h1_results if r["correct"]) / len(h1_results) * 100 if h1_results else 0
print(f"\n>>> H1 Genauigkeit: {results['h1']['accuracy']:.1f}%")

# ===== H4: Latenz-Tests =====
print("\n" + "=" * 50)
print("H4: Latenz-Tests (Simple Query)")
print("=" * 50)

latency_tests = ["Was ist REST?", "Erkläre API-Design", "Was ist Machine Learning?"]
latencies = []

for msg in latency_tests:
    try:
        start = time.time()
        resp = requests.post(f"{BASE_URL}/api/v1/multi-agent/chat", 
            json={"user_message": msg, "privacy_mode": "auto"}, timeout=120)
        latency = time.time() - start
        latencies.append(latency)
        print(f"  '{msg[:25]}...' -> {latency:.2f}s")
    except Exception as e:
        print(f"  FEHLER: {e}")

if latencies:
    results["h4"]["latencies"] = latencies
    results["h4"]["avg"] = statistics.mean(latencies)
    results["h4"]["min"] = min(latencies)
    results["h4"]["max"] = max(latencies)
    print(f"\n>>> H4 Durchschnitt: {results['h4']['avg']:.2f}s (Min: {results['h4']['min']:.2f}s, Max: {results['h4']['max']:.2f}s)")

# ===== H3: Kosten-Tracking =====
print("\n" + "=" * 50)
print("H3: Kosten-Analyse")
print("=" * 50)

try:
    resp = requests.get(f"{BASE_URL}/api/v1/costs/summary", timeout=30)
    if resp.status_code == 200:
        costs = resp.json()
        results["h3"] = costs
        print(f"  Gesamtkosten: ${costs.get('total_cost_usd', 0):.4f}")
        print(f"  Lokale Anfragen: {costs.get('local_requests', 0)}")
        print(f"  Cloud Anfragen: {costs.get('cloud_requests', 0)}")
    else:
        print(f"  Kosten-API Status: {resp.status_code}")
except Exception as e:
    print(f"  Kosten-API nicht verfügbar: {e}")

# ===== H2: Parallele Anfragen =====
print("\n" + "=" * 50)
print("H2: Skalierbarkeit (5 parallele Anfragen)")
print("=" * 50)

import concurrent.futures

def send_parallel_request(i):
    start = time.time()
    try:
        resp = requests.post(f"{BASE_URL}/api/v1/multi-agent/chat", 
            json={"user_message": f"Test {i}", "privacy_mode": "auto"}, timeout=60)
        return {"id": i, "latency": time.time() - start, "success": resp.status_code == 200}
    except:
        return {"id": i, "latency": time.time() - start, "success": False}

with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
    h2_results = list(executor.map(send_parallel_request, range(1, 6)))

results["h2"]["tests"] = h2_results
results["h2"]["success_rate"] = sum(1 for r in h2_results if r["success"]) / len(h2_results) * 100
results["h2"]["avg_latency"] = statistics.mean([r["latency"] for r in h2_results])

for r in h2_results:
    print(f"  Anfrage {r['id']}: {r['latency']:.2f}s - {'✓' if r['success'] else '✗'}")
print(f"\n>>> H2 Erfolgsrate: {results['h2']['success_rate']:.0f}%, Durchschnitt: {results['h2']['avg_latency']:.2f}s")

# ===== ZUSAMMENFASSUNG =====
print("\n" + "=" * 50)
print("ZUSAMMENFASSUNG ECHTE TESTERGEBNISSE")
print("=" * 50)
print(f"H1 Privacy Detection: {results['h1']['accuracy']:.1f}% Genauigkeit")
print(f"H2 Skalierbarkeit: {results['h2']['success_rate']:.0f}% Erfolgsrate, {results['h2']['avg_latency']:.2f}s avg")
print(f"H3 Kosten: {results['h3'].get('total_cost_usd', 'N/A')}")
print(f"H4 Latenz: {results['h4'].get('avg', 'N/A'):.2f}s Durchschnitt")