backend/scripts/h2_h3_tests.py

#!/usr/bin/env python3
"""
H2 & H3 Evaluationstests für die Masterarbeit
Führt Tests für Skalierbarkeit (H2) und Kostenvorteile (H3) durch
"""

import asyncio
import aiohttp
import time
import statistics
import json

async def send_request(session, request_id, message, privacy_mode="auto"):
    """Sendet eine einzelne Anfrage an die SAAP-API"""
    start = time.time()
    try:
        async with session.post(
            "http://localhost:8000/api/v1/multi-agent/chat",
            json={"user_message": message, "privacy_mode": privacy_mode},
            timeout=aiohttp.ClientTimeout(total=120)
        ) as response:
            data = await response.json()
            latency = time.time() - start
            return {
                "id": request_id,
                "latency": latency,
                "success": data.get("success", False),
                "provider": data.get("response", {}).get("provider", "unknown"),
                "cost_usd": data.get("response", {}).get("cost_usd", 0),
                "privacy_level": data.get("privacy_protection", {}).get("privacy_level", "unknown")
            }
    except Exception as e:
        return {
            "id": request_id,
            "latency": time.time() - start,
            "success": False,
            "error": str(e)[:50]
        }

async def test_h2_scalability():
    """H2 Test: 10 parallele Anfragen für Skalierbarkeit"""
    print("=" * 65)
    print("H2 TEST: SKALIERBARKEIT - 10 PARALLELE ANFRAGEN")
    print("=" * 65)
    print(f"Startzeit: {time.strftime('%Y-%m-%d %H:%M:%S')}")
    print("-" * 65)
    
    async with aiohttp.ClientSession() as session:
        # 10 parallele, nicht-sensible Anfragen
        messages = [
            "Was ist Python?",
            "Erkläre REST APIs kurz",
            "Was macht Docker?",
            "Was ist Machine Learning?",
            "Beschreibe Microservices",
            "Was ist Cloud Computing?",
            "Erkläre Agile kurz",
            "Was ist DevOps?",
            "Beschreibe Git",
            "Was ist CI/CD?"
        ]
        
        tasks = [send_request(session, i+1, msg) for i, msg in enumerate(messages)]
        results = await asyncio.gather(*tasks)
    
    print(f"\n{'Nr':<5} {'Latenz (s)':<12} {'Provider':<12} {'Status':<15}")
    print("-" * 50)
    
    latencies = []
    successes = 0
    for r in results:
        status = "✓ Erfolg" if r.get("success") else f"✗ {r.get('error', 'Fehler')[:15]}"
        provider = r.get("provider", "?")
        print(f"{r['id']:<5} {r['latency']:<12.2f} {provider:<12} {status}")
        latencies.append(r["latency"])
        if r.get("success"):
            successes += 1
    
    print("-" * 50)
    print(f"\n📊 STATISTIK H2:")
    mean_lat = statistics.mean(latencies)
    print(f"  Durchschnittliche Latenz: {mean_lat:.2f}s")
    print(f"  Min: {min(latencies):.2f}s")
    print(f"  Max: {max(latencies):.2f}s")
    if len(latencies) > 1:
        print(f"  Standardabweichung: {statistics.stdev(latencies):.2f}s")
    print(f"  Erfolgsrate: {successes}/{len(results)} ({100*successes/len(results):.0f}%)")
    
    return {
        "mean": mean_lat,
        "min": min(latencies),
        "max": max(latencies),
        "stddev": statistics.stdev(latencies) if len(latencies) > 1 else 0,
        "success_rate": successes / len(results),
        "raw_latencies": latencies
    }

async def test_h3_costs():
    """H3 Test: Kostenvergleich zwischen lokal und Cloud"""
    print("\n" + "=" * 65)
    print("H3 TEST: KOSTENVORTEILE - HYBRID-ROUTING ANALYSE")
    print("=" * 65)
    print(f"Startzeit: {time.strftime('%Y-%m-%d %H:%M:%S')}")
    print("-" * 65)
    
    test_cases = [
        # Sensible Daten → lokales LLM (kostenlos)
        {"msg": "Meine IBAN ist DE89370400440532013000", "type": "IBAN", "expected_local": True},
        {"msg": "Patient hat Diabetes Typ 2 diagnostiziert", "type": "Medizin", "expected_local": True},
        {"msg": "Mein Gehalt beträgt 75.000 Euro", "type": "Finanzen", "expected_local": True},
        {"msg": "Kreditkartennummer 4111111111111111", "type": "Kreditkarte", "expected_local": True},
        # Nicht-sensible Daten → Cloud (kostenpflichtig)
        {"msg": "Was ist REST?", "type": "Allgemein", "expected_local": False},
        {"msg": "Erkläre Docker", "type": "Allgemein", "expected_local": False},
    ]
    
    async with aiohttp.ClientSession() as session:
        results = []
        for i, tc in enumerate(test_cases):
            print(f"  Test {i+1}/{len(test_cases)}: {tc['type']}...", end=" ", flush=True)
            result = await send_request(session, i+1, tc["msg"])
            result["type"] = tc["type"]
            result["expected_local"] = tc["expected_local"]
            results.append(result)
            
            is_local = result.get("provider") == "colossus"
            routing_ok = is_local == tc["expected_local"]
            print(f"Provider: {result.get('provider', '?')}, Kosten: ${result.get('cost_usd', 0):.4f} {'✓' if routing_ok else '✗'}")
    
    print("\n" + "-" * 65)
    print(f"\n{'Typ':<15} {'Provider':<12} {'Kosten':<12} {'Routing':<10}")
    print("-" * 50)
    
    total_cost = 0
    local_count = 0
    cloud_count = 0
    routing_correct = 0
    
    for r in results:
        provider = r.get("provider", "unknown")
        cost = r.get("cost_usd", 0)
        total_cost += cost
        
        is_local = provider == "colossus"
        if is_local:
            local_count += 1
        else:
            cloud_count += 1
        
        routing_ok = is_local == r["expected_local"]
        if routing_ok:
            routing_correct += 1
        
        print(f"{r['type']:<15} {provider:<12} ${cost:<11.4f} {'✓ korrekt' if routing_ok else '✗ falsch'}")
    
    print("-" * 50)
    
    # Kostenberechnung
    cloud_only_cost = len(results) * 0.03  # Annahme: $0.03 pro Cloud-Anfrage
    
    print(f"\n📊 STATISTIK H3:")
    print(f"  Lokale Anfragen: {local_count}/{len(results)} ({100*local_count/len(results):.0f}%)")
    print(f"  Cloud Anfragen: {cloud_count}/{len(results)} ({100*cloud_count/len(results):.0f}%)")
    print(f"  Routing-Korrektheit: {routing_correct}/{len(results)} ({100*routing_correct/len(results):.0f}%)")
    print(f"\n💰 KOSTENANALYSE:")
    print(f"  SAAP Hybrid-Kosten: ${total_cost:.4f}")
    print(f"  Pure-Cloud-Kosten (geschätzt): ${cloud_only_cost:.4f}")
    if cloud_only_cost > 0:
        savings = (cloud_only_cost - total_cost) / cloud_only_cost * 100
        print(f"  Ersparnis: {savings:.1f}%")
    
    return {
        "total_cost": total_cost,
        "local_requests": local_count,
        "cloud_requests": cloud_count,
        "routing_accuracy": routing_correct / len(results),
        "estimated_savings_percent": (cloud_only_cost - total_cost) / cloud_only_cost * 100 if cloud_only_cost > 0 else 0
    }

async def main():
    """Hauptfunktion - führt alle Tests durch"""
    print("\n" + "=" * 65)
    print("    SAAP EVALUATIONSTESTS - H2 (Skalierbarkeit) & H3 (Kosten)")
    print("=" * 65)
    print(f"Datum: {time.strftime('%Y-%m-%d %H:%M:%S')}")
    print("=" * 65)
    
    # H2 Test
    h2_results = await test_h2_scalability()
    
    # H3 Test
    h3_results = await test_h3_costs()
    
    # Zusammenfassung
    print("\n" + "=" * 65)
    print("    ZUSAMMENFASSUNG DER TESTERGEBNISSE")
    print("=" * 65)
    
    print("\n📈 H2 (Skalierbarkeit):")
    print(f"   → Ø Latenz bei 10 parallelen Anfragen: {h2_results['mean']:.2f}s")
    print(f"   → Erfolgsrate: {h2_results['success_rate']*100:.0f}%")
    h2_status = "✓ ERFÜLLT" if h2_results['success_rate'] >= 0.8 else "○ TEILWEISE"
    print(f"   → Status: {h2_status}")
    
    print("\n💰 H3 (Kostenvorteile):")
    print(f"   → Routing-Korrektheit: {h3_results['routing_accuracy']*100:.0f}%")
    print(f"   → Geschätzte Ersparnis: {h3_results['estimated_savings_percent']:.1f}%")
    h3_status = "✓ ERFÜLLT" if h3_results['routing_accuracy'] >= 0.8 else "○ TEILWEISE"
    print(f"   → Status: {h3_status}")
    
    print("\n" + "=" * 65)
    print("Tests abgeschlossen.")
    
    return {"h2": h2_results, "h3": h3_results}

if __name__ == "__main__":
    asyncio.run(main())