server/simulation/scenarios/task3_multi_root.py

from .base_scenario import BaseScenario
from server.simulation.topology import ENTERPRISE_TOPOLOGY
from server.models.observation import ServiceMetrics, DependencyEdge, Alert
from typing import Dict, Any, List

class Task3MultiRootCauseScenario(BaseScenario):
    """
    TWO simultaneous independent root causes:
    (A) redis-cache Redis cluster in eviction storm: maxmemory hit, 88% of keys evicted
        - Affects: rate-limiter (throttling breaking), inventory-service (cache misses),
          fraud-detection (feature store unavailable)
    (B) inventory-service bad deployment 15 minutes ago with memory leak in Go code
        - Affects: order-service (cannot check stock), inventory-db (hammered by retries)
    
    Complexity: 12 active alerts, most point to downstream services
    Evidence contradictory: order-service errors could be from redis OR inventory
    Correct ORDER matters: fix redis first, then rollback inventory
    """
    
    def get_root_causes(self) -> List[str]:
        return ["redis-cache", "inventory-service"]
    
    def get_initial_state(self) -> Dict[str, Any]:
        service_metrics = []
        for service_name, config in ENTERPRISE_TOPOLOGY.items():
            service_metrics.append(ServiceMetrics(
                service=service_name,
                status="healthy",
                error_rate=0.0,
                latency_p99_ms=config.get("slo_latency_ms", 100),
                cpu_percent=20.0,
                memory_percent=35.0,
                rps=200.0,
                replica_count=config.get("replicas", 1)
            ))

        # Dependency graph
        dependency_graph = []
        for service_name, config in ENTERPRISE_TOPOLOGY.items():
            for dep in config["depends_on"]:
                dependency_graph.append(DependencyEdge(
                    source=service_name,
                    target=dep,
                    protocol="HTTP",
                    is_healthy=True,
                    error_rate_on_edge=0.0
                ))

        state = {
            "service_metrics": service_metrics,
            "dependency_graph": dependency_graph,
            "blast_radius": 0.0,
            "error_budget_burn_rate": 0.0,
            "active_alerts": [],
            "logs": [],
            "actions_taken": 0,
            "safety_violations": 0,
            "episode_id": "task3_multi_root_test",
            "step": 0,
            "elapsed_seconds": 0.0,
            "max_steps": 30,
            "deploy_history_checked": {},
            "cache_flushed_services": set(),
        }

        self.apply_scenario_effects(state)
        return state

    def apply_scenario_effects(self, state: Dict[str, Any]) -> None:
        # Root Cause A: Redis cache in eviction storm
        for sm in state["service_metrics"]:
            if sm.service == "redis-cache":
                sm.status = "degraded"
                sm.cache_hit_rate = 0.12  # 88% eviction rate
                sm.memory_percent = 99.5
                break

        # Root Cause B: Inventory service memory leak
        for sm in state["service_metrics"]:
            if sm.service == "inventory-service":
                sm.status = "degraded"
                sm.memory_percent = 91.0
                sm.error_rate = 0.38
                break

        # Downstream effects of Redis degradation
        for sm in state["service_metrics"]:
            if sm.service == "rate-limiter":
                sm.status = "degraded"
                sm.error_rate = 0.15  # Throttling broken
                break
        
        for sm in state["service_metrics"]:
            if sm.service == "fraud-detection":
                sm.status = "critical"
                sm.error_rate = 0.42
                sm.queue_consumer_lag = 62000
                break

        # Downstream effects of Inventory degradation
        for sm in state["service_metrics"]:
            if sm.service == "order-service":
                sm.status = "critical"
                sm.error_rate = 0.65
                break
        
        for sm in state["service_metrics"]:
            if sm.service == "inventory-db":
                sm.status = "degraded"
                sm.db_conn_pool_utilization = 0.85
                break

        # API gateway experiences latency from rate-limiter
        for sm in state["service_metrics"]:
            if sm.service == "api-gateway":
                sm.latency_p99_ms = 850
                break

        # Red herring: HAProxy CPU spike (normal high-traffic pattern)
        for sm in state["service_metrics"]:
            if sm.service == "haproxy-lb":
                sm.cpu_percent = 75.0
                break

        state["blast_radius"] = 0.75
        state["active_alerts"] = [
            Alert(
                alert_id="redis-memory-1",
                severity="SEV2",
                service="redis-cache",
                title="Redis eviction storm",
                description="Eviction rate 88%, cache hit rate 12%",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=900.0
            ),
            Alert(
                alert_id="inv-service-memory-1",
                severity="SEV2",
                service="inventory-service",
                title="Inventory service memory leak",
                description="Memory 91%, growing over time",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=900.0
            ),
            Alert(
                alert_id="order-critical-1",
                severity="SEV1",
                service="order-service",
                title="Order service critical - 65% error rate",
                description="Checkout unavailable",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=600.0
            ),
            Alert(
                alert_id="fraud-critical-1",
                severity="SEV1",
                service="fraud-detection",
                title="Fraud detection degraded",
                description="42% error rate, queue lag 62k",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=600.0
            ),
            Alert(
                alert_id="rate-limit-err-1",
                severity="SEV2",
                service="rate-limiter",
                title="Rate limiter errors",
                description="Throttling broken, 15% failures",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=500.0
            ),
            Alert(
                alert_id="api-gw-latency-1",
                severity="SEV3",
                service="api-gateway",
                title="API gateway latency spike",
                description="p99 850ms (SLO 100ms)",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=400.0
            ),
            Alert(
                alert_id="inv-db-conn-1",
                severity="SEV3",
                service="inventory-db",
                title="Inventory DB connection pool",
                description="85% utilization",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=350.0
            ),
            Alert(
                alert_id="lb-cpu-1",
                severity="SEV3",
                service="haproxy-lb",
                title="HAProxy high CPU",
                description="CPU 75% (normal for high traffic)",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=300.0
            ),
            Alert(
                alert_id="order-error-2",
                severity="SEV2",
                service="order-service",
                title="Order service degraded",
                description="Cannot check inventory",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=550.0
            ),
            Alert(
                alert_id="fraud-lag-2",
                severity="SEV3",
                service="fraud-detection",
                title="Fraud consumer lag",
                description="Processing events delayed",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=600.0
            ),
            Alert(
                alert_id="cache-hit-1",
                severity="SEV3",
                service="inventory-service",
                title="Cache effectiveness low",
                description="Hit rate degraded",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=700.0
            ),
            Alert(
                alert_id="payment-margin-1",
                severity="SEV4",
                service="payment-service",
                title="Payment service margin warning",
                description="Slightly elevated latency",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=450.0
            ),
        ]

        state["service_deployments"] = {
            "inventory-service": {
                "current_sha": "b9d3e7f2",
                "deployed_at_minutes_ago": 15,
                "stable_sha": "c1a2b3e4"
            }
        }