from .base_scenario import BaseScenario from server.simulation.topology import ENTERPRISE_TOPOLOGY from server.models.observation import ServiceMetrics, DependencyEdge, Alert from typing import Dict, Any, List class Task3MultiRootCauseScenario(BaseScenario): """ TWO simultaneous independent root causes: (A) redis-cache Redis cluster in eviction storm: maxmemory hit, 88% of keys evicted - Affects: rate-limiter (throttling breaking), inventory-service (cache misses), fraud-detection (feature store unavailable) (B) inventory-service bad deployment 15 minutes ago with memory leak in Go code - Affects: order-service (cannot check stock), inventory-db (hammered by retries) Complexity: 12 active alerts, most point to downstream services Evidence contradictory: order-service errors could be from redis OR inventory Correct ORDER matters: fix redis first, then rollback inventory """ def get_root_causes(self) -> List[str]: return ["redis-cache", "inventory-service"] def get_initial_state(self) -> Dict[str, Any]: service_metrics = [] for service_name, config in ENTERPRISE_TOPOLOGY.items(): service_metrics.append(ServiceMetrics( service=service_name, status="healthy", error_rate=0.0, latency_p99_ms=config.get("slo_latency_ms", 100), cpu_percent=20.0, memory_percent=35.0, rps=200.0, replica_count=config.get("replicas", 1) )) # Dependency graph dependency_graph = [] for service_name, config in ENTERPRISE_TOPOLOGY.items(): for dep in config["depends_on"]: dependency_graph.append(DependencyEdge( source=service_name, target=dep, protocol="HTTP", is_healthy=True, error_rate_on_edge=0.0 )) state = { "service_metrics": service_metrics, "dependency_graph": dependency_graph, "blast_radius": 0.0, "error_budget_burn_rate": 0.0, "active_alerts": [], "logs": [], "actions_taken": 0, "safety_violations": 0, "episode_id": "task3_multi_root_test", "step": 0, "elapsed_seconds": 0.0, "max_steps": 30, "deploy_history_checked": {}, "cache_flushed_services": set(), } self.apply_scenario_effects(state) return state def apply_scenario_effects(self, state: Dict[str, Any]) -> None: # Root Cause A: Redis cache in eviction storm for sm in state["service_metrics"]: if sm.service == "redis-cache": sm.status = "degraded" sm.cache_hit_rate = 0.12 # 88% eviction rate sm.memory_percent = 99.5 break # Root Cause B: Inventory service memory leak for sm in state["service_metrics"]: if sm.service == "inventory-service": sm.status = "degraded" sm.memory_percent = 91.0 sm.error_rate = 0.38 break # Downstream effects of Redis degradation for sm in state["service_metrics"]: if sm.service == "rate-limiter": sm.status = "degraded" sm.error_rate = 0.15 # Throttling broken break for sm in state["service_metrics"]: if sm.service == "fraud-detection": sm.status = "critical" sm.error_rate = 0.42 sm.queue_consumer_lag = 62000 break # Downstream effects of Inventory degradation for sm in state["service_metrics"]: if sm.service == "order-service": sm.status = "critical" sm.error_rate = 0.65 break for sm in state["service_metrics"]: if sm.service == "inventory-db": sm.status = "degraded" sm.db_conn_pool_utilization = 0.85 break # API gateway experiences latency from rate-limiter for sm in state["service_metrics"]: if sm.service == "api-gateway": sm.latency_p99_ms = 850 break # Red herring: HAProxy CPU spike (normal high-traffic pattern) for sm in state["service_metrics"]: if sm.service == "haproxy-lb": sm.cpu_percent = 75.0 break state["blast_radius"] = 0.75 state["active_alerts"] = [ Alert( alert_id="redis-memory-1", severity="SEV2", service="redis-cache", title="Redis eviction storm", description="Eviction rate 88%, cache hit rate 12%", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=900.0 ), Alert( alert_id="inv-service-memory-1", severity="SEV2", service="inventory-service", title="Inventory service memory leak", description="Memory 91%, growing over time", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=900.0 ), Alert( alert_id="order-critical-1", severity="SEV1", service="order-service", title="Order service critical - 65% error rate", description="Checkout unavailable", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=600.0 ), Alert( alert_id="fraud-critical-1", severity="SEV1", service="fraud-detection", title="Fraud detection degraded", description="42% error rate, queue lag 62k", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=600.0 ), Alert( alert_id="rate-limit-err-1", severity="SEV2", service="rate-limiter", title="Rate limiter errors", description="Throttling broken, 15% failures", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=500.0 ), Alert( alert_id="api-gw-latency-1", severity="SEV3", service="api-gateway", title="API gateway latency spike", description="p99 850ms (SLO 100ms)", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=400.0 ), Alert( alert_id="inv-db-conn-1", severity="SEV3", service="inventory-db", title="Inventory DB connection pool", description="85% utilization", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=350.0 ), Alert( alert_id="lb-cpu-1", severity="SEV3", service="haproxy-lb", title="HAProxy high CPU", description="CPU 75% (normal for high traffic)", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=300.0 ), Alert( alert_id="order-error-2", severity="SEV2", service="order-service", title="Order service degraded", description="Cannot check inventory", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=550.0 ), Alert( alert_id="fraud-lag-2", severity="SEV3", service="fraud-detection", title="Fraud consumer lag", description="Processing events delayed", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=600.0 ), Alert( alert_id="cache-hit-1", severity="SEV3", service="inventory-service", title="Cache effectiveness low", description="Hit rate degraded", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=700.0 ), Alert( alert_id="payment-margin-1", severity="SEV4", service="payment-service", title="Payment service margin warning", description="Slightly elevated latency", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=450.0 ), ] state["service_deployments"] = { "inventory-service": { "current_sha": "b9d3e7f2", "deployed_at_minutes_ago": 15, "stable_sha": "c1a2b3e4" } }