from .base_scenario import BaseScenario from server.simulation.topology import ENTERPRISE_TOPOLOGY from server.models.observation import ServiceMetrics, DependencyEdge, Alert from typing import Dict, Any, List class Task2CascadeFailureScenario(BaseScenario): """ payment-db PostgreSQL connection pool exhausted (99% used) because payment-service had a bad deployment 22 minutes ago - query holds connections too long (missing connection.close() call) - This cascades to payment-service → order-service failures - Red herring: notification-service backed up queue looks problematic """ def get_root_causes(self) -> List[str]: return ["payment-db", "payment-service"] def get_initial_state(self) -> Dict[str, Any]: # Start with all services mostly healthy service_metrics = [] for service_name, config in ENTERPRISE_TOPOLOGY.items(): service_metrics.append(ServiceMetrics( service=service_name, status="healthy", error_rate=0.0, latency_p99_ms=config.get("slo_latency_ms", 100), cpu_percent=15.0, memory_percent=30.0, rps=150.0, replica_count=config.get("replicas", 1) )) # Dependency graph dependency_graph = [] for service_name, config in ENTERPRISE_TOPOLOGY.items(): for dep in config["depends_on"]: dependency_graph.append(DependencyEdge( source=service_name, target=dep, protocol="HTTP", is_healthy=True, error_rate_on_edge=0.0 )) state = { "service_metrics": service_metrics, "dependency_graph": dependency_graph, "blast_radius": 0.0, "error_budget_burn_rate": 0.0, "active_alerts": [], "logs": [], "actions_taken": 0, "safety_violations": 0, "episode_id": "task2_cascade_test", "step": 0, "elapsed_seconds": 0.0, "max_steps": 25, "deploy_history_checked": {}, "cache_flushed_services": set(), } # Apply scenario-specific effects self.apply_scenario_effects(state) return state def apply_scenario_effects(self, state: Dict[str, Any]) -> None: # payment-db connection pool exhausted for sm in state["service_metrics"]: if sm.service == "payment-db": sm.status = "critical" sm.db_conn_pool_utilization = 0.99 sm.latency_p99_ms = 8500 # Huge latency due to connection wait break # payment-service cascading failure for sm in state["service_metrics"]: if sm.service == "payment-service": sm.status = "critical" sm.error_rate = 0.78 sm.latency_p99_ms = 12000 break # order-service degraded (depends on payment-service) for sm in state["service_metrics"]: if sm.service == "order-service": sm.status = "degraded" sm.error_rate = 0.45 break # fraud-detection queue backed up for sm in state["service_metrics"]: if sm.service == "fraud-detection": sm.status = "degraded" sm.queue_consumer_lag = 45000 break # notification-service backed up (red herring) for sm in state["service_metrics"]: if sm.service == "notification-service": sm.status = "degraded" sm.queue_consumer_lag = 12000 break # Alerts state["blast_radius"] = 0.70 state["active_alerts"] = [ Alert( alert_id="payment-critical-1", severity="SEV1", service="payment-service", title="Payment service 78% error rate", description="Unable to process transactions", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=1320.0 # 22 minutes ), Alert( alert_id="order-high-err-1", severity="SEV2", service="order-service", title="Order service degraded", description="Checkout failures at 45% rate", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=1080.0 ), Alert( alert_id="db-conn-pool-1", severity="SEV2", service="payment-db", title="Connection pool exhaustion", description="99% of connections in use", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=900.0 ), Alert( alert_id="fraud-queue-lag-1", severity="SEV3", service="fraud-detection", title="Fraud queue lag high", description="Consumer lag 45000, events delayed", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=600.0 ), Alert( alert_id="notif-queue-lag-1", severity="SEV3", service="notification-service", title="Notification queue lag", description="Consumer lag 12000, emails delayed", triggered_at=0.0, is_acknowledged=False, firing_for_seconds=1200.0 ), ] # Store deployment info for rollback state["service_deployments"] = { "payment-service": { "current_sha": "f4a8b2c1", "deployed_at_minutes_ago": 22, "stable_sha": "a1f3c9e2" } }