server/simulation/scenarios/task2_cascade.py

from .base_scenario import BaseScenario
from server.simulation.topology import ENTERPRISE_TOPOLOGY
from server.models.observation import ServiceMetrics, DependencyEdge, Alert
from typing import Dict, Any, List

class Task2CascadeFailureScenario(BaseScenario):
    """
    payment-db PostgreSQL connection pool exhausted (99% used) because
    payment-service had a bad deployment 22 minutes ago
    - query holds connections too long (missing connection.close() call)
    - This cascades to payment-service → order-service failures
    - Red herring: notification-service backed up queue looks problematic
    """
    
    def get_root_causes(self) -> List[str]:
        return ["payment-db", "payment-service"]
    
    def get_initial_state(self) -> Dict[str, Any]:
        # Start with all services mostly healthy
        service_metrics = []
        for service_name, config in ENTERPRISE_TOPOLOGY.items():
            service_metrics.append(ServiceMetrics(
                service=service_name,
                status="healthy",
                error_rate=0.0,
                latency_p99_ms=config.get("slo_latency_ms", 100),
                cpu_percent=15.0,
                memory_percent=30.0,
                rps=150.0,
                replica_count=config.get("replicas", 1)
            ))

        # Dependency graph
        dependency_graph = []
        for service_name, config in ENTERPRISE_TOPOLOGY.items():
            for dep in config["depends_on"]:
                dependency_graph.append(DependencyEdge(
                    source=service_name,
                    target=dep,
                    protocol="HTTP",
                    is_healthy=True,
                    error_rate_on_edge=0.0
                ))

        state = {
            "service_metrics": service_metrics,
            "dependency_graph": dependency_graph,
            "blast_radius": 0.0,
            "error_budget_burn_rate": 0.0,
            "active_alerts": [],
            "logs": [],
            "actions_taken": 0,
            "safety_violations": 0,
            "episode_id": "task2_cascade_test",
            "step": 0,
            "elapsed_seconds": 0.0,
            "max_steps": 25,
            "deploy_history_checked": {},
            "cache_flushed_services": set(),
        }

        # Apply scenario-specific effects
        self.apply_scenario_effects(state)
        return state

    def apply_scenario_effects(self, state: Dict[str, Any]) -> None:
        # payment-db connection pool exhausted
        for sm in state["service_metrics"]:
            if sm.service == "payment-db":
                sm.status = "critical"
                sm.db_conn_pool_utilization = 0.99
                sm.latency_p99_ms = 8500  # Huge latency due to connection wait
                break

        # payment-service cascading failure
        for sm in state["service_metrics"]:
            if sm.service == "payment-service":
                sm.status = "critical"
                sm.error_rate = 0.78
                sm.latency_p99_ms = 12000
                break

        # order-service degraded (depends on payment-service)
        for sm in state["service_metrics"]:
            if sm.service == "order-service":
                sm.status = "degraded"
                sm.error_rate = 0.45
                break

        # fraud-detection queue backed up
        for sm in state["service_metrics"]:
            if sm.service == "fraud-detection":
                sm.status = "degraded"
                sm.queue_consumer_lag = 45000
                break

        # notification-service backed up (red herring)
        for sm in state["service_metrics"]:
            if sm.service == "notification-service":
                sm.status = "degraded"
                sm.queue_consumer_lag = 12000
                break

        # Alerts
        state["blast_radius"] = 0.70
        state["active_alerts"] = [
            Alert(
                alert_id="payment-critical-1",
                severity="SEV1",
                service="payment-service",
                title="Payment service 78% error rate",
                description="Unable to process transactions",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=1320.0  # 22 minutes
            ),
            Alert(
                alert_id="order-high-err-1",
                severity="SEV2",
                service="order-service",
                title="Order service degraded",
                description="Checkout failures at 45% rate",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=1080.0
            ),
            Alert(
                alert_id="db-conn-pool-1",
                severity="SEV2",
                service="payment-db",
                title="Connection pool exhaustion",
                description="99% of connections in use",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=900.0
            ),
            Alert(
                alert_id="fraud-queue-lag-1",
                severity="SEV3",
                service="fraud-detection",
                title="Fraud queue lag high",
                description="Consumer lag 45000, events delayed",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=600.0
            ),
            Alert(
                alert_id="notif-queue-lag-1",
                severity="SEV3",
                service="notification-service",
                title="Notification queue lag",
                description="Consumer lag 12000, emails delayed",
                triggered_at=0.0,
                is_acknowledged=False,
                firing_for_seconds=1200.0
            ),
        ]

        # Store deployment info for rollback
        state["service_deployments"] = {
            "payment-service": {
                "current_sha": "f4a8b2c1",
                "deployed_at_minutes_ago": 22,
                "stable_sha": "a1f3c9e2"
            }
        }