from __future__ import annotations from tasks.base_task import BaseTask class MediumTask(BaseTask): def __init__(self) -> None: self.task_id = "cascading-db-overload" self.difficulty = "medium" self.incident_summary = ( "checkout-service and order-processor are degraded. Root signal suggests a " "Postgres connection pool exhaustion after config drift." ) self.all_services = [ "postgres", "checkout-service", "order-processor", "auth-service", ] self.root_cause_service = "postgres" self.affected_services = ["postgres", "checkout-service", "order-processor"] self.dependency_graph = { "postgres": [], "checkout-service": ["postgres"], "order-processor": ["postgres", "checkout-service"], "auth-service": ["postgres"], } self.resolution_actions = [ { "action_type": "update_config", "params": { "service": "postgres", "key": "max_connections", "value": 50, }, "allow_string_number": True, "resolves_fault": True, "state_updates": { "postgres": { "fault_active": False, "fault_type": None, "config": {"max_connections": 50}, "metrics": { "status": "healthy", "error_rate": 0.03, "latency_p99_ms": 35.0, "cpu_percent": 55.0, }, } }, }, { "action_type": "restart_service", "params": {"service": "checkout-service"}, "resolves_fault": True, "state_updates": { "checkout-service": { "fault_active": False, "fault_type": None, "metrics": { "status": "healthy", "error_rate": 0.04, "latency_p99_ms": 300.0, "cpu_percent": 45.0, }, } }, }, { "action_type": "mark_resolved", "params": {}, }, ] self.red_herring_services = ["auth-service"] self.strict_sequence = True def initial_state(self) -> dict: return { "postgres": { "metrics": { "service": "postgres", "cpu_percent": 95.0, "memory_mb": 3800.0, "error_rate": 0.80, "latency_p99_ms": 8000.0, "status": "degraded", }, "logs": [ { "timestamp": "2026-04-11T10:03:00Z", "service": "postgres", "level": "FATAL", "message": "FATAL: remaining connection slots are reserved", }, { "timestamp": "2026-04-11T10:03:05Z", "service": "postgres", "level": "ERROR", "message": "connection pool exhausted: max_connections=5", }, ], "deployments": [ { "service": "postgres", "version": "14.11", "deployed_at": "2026-03-28T03:00:00Z", "deployed_by": "db-admin", "config_hash": "cfg-db-overload", } ], "config": {"max_connections": 5}, "fault_active": True, "fault_type": "exhausted_pool", "current_version": "14.11", "secondary_fault": False, "replicas": 1, }, "checkout-service": { "metrics": { "service": "checkout-service", "cpu_percent": 70.0, "memory_mb": 900.0, "error_rate": 0.60, "latency_p99_ms": 5000.0, "status": "degraded", }, "logs": [ { "timestamp": "2026-04-11T10:03:10Z", "service": "checkout-service", "level": "ERROR", "message": "could not connect to database: too many clients", } ], "deployments": [ { "service": "checkout-service", "version": "v3.7.0", "deployed_at": "2026-04-11T08:20:00Z", "deployed_by": "ci", "config_hash": "cfg-checkout-370", } ], "config": {}, "fault_active": True, "fault_type": "secondary_db_pressure", "current_version": "v3.7.0", "secondary_fault": True, "requires_restart_after_upstream_fix": True, "replicas": 4, }, "order-processor": { "metrics": { "service": "order-processor", "cpu_percent": 65.0, "memory_mb": 850.0, "error_rate": 0.55, "latency_p99_ms": 6200.0, "status": "degraded", }, "logs": [ { "timestamp": "2026-04-11T10:03:12Z", "service": "order-processor", "level": "ERROR", "message": "downstream checkout timeout while acquiring DB session", } ], "deployments": [ { "service": "order-processor", "version": "v2.9.3", "deployed_at": "2026-04-11T08:30:00Z", "deployed_by": "ci", "config_hash": "cfg-order-293", } ], "config": {}, "fault_active": True, "fault_type": "secondary_cascade", "current_version": "v2.9.3", "secondary_fault": True, "replicas": 3, }, "auth-service": { "metrics": { "service": "auth-service", "cpu_percent": 25.0, "memory_mb": 400.0, "error_rate": 0.05, "latency_p99_ms": 200.0, "status": "healthy", }, "logs": [ { "timestamp": "2026-04-11T09:58:00Z", "service": "auth-service", "level": "WARN", "message": "deploy event: v1.4.2 rolled out", } ], "deployments": [ { "service": "auth-service", "version": "v1.4.2", "deployed_at": "2026-04-11T09:56:00Z", "deployed_by": "release-bot", "config_hash": "cfg-auth-142", } ], "config": {}, "fault_active": False, "fault_type": None, "current_version": "v1.4.2", "secondary_fault": False, "replicas": 2, }, } def is_resolved(self, state: dict) -> bool: postgres = state.get("postgres", {}) checkout = state.get("checkout-service", {}) order_proc = state.get("order-processor", {}) return ( postgres.get("fault_active") is False and checkout.get("fault_active") is False and order_proc.get("fault_active") is False )