Spaces:
Sleeping
Sleeping
| from __future__ import annotations | |
| from tasks.base_task import BaseTask | |
| class MediumTask(BaseTask): | |
| def __init__(self) -> None: | |
| self.task_id = "cascading-db-overload" | |
| self.difficulty = "medium" | |
| self.incident_summary = ( | |
| "checkout-service and order-processor are degraded. Root signal suggests a " | |
| "Postgres connection pool exhaustion after config drift." | |
| ) | |
| self.all_services = [ | |
| "postgres", | |
| "checkout-service", | |
| "order-processor", | |
| "auth-service", | |
| ] | |
| self.root_cause_service = "postgres" | |
| self.affected_services = ["postgres", "checkout-service", "order-processor"] | |
| self.dependency_graph = { | |
| "postgres": [], | |
| "checkout-service": ["postgres"], | |
| "order-processor": ["postgres", "checkout-service"], | |
| "auth-service": ["postgres"], | |
| } | |
| self.resolution_actions = [ | |
| { | |
| "action_type": "update_config", | |
| "params": { | |
| "service": "postgres", | |
| "key": "max_connections", | |
| "value": 50, | |
| }, | |
| "allow_string_number": True, | |
| "resolves_fault": True, | |
| "state_updates": { | |
| "postgres": { | |
| "fault_active": False, | |
| "fault_type": None, | |
| "config": {"max_connections": 50}, | |
| "metrics": { | |
| "status": "healthy", | |
| "error_rate": 0.03, | |
| "latency_p99_ms": 35.0, | |
| "cpu_percent": 55.0, | |
| }, | |
| } | |
| }, | |
| }, | |
| { | |
| "action_type": "restart_service", | |
| "params": {"service": "checkout-service"}, | |
| "resolves_fault": True, | |
| "state_updates": { | |
| "checkout-service": { | |
| "fault_active": False, | |
| "fault_type": None, | |
| "metrics": { | |
| "status": "healthy", | |
| "error_rate": 0.04, | |
| "latency_p99_ms": 300.0, | |
| "cpu_percent": 45.0, | |
| }, | |
| } | |
| }, | |
| }, | |
| { | |
| "action_type": "mark_resolved", | |
| "params": {}, | |
| }, | |
| ] | |
| self.red_herring_services = ["auth-service"] | |
| self.strict_sequence = True | |
| def initial_state(self) -> dict: | |
| return { | |
| "postgres": { | |
| "metrics": { | |
| "service": "postgres", | |
| "cpu_percent": 95.0, | |
| "memory_mb": 3800.0, | |
| "error_rate": 0.80, | |
| "latency_p99_ms": 8000.0, | |
| "status": "degraded", | |
| }, | |
| "logs": [ | |
| { | |
| "timestamp": "2026-04-11T10:03:00Z", | |
| "service": "postgres", | |
| "level": "FATAL", | |
| "message": "FATAL: remaining connection slots are reserved", | |
| }, | |
| { | |
| "timestamp": "2026-04-11T10:03:05Z", | |
| "service": "postgres", | |
| "level": "ERROR", | |
| "message": "connection pool exhausted: max_connections=5", | |
| }, | |
| ], | |
| "deployments": [ | |
| { | |
| "service": "postgres", | |
| "version": "14.11", | |
| "deployed_at": "2026-03-28T03:00:00Z", | |
| "deployed_by": "db-admin", | |
| "config_hash": "cfg-db-overload", | |
| } | |
| ], | |
| "config": {"max_connections": 5}, | |
| "fault_active": True, | |
| "fault_type": "exhausted_pool", | |
| "current_version": "14.11", | |
| "secondary_fault": False, | |
| "replicas": 1, | |
| }, | |
| "checkout-service": { | |
| "metrics": { | |
| "service": "checkout-service", | |
| "cpu_percent": 70.0, | |
| "memory_mb": 900.0, | |
| "error_rate": 0.60, | |
| "latency_p99_ms": 5000.0, | |
| "status": "degraded", | |
| }, | |
| "logs": [ | |
| { | |
| "timestamp": "2026-04-11T10:03:10Z", | |
| "service": "checkout-service", | |
| "level": "ERROR", | |
| "message": "could not connect to database: too many clients", | |
| } | |
| ], | |
| "deployments": [ | |
| { | |
| "service": "checkout-service", | |
| "version": "v3.7.0", | |
| "deployed_at": "2026-04-11T08:20:00Z", | |
| "deployed_by": "ci", | |
| "config_hash": "cfg-checkout-370", | |
| } | |
| ], | |
| "config": {}, | |
| "fault_active": True, | |
| "fault_type": "secondary_db_pressure", | |
| "current_version": "v3.7.0", | |
| "secondary_fault": True, | |
| "requires_restart_after_upstream_fix": True, | |
| "replicas": 4, | |
| }, | |
| "order-processor": { | |
| "metrics": { | |
| "service": "order-processor", | |
| "cpu_percent": 65.0, | |
| "memory_mb": 850.0, | |
| "error_rate": 0.55, | |
| "latency_p99_ms": 6200.0, | |
| "status": "degraded", | |
| }, | |
| "logs": [ | |
| { | |
| "timestamp": "2026-04-11T10:03:12Z", | |
| "service": "order-processor", | |
| "level": "ERROR", | |
| "message": "downstream checkout timeout while acquiring DB session", | |
| } | |
| ], | |
| "deployments": [ | |
| { | |
| "service": "order-processor", | |
| "version": "v2.9.3", | |
| "deployed_at": "2026-04-11T08:30:00Z", | |
| "deployed_by": "ci", | |
| "config_hash": "cfg-order-293", | |
| } | |
| ], | |
| "config": {}, | |
| "fault_active": True, | |
| "fault_type": "secondary_cascade", | |
| "current_version": "v2.9.3", | |
| "secondary_fault": True, | |
| "replicas": 3, | |
| }, | |
| "auth-service": { | |
| "metrics": { | |
| "service": "auth-service", | |
| "cpu_percent": 25.0, | |
| "memory_mb": 400.0, | |
| "error_rate": 0.05, | |
| "latency_p99_ms": 200.0, | |
| "status": "healthy", | |
| }, | |
| "logs": [ | |
| { | |
| "timestamp": "2026-04-11T09:58:00Z", | |
| "service": "auth-service", | |
| "level": "WARN", | |
| "message": "deploy event: v1.4.2 rolled out", | |
| } | |
| ], | |
| "deployments": [ | |
| { | |
| "service": "auth-service", | |
| "version": "v1.4.2", | |
| "deployed_at": "2026-04-11T09:56:00Z", | |
| "deployed_by": "release-bot", | |
| "config_hash": "cfg-auth-142", | |
| } | |
| ], | |
| "config": {}, | |
| "fault_active": False, | |
| "fault_type": None, | |
| "current_version": "v1.4.2", | |
| "secondary_fault": False, | |
| "replicas": 2, | |
| }, | |
| } | |
| def is_resolved(self, state: dict) -> bool: | |
| postgres = state.get("postgres", {}) | |
| checkout = state.get("checkout-service", {}) | |
| order_proc = state.get("order-processor", {}) | |
| return ( | |
| postgres.get("fault_active") is False | |
| and checkout.get("fault_active") is False | |
| and order_proc.get("fault_active") is False | |
| ) | |