Spaces:
Sleeping
Sleeping
| from __future__ import annotations | |
| from tasks.base_task import BaseTask | |
| class EasyTask(BaseTask): | |
| def __init__(self) -> None: | |
| self.task_id = "single-config-fault" | |
| self.difficulty = "easy" | |
| self.incident_summary = ( | |
| "api-gateway is returning widespread 502s after a bad upstream configuration " | |
| "change. Diagnose and remediate safely." | |
| ) | |
| self.all_services = ["api-gateway", "backend", "postgres"] | |
| self.root_cause_service = "api-gateway" | |
| self.affected_services = ["api-gateway"] | |
| self.dependency_graph = { | |
| "api-gateway": ["backend"], | |
| "backend": ["postgres"], | |
| "postgres": [], | |
| } | |
| self.resolution_actions = [ | |
| { | |
| "action_type": "update_config", | |
| "params": { | |
| "service": "api-gateway", | |
| "key": "upstream_url", | |
| "value": "http://backend:8080", | |
| }, | |
| "resolves_fault": True, | |
| "state_updates": { | |
| "api-gateway": { | |
| "fault_active": False, | |
| "fault_type": None, | |
| "metrics": { | |
| "status": "healthy", | |
| "error_rate": 0.01, | |
| "latency_p99_ms": 150.0, | |
| }, | |
| "config": {"upstream_url": "http://backend:8080"}, | |
| } | |
| }, | |
| }, | |
| { | |
| "action_type": "rollback_deployment", | |
| "params": { | |
| "service": "api-gateway", | |
| "version": "v2.0.9", | |
| }, | |
| "allow_missing_version": True, | |
| "resolves_fault": True, | |
| "state_updates": { | |
| "api-gateway": { | |
| "fault_active": False, | |
| "fault_type": None, | |
| "current_version": "v2.0.9", | |
| "metrics": { | |
| "status": "healthy", | |
| "error_rate": 0.01, | |
| "latency_p99_ms": 145.0, | |
| }, | |
| "config": {"upstream_url": "http://backend:8080"}, | |
| } | |
| }, | |
| }, | |
| ] | |
| self.red_herring_services = ["postgres"] | |
| def initial_state(self) -> dict: | |
| return { | |
| "api-gateway": { | |
| "metrics": { | |
| "service": "api-gateway", | |
| "cpu_percent": 45.0, | |
| "memory_mb": 512.0, | |
| "error_rate": 0.95, | |
| "latency_p99_ms": 3200.0, | |
| "status": "degraded", | |
| }, | |
| "logs": [ | |
| { | |
| "timestamp": "2026-04-11T10:01:03Z", | |
| "service": "api-gateway", | |
| "level": "FATAL", | |
| "message": "upstream connect error: connection refused (http://backend-old:8080)", | |
| }, | |
| { | |
| "timestamp": "2026-04-11T10:01:04Z", | |
| "service": "api-gateway", | |
| "level": "ERROR", | |
| "message": "502 Bad Gateway returned to client", | |
| }, | |
| { | |
| "timestamp": "2026-04-11T10:01:05Z", | |
| "service": "api-gateway", | |
| "level": "WARN", | |
| "message": "Health check failed for upstream backend-old", | |
| }, | |
| ], | |
| "deployments": [ | |
| { | |
| "service": "api-gateway", | |
| "version": "v2.1.0", | |
| "deployed_at": "2026-04-11T09:45:00Z", | |
| "deployed_by": "oncall-bot", | |
| "config_hash": "cfg-gw-bad-001", | |
| }, | |
| { | |
| "service": "api-gateway", | |
| "version": "v2.0.9", | |
| "deployed_at": "2026-04-10T22:15:00Z", | |
| "deployed_by": "release-engineer", | |
| "config_hash": "cfg-gw-good-778", | |
| }, | |
| ], | |
| "config": {"upstream_url": "http://backend-old:8080"}, | |
| "fault_active": True, | |
| "fault_type": "wrong_config", | |
| "current_version": "v2.1.0", | |
| "secondary_fault": False, | |
| "replicas": 3, | |
| }, | |
| "backend": { | |
| "metrics": { | |
| "service": "backend", | |
| "cpu_percent": 30.0, | |
| "memory_mb": 768.0, | |
| "error_rate": 0.02, | |
| "latency_p99_ms": 120.0, | |
| "status": "healthy", | |
| }, | |
| "logs": [ | |
| { | |
| "timestamp": "2026-04-11T10:00:00Z", | |
| "service": "backend", | |
| "level": "INFO", | |
| "message": "backend healthy", | |
| } | |
| ], | |
| "deployments": [ | |
| { | |
| "service": "backend", | |
| "version": "v5.4.1", | |
| "deployed_at": "2026-04-11T08:00:00Z", | |
| "deployed_by": "ci", | |
| "config_hash": "cfg-be-541", | |
| } | |
| ], | |
| "config": {}, | |
| "fault_active": False, | |
| "fault_type": None, | |
| "current_version": "v5.4.1", | |
| "secondary_fault": False, | |
| "replicas": 4, | |
| }, | |
| "postgres": { | |
| "metrics": { | |
| "service": "postgres", | |
| "cpu_percent": 20.0, | |
| "memory_mb": 2048.0, | |
| "error_rate": 0.0, | |
| "latency_p99_ms": 8.0, | |
| "status": "healthy", | |
| }, | |
| "logs": [ | |
| { | |
| "timestamp": "2026-04-11T10:00:01Z", | |
| "service": "postgres", | |
| "level": "INFO", | |
| "message": "autovacuum completed", | |
| } | |
| ], | |
| "deployments": [ | |
| { | |
| "service": "postgres", | |
| "version": "14.11", | |
| "deployed_at": "2026-03-20T02:00:00Z", | |
| "deployed_by": "db-admin", | |
| "config_hash": "cfg-db-1411", | |
| } | |
| ], | |
| "config": {}, | |
| "fault_active": False, | |
| "fault_type": None, | |
| "current_version": "14.11", | |
| "secondary_fault": False, | |
| "replicas": 1, | |
| }, | |
| } | |
| def is_resolved(self, state: dict) -> bool: | |
| api_state = state.get("api-gateway", {}) | |
| metrics = api_state.get("metrics", {}) | |
| return ( | |
| api_state.get("fault_active") is False | |
| and metrics.get("status") == "healthy" | |
| ) | |