""" Task 3: Cascading Service Failure (Hard) ========================================= Scenario: Four production services are down simultaneously. A recent deployment of config-service v1.2.0 introduced a broken service-discovery URL, causing all services that depend on it to fail to resolve each other. Traps: - user-service has an unrelated memory warning (red herring) - order-service v2.1.0 was deployed 2h ago (not the cause) - Restarting any individual service without fixing config-service makes no progress - Need to investigate and identify config-service as the common dependency Optimal: check_config(config-service) OR query_logs(any-down-service) → rollback_deployment(config-service) → resolve_incident() Max steps: 20 | Passing score: 0.6 """ from typing import Dict, Any, Tuple, List from app.models import Observation, Alert, ServiceStatus from app.tasks.base import BaseTask, AVAILABLE_ACTIONS, BASE_INCIDENT_TIME class CascadingFailureTask(BaseTask): task_id = "task3" name = "Cascading Service Failure" description = ( "Four services (api-gateway, user-service, order-service, payment-service) " "are simultaneously down. Identify the common root cause and remediate " "the entire incident with minimal blast radius." ) difficulty = "hard" max_steps = 20 passing_score = 0.6 def initial_state(self, seed: int = 42) -> Dict[str, Any]: return { "services": { "api-gateway": { "status": "down", "cpu": 0.5, "memory": 12.0, "error_rate": 100.0, "version": "1.4.2", "replicas": 3, }, "user-service": { "status": "down", "cpu": 0.8, "memory": 88.0, "error_rate": 100.0, "version": "4.0.1", "replicas": 2, }, "order-service": { "status": "down", "cpu": 0.3, "memory": 45.0, "error_rate": 100.0, "version": "2.1.0", "replicas": 2, }, "payment-service": { "status": "down", "cpu": 0.2, "memory": 40.0, "error_rate": 100.0, "version": "5.2.3", "replicas": 2, }, "config-service": { "status": "healthy", "cpu": 18.0, "memory": 35.0, "error_rate": 0.0, "version": "1.2.0", "replicas": 1, }, "db-primary": { "status": "healthy", "cpu": 22.0, "memory": 55.0, "error_rate": 0.0, "connections": 15, "max_connections": 100, "version": "14.8", }, "message-queue": { "status": "healthy", "cpu": 10.0, "memory": 42.0, "error_rate": 0.0, "version": "3.12.0", }, }, "alerts": [ {"id": "ALT-020", "sev": "critical", "svc": "api-gateway", "msg": "api-gateway is DOWN — all requests returning 503", "ack": False}, {"id": "ALT-021", "sev": "critical", "svc": "user-service", "msg": "user-service is DOWN — health check failing for 8 minutes", "ack": False}, {"id": "ALT-022", "sev": "critical", "svc": "order-service", "msg": "order-service is DOWN — all replicas unhealthy", "ack": False}, {"id": "ALT-023", "sev": "critical", "svc": "payment-service", "msg": "payment-service is DOWN — cannot process transactions", "ack": False}, {"id": "ALT-024", "sev": "warning", "svc": "user-service", "msg": "user-service memory at 88% (elevated but not critical)", "ack": False}, ], "recent_deployments": [ {"service": "config-service", "version": "1.2.0", "previous": "1.1.9", "deployed_at": "2024-11-15T09:30:00Z", "deployer": "platform-team", "change": "Updated service discovery URLs for new datacenter migration"}, {"service": "order-service", "version": "2.1.0", "previous": "2.0.8", "deployed_at": "2024-11-15T07:45:00Z", "deployer": "ci-pipeline", "change": "New checkout flow feature"}, {"service": "user-service", "version": "4.0.1", "previous": "4.0.0", "deployed_at": "2024-11-14T14:00:00Z", "deployer": "ci-pipeline", "change": "Bug fix for profile update endpoint"}, ], # Tracking agent progress "logs_queried": [], "metrics_checked": [], "configs_checked": [], "services_restarted": [], "rollbacks_attempted": {}, "wrong_rollbacks": 0, "config_service_rolledback": False, "services_recovered": [], "incident_resolved": False, "_all_services_down_due_to_config": True, } def _check_recovery(self, state: Dict[str, Any]) -> None: """Update service statuses based on whether config-service was fixed.""" if state["config_service_rolledback"]: for svc in ["api-gateway", "user-service", "order-service", "payment-service"]: if svc not in state["services_recovered"]: state["services_recovered"].append(svc) state["services"][svc]["status"] = "healthy" state["services"][svc]["error_rate"] = 0.0 state["services"][svc]["cpu"] = float( {"api-gateway": 8.0, "user-service": 22.0, "order-service": 15.0, "payment-service": 12.0}[svc] ) def process_action( self, action_type: str, params: Dict[str, Any], state: Dict[str, Any] ) -> Tuple[Dict[str, Any], float, bool, str]: reward = 0.0 done = False message = "" service = params.get("service", "").strip() if action_type == "query_logs": if not service: return state, -0.02, False, "Parameter 'service' is required." if service in state["logs_queried"]: return state, 0.0, False, f"[Cached] Logs for {service} already retrieved." state["logs_queried"].append(service) down_services = ["api-gateway", "user-service", "order-service", "payment-service"] if service in down_services: reward = 0.08 service_logs = { "api-gateway": ( "2024-11-15T09:31:00Z [ERROR] api-gateway: failed to resolve " "user-service endpoint via config-service: " "GET http://config-service/discover/user-service → " "returned 'http://svc-mesh-BROKEN.internal:8080' (unreachable)\n" "2024-11-15T09:31:01Z [ERROR] api-gateway: failed to resolve " "order-service — same issue\n" "2024-11-15T09:31:05Z [FATAL] api-gateway: no healthy upstreams " "available — entering 503 mode\n" "PATTERN: All service discovery calls returning broken URLs from config-service." ), "user-service": ( "2024-11-15T09:31:00Z [ERROR] user-service: startup failed — cannot " "resolve db endpoint via config-service: received 'db-BROKEN.internal' " "(expected 'db-primary.internal')\n" "2024-11-15T09:31:02Z [FATAL] user-service: health check failed — " "cannot connect to database\n" "PATTERN: config-service returning incorrect service discovery data." ), "order-service": ( "2024-11-15T09:31:00Z [ERROR] order-service: failed to start — " "config-service returned broken payment-service URL\n" "2024-11-15T09:31:03Z [FATAL] order-service: dependency check failed, " "exiting\n" "NOTE: order-service v2.1.0 deployed at 07:45 ran fine until 09:30 " "when config-service was updated." ), "payment-service": ( "2024-11-15T09:31:00Z [ERROR] payment-service: cannot resolve fraud-check " "service — config-service lookup returned null endpoint\n" "2024-11-15T09:31:05Z [FATAL] payment-service: aborting startup due to " "missing required service dependencies\n" ), } message = service_logs.get(service, "No logs found.") elif service == "config-service": reward = 0.12 message = ( "2024-11-15T09:28:00Z [INFO] config-service: v1.2.0 deployment started\n" "2024-11-15T09:29:50Z [INFO] config-service: service discovery URLs updated " "for datacenter migration\n" "2024-11-15T09:30:00Z [INFO] config-service: v1.2.0 deployment complete\n" "2024-11-15T09:30:05Z [WARN] config-service: 4 downstream services " "reporting connection failures immediately after deploy\n" "2024-11-15T09:30:10Z [ERROR] config-service: config validation failed " "in post-deploy check — service_discovery_urls contain unreachable hosts\n" "ROOT CAUSE CONFIRMED: config-service v1.2.0 deployed broken service " "discovery URLs. All dependent services cannot resolve each other." ) else: reward = 0.02 message = f"No anomalies in logs for {service}." elif action_type == "check_metrics": if not service: return state, -0.02, False, "Parameter 'service' is required." if service in state["metrics_checked"]: return state, 0.0, False, f"[Cached] Metrics for {service} already retrieved." state["metrics_checked"].append(service) if service in ["api-gateway", "user-service", "order-service", "payment-service"]: reward = 0.06 message = ( f"{service} metrics:\n" f" status: DOWN\n" f" error_rate: 100% (all requests failing)\n" f" last_healthy: 2024-11-15T09:30:02Z\n" f" restart_attempts: 3 (all failed)\n" f" failure_reason: dependency resolution failure at startup\n" f"CORRELATES: All 4 services went down within 15 seconds of each other " f"at 09:30 — timing matches config-service v1.2.0 deployment." ) elif service == "config-service": reward = 0.08 message = ( "config-service metrics:\n" " status: healthy\n" " cpu: 18%\n" " requests_per_sec: 240\n" " cache_hit_rate: 12% ← very low (normal: 95%+)\n" " discovery_errors_per_sec: 180 ← HIGH\n" " version: 1.2.0 (deployed 17min ago)\n" "SUSPICIOUS: High discovery_errors and low cache_hit_rate after recent deploy." ) else: reward = 0.02 message = f"Metrics for {service}: Normal." elif action_type == "check_config": state["configs_checked"].append(service) if service == "config-service": reward = 0.15 message = ( "config-service LIVE CONFIG (v1.2.0):\n" " service_discovery:\n" " user-service: http://svc-mesh-BROKEN.dc2.internal:8080\n" " order-service: http://svc-mesh-BROKEN.dc2.internal:8081\n" " payment-service: http://svc-mesh-BROKEN.dc2.internal:8082\n" " db-primary: http://db-BROKEN.dc2.internal:5432\n" " api-gateway: http://gw-BROKEN.dc2.internal:80\n\n" "config-service PREVIOUS CONFIG (v1.1.9):\n" " service_discovery:\n" " user-service: http://user-service.svc.cluster.local:8080 ✓\n" " order-service: http://order-service.svc.cluster.local:8081 ✓\n" " payment-service: http://payment-service.svc.cluster.local:8082 ✓\n" " db-primary: http://db-primary.svc.cluster.local:5432 ✓\n\n" "ROOT CAUSE CONFIRMED: v1.2.0 changed ALL service discovery URLs to " "non-existent dc2.internal addresses. Datacenter migration was incomplete." ) elif service in ["api-gateway", "user-service", "order-service", "payment-service"]: reward = 0.04 message = ( f"{service} config appears normal. Service discovery endpoint " f"points to config-service (as expected). The issue is in what " f"config-service returns, not in {service}'s config itself." ) else: reward = 0.01 message = f"Config for {service}: Nothing unusual." elif action_type == "examine_trace": trace_id = params.get("trace_id", "unknown") state["logs_queried"].append(f"trace:{trace_id}") reward = 0.06 message = ( f"Trace {trace_id}:\n" " api-gateway → [service discovery lookup] → config-service (2ms)\n" " config-service → returned URL: http://svc-mesh-BROKEN.dc2.internal\n" " api-gateway → [connection attempt to broken URL] → TIMEOUT after 5000ms\n" " Root span: 100% of failures originate from bad service discovery response." ) elif action_type == "restart_service": if service in ["api-gateway", "user-service", "order-service", "payment-service"]: if not state["config_service_rolledback"]: state["services_restarted"].append(service) # Restarting without fixing config does nothing reward = -0.05 message = ( f"Restarted {service}... but it failed to start again.\n" f" Startup error: cannot resolve service dependencies via config-service\n" f" Status: still DOWN\n" f"The underlying config-service issue must be fixed first." ) else: # After config fix, manual restart not needed (auto-recovery) reward = 0.0 message = f"{service} already recovering after config-service rollback." elif service == "config-service": # Restarting config-service doesn't fix the bad config state["services_restarted"].append(service) reward = -0.08 message = ( "config-service restarted — but it loaded the same broken v1.2.0 config.\n" " All downstream services still failing.\n" " A restart does not fix a misconfiguration. Use rollback_deployment." ) else: reward = 0.0 message = f"{service} is healthy and does not need a restart." elif action_type == "rollback_deployment": if service == "config-service": state["config_service_rolledback"] = True self._check_recovery(state) reward = 0.45 message = ( "✓ config-service rolled back from v1.2.0 → v1.1.9.\n" " Service discovery URLs restored to cluster-internal addresses.\n" " api-gateway: DOWN → healthy (restarted automatically)\n" " user-service: DOWN → healthy (restarted automatically)\n" " order-service: DOWN → healthy (restarted automatically)\n" " payment-service: DOWN → healthy (restarted automatically)\n" "All 4 services recovered within 45 seconds of config-service rollback." ) elif service == "order-service": # Red herring — order-service v2.1.0 was NOT the cause state["rollbacks_attempted"][service] = True state["wrong_rollbacks"] += 1 reward = -0.08 message = ( "Rolled back order-service to v2.0.8... but it immediately failed again.\n" " Error: still cannot resolve service dependencies via config-service.\n" " RESULT: order-service v2.1.0 was not the root cause. " "The issue is upstream." ) elif service in ["api-gateway", "user-service", "payment-service"]: state["rollbacks_attempted"][service] = True state["wrong_rollbacks"] += 1 reward = -0.06 message = ( f"Rolled back {service}... but it still cannot start.\n" f" Error: service discovery failing — same as before.\n" f" This service is not the root cause." ) else: reward = -0.02 message = f"Rolling back {service} has no effect on the current incident." elif action_type == "scale_service": reward = -0.05 message = ( "Scaling has no effect — services are failing due to misconfiguration, " "not insufficient capacity." ) elif action_type == "acknowledge_alert": alert_id = params.get("alert_id", "") for a in state["alerts"]: if a["id"] == alert_id: a["ack"] = True reward = 0.01 message = f"Alert {alert_id} acknowledged." elif action_type == "resolve_incident": all_healthy = all( state["services"][svc]["status"] == "healthy" for svc in ["api-gateway", "user-service", "order-service", "payment-service"] ) if all_healthy: state["incident_resolved"] = True done = True reward = 0.25 message = ( "✓ Incident resolved.\n" "Post-mortem: config-service v1.2.0 was deployed with incorrect service " "discovery URLs targeting a non-existent dc2 datacenter. This caused all " "dependent services to fail at startup. Rollback to v1.1.9 restored service.\n" "Recommendation: Add config validation to deployment pipeline." ) else: still_down = [ s for s in ["api-gateway", "user-service", "order-service", "payment-service"] if state["services"][s]["status"] != "healthy" ] reward = -0.05 message = ( f"Cannot resolve: {len(still_down)} services still down: " f"{', '.join(still_down)}. Fix the root cause first." ) else: reward = -0.03 message = f"Unknown or inapplicable action: {action_type}." return state, reward, done, message def get_observation(self, state: Dict[str, Any], session_id: str, step: int) -> Observation: services = {} for name, s in state["services"].items(): services[name] = ServiceStatus( name=name, status=s["status"], cpu_percent=s["cpu"], memory_percent=s["memory"], error_rate=s["error_rate"], connections=s.get("connections"), max_connections=s.get("max_connections"), version=s.get("version", "1.0.0"), replicas=s.get("replicas", 1), ) alerts = [ Alert( alert_id=a["id"], severity=a["sev"], service=a["svc"], message=a["msg"], triggered_at=BASE_INCIDENT_TIME, acknowledged=a["ack"], ) for a in state["alerts"] ] return Observation( session_id=session_id, task_id=self.task_id, step=step, timestamp=BASE_INCIDENT_TIME, alerts=alerts, services=services, available_actions=AVAILABLE_ACTIONS, incident_resolved=state["incident_resolved"], message="", recent_deployments=state["recent_deployments"], runbook_hints=[ "When multiple services fail simultaneously, look for a common dependency.", "Check the timing: what changed just before the incident?", "check_config reveals live runtime configuration values.", "Restarting services without fixing the root cause will not help.", "rollback_deployment reverts to the previous known-good version.", ], ) def grade(self, state: Dict[str, Any], history: List[Dict]) -> Tuple[float, Dict[str, float]]: breakdown = {} score = 0.0 # Root cause investigated? root_investigated = ( "config-service" in state.get("configs_checked", []) or "config-service" in state.get("logs_queried", []) or "config-service" in state.get("metrics_checked", []) or any(svc in state.get("logs_queried", []) for svc in ["api-gateway", "user-service", "order-service", "payment-service"]) ) if root_investigated: breakdown["investigated_root_cause"] = 0.14 score += 0.14 # config-service identified and rolled back? if state.get("config_service_rolledback", False): breakdown["correct_rollback"] = 0.40 score += 0.40 # All services recovered? recovered = state.get("services_recovered", []) if len(recovered) >= 4: breakdown["full_recovery"] = 0.20 score += 0.20 elif len(recovered) >= 2: breakdown["partial_recovery"] = 0.10 score += 0.10 # Incident formally resolved? if state.get("incident_resolved", False): breakdown["incident_resolved"] = 0.15 score += 0.15 # Efficiency bonus steps = len(history) if steps <= 4: breakdown["efficiency_bonus"] = 0.10 score += 0.10 elif steps <= 7: breakdown["efficiency_bonus"] = 0.07 score += 0.07 elif steps <= 12: breakdown["efficiency_bonus"] = 0.03 score += 0.03 # Penalty for wrong rollbacks wrong_rollbacks = state.get("wrong_rollbacks", 0) if wrong_rollbacks > 0: p = min(wrong_rollbacks * 0.08, 0.20) breakdown["wrong_rollback_penalty"] = p score -= p score = round(min(max(score, 0.0), 1.0), 4) return self.clamp_score_strict(score), breakdown