Spaces:
Sleeping
Sleeping
| """ | |
| Task: Single Service OOM Crash | |
| To add a new task, copy this file, modify the SCENARIO definition, and place it in tasks/. | |
| The task loader will auto-discover it. | |
| """ | |
| from env.scenario import IncidentScenario, RequiredFix, ServiceConfig | |
| from models import RootCauseCategory, ServiceStatus | |
| SCENARIO = IncidentScenario( | |
| task_id="easy", | |
| name="Single Service OOM Crash", | |
| difficulty="easy", | |
| max_steps=15, | |
| incident_summary=( | |
| "PagerDuty alert fired at 02:15 UTC. auth-service is down with elevated error rates " | |
| "and pod restarts. api-gateway reporting 503s on login endpoints. Other services appear " | |
| "unaffected. On-call engineer needed to investigate and restore service." | |
| ), | |
| services={ | |
| "api-gateway": ServiceConfig( | |
| status=ServiceStatus.DEGRADED, deps=["auth-service", "user-service", "payment-service"], | |
| version="v1.12.0", replicas=3, | |
| ), | |
| "auth-service": ServiceConfig( | |
| status=ServiceStatus.DOWN, deps=["cache-redis"], | |
| version="v2.14.0", replicas=2, is_root_cause=True, fault_type="oom_crash", | |
| ), | |
| "user-service": ServiceConfig( | |
| status=ServiceStatus.HEALTHY, deps=["db-postgres"], | |
| version="v4.2.1", replicas=2, | |
| ), | |
| "payment-service": ServiceConfig( | |
| status=ServiceStatus.HEALTHY, deps=["db-postgres"], | |
| version="v3.8.1", replicas=2, | |
| ), | |
| "db-postgres": ServiceConfig( | |
| status=ServiceStatus.HEALTHY, deps=[], | |
| version="v15.4", replicas=1, | |
| ), | |
| "cache-redis": ServiceConfig( | |
| status=ServiceStatus.HEALTHY, deps=[], | |
| version="v7.2.4", replicas=1, | |
| ), | |
| "notification-service": ServiceConfig( | |
| status=ServiceStatus.HEALTHY, deps=["auth-service"], | |
| version="v1.5.0", replicas=1, | |
| ), | |
| }, | |
| initial_alerts=[ | |
| "[ALERT SEV-2] auth-service: error rate >50%, pod restarts detected (3 restarts in 5m)", | |
| "[ALERT SEV-3] api-gateway: elevated 503 responses on /api/v2/login and /api/v2/verify", | |
| ], | |
| logs={ | |
| "auth-service": [ | |
| "2026-04-06T02:10:01Z INFO [auth-service] Request processed: POST /auth/token uid=user_8832 latency=45ms", | |
| "2026-04-06T02:10:02Z INFO [auth-service] Request processed: POST /auth/token uid=user_1204 latency=52ms", | |
| "2026-04-06T02:10:03Z INFO [auth-service] Cache hit for session sid=a8f32c, returning cached token", | |
| "2026-04-06T02:10:04Z INFO [auth-service] Request processed: POST /auth/verify uid=user_6650 latency=41ms", | |
| "2026-04-06T02:10:05Z INFO [auth-service] Request processed: POST /auth/verify uid=user_3310 latency=38ms", | |
| "2026-04-06T02:10:06Z INFO [auth-service] Cache hit for session sid=b2e19f, returning cached token", | |
| "2026-04-06T02:10:07Z INFO [auth-service] Request processed: POST /auth/token uid=user_7712 latency=48ms", | |
| "2026-04-06T02:10:08Z DEBUG [auth-service] GC pause: 120ms (heap=1.8GB/2.0GB)", | |
| "2026-04-06T02:10:09Z INFO [auth-service] Request processed: POST /auth/token uid=user_2290 latency=155ms", | |
| "2026-04-06T02:10:10Z INFO [auth-service] Request processed: POST /auth/token uid=user_5571 latency=310ms", | |
| "2026-04-06T02:10:11Z WARN [auth-service] Heap usage at 91% (1.82GB/2.0GB), approaching limit", | |
| "2026-04-06T02:10:12Z INFO [auth-service] Request processed: POST /auth/token uid=user_9912 latency=580ms", | |
| "2026-04-06T02:10:13Z INFO [auth-service] Request processed: POST /auth/verify uid=user_4105 latency=620ms", | |
| "2026-04-06T02:10:14Z WARN [auth-service] GC overhead limit exceeded, full GC triggered (heap=1.95GB/2.0GB)", | |
| "2026-04-06T02:10:15Z INFO [auth-service] Full GC completed in 2100ms, freed 50MB", | |
| "2026-04-06T02:10:16Z INFO [auth-service] Request processed: POST /auth/token uid=user_4423 latency=2400ms", | |
| "2026-04-06T02:10:17Z INFO [auth-service] Request processed: POST /auth/verify uid=user_8001 latency=1900ms", | |
| "2026-04-06T02:10:18Z ERROR [auth-service] OutOfMemoryError: unable to allocate 64MB for token cache expansion", | |
| "2026-04-06T02:10:18Z ERROR [auth-service] Worker pid=1842 killed by OOM killer (resident=2.01GB, limit=2.0GB)", | |
| "2026-04-06T02:10:19Z WARN [auth-service] Process supervisor restarting worker (attempt 1/3)", | |
| "2026-04-06T02:10:22Z INFO [auth-service] Worker pid=1901 started, initializing token cache...", | |
| "2026-04-06T02:10:25Z INFO [auth-service] Request processed: POST /auth/token uid=user_7781 latency=65ms", | |
| "2026-04-06T02:10:28Z INFO [auth-service] Request processed: POST /auth/token uid=user_2209 latency=72ms", | |
| "2026-04-06T02:10:30Z INFO [auth-service] Cache hit for session sid=f8a21c, returning cached token", | |
| "2026-04-06T02:10:33Z INFO [auth-service] Request processed: POST /auth/verify uid=user_3390 latency=55ms", | |
| "2026-04-06T02:10:35Z INFO [auth-service] Request processed: POST /auth/token uid=user_1150 latency=68ms", | |
| "2026-04-06T02:10:40Z INFO [auth-service] Request processed: POST /auth/token uid=user_4482 latency=75ms", | |
| "2026-04-06T02:10:45Z DEBUG [auth-service] GC pause: 85ms (heap=1.5GB/2.0GB)", | |
| "2026-04-06T02:11:00Z INFO [auth-service] Request processed: POST /auth/token uid=user_6633 latency=90ms", | |
| "2026-04-06T02:11:30Z INFO [auth-service] Request processed: POST /auth/verify uid=user_9901 latency=110ms", | |
| "2026-04-06T02:12:00Z WARN [auth-service] Heap usage at 82% (1.64GB/2.0GB) — growing again after restart", | |
| "2026-04-06T02:12:30Z INFO [auth-service] Request processed: POST /auth/token uid=user_5510 latency=180ms", | |
| "2026-04-06T02:12:45Z WARN [auth-service] Heap usage at 88% (1.76GB/2.0GB) — growing linearly", | |
| "2026-04-06T02:13:00Z DEBUG [auth-service] GC pause: 350ms (heap=1.85GB/2.0GB)", | |
| "2026-04-06T02:13:05Z INFO [auth-service] Request processed: POST /auth/token uid=user_8820 latency=890ms", | |
| "2026-04-06T02:13:10Z ERROR [auth-service] OutOfMemoryError: unable to allocate 32MB for request buffer", | |
| "2026-04-06T02:13:10Z ERROR [auth-service] Worker pid=1901 killed by OOM killer (resident=1.98GB, limit=2.0GB)", | |
| "2026-04-06T02:13:11Z WARN [auth-service] Process supervisor restarting worker (attempt 2/3)", | |
| "2026-04-06T02:13:14Z INFO [auth-service] Worker pid=1955 started, initializing token cache...", | |
| "2026-04-06T02:13:20Z INFO [auth-service] Request processed: POST /auth/token uid=user_1122 latency=58ms", | |
| "2026-04-06T02:13:45Z WARN [auth-service] Heap usage at 80% (1.60GB/2.0GB)", | |
| "2026-04-06T02:14:15Z WARN [auth-service] Heap usage at 87% (1.74GB/2.0GB)", | |
| "2026-04-06T02:14:45Z WARN [auth-service] GC overhead limit exceeded, full GC triggered", | |
| "2026-04-06T02:15:00Z INFO [auth-service] Full GC completed in 2800ms, freed 30MB — diminishing returns", | |
| "2026-04-06T02:15:20Z ERROR [auth-service] OutOfMemoryError: Java heap space", | |
| "2026-04-06T02:15:33Z ERROR [auth-service] Worker pid=1955 killed by OOM killer (resident=2.03GB, limit=2.0GB)", | |
| "2026-04-06T02:15:34Z ERROR [auth-service] Process supervisor: all 3 restart attempts exhausted", | |
| "2026-04-06T02:15:34Z FATAL [auth-service] Service entering crash loop backoff — no healthy workers remaining", | |
| "2026-04-06T02:15:35Z ERROR [auth-service] Health check failed: connection refused on :8080/healthz", | |
| ], | |
| "api-gateway": [ | |
| "2026-04-06T02:10:01Z INFO [api-gateway] Route: POST /api/v2/login -> auth-service (200, 48ms)", | |
| "2026-04-06T02:10:02Z INFO [api-gateway] Route: GET /api/v2/user/profile -> user-service (200, 32ms)", | |
| "2026-04-06T02:10:03Z INFO [api-gateway] Route: POST /api/v2/pay -> payment-service (200, 95ms)", | |
| "2026-04-06T02:10:05Z INFO [api-gateway] Route: GET /api/v2/user/settings -> user-service (200, 28ms)", | |
| "2026-04-06T02:10:08Z INFO [api-gateway] Route: POST /api/v2/login -> auth-service (200, 155ms)", | |
| "2026-04-06T02:10:10Z INFO [api-gateway] Route: POST /api/v2/login -> auth-service (200, 320ms)", | |
| "2026-04-06T02:10:15Z WARN [api-gateway] Route: POST /api/v2/login -> auth-service (200, 2500ms) — slow", | |
| "2026-04-06T02:10:18Z ERROR [api-gateway] Route: POST /api/v2/login -> auth-service (503, timeout after 5000ms)", | |
| "2026-04-06T02:10:20Z INFO [api-gateway] Route: GET /api/v2/user/profile -> user-service (200, 30ms)", | |
| "2026-04-06T02:10:22Z INFO [api-gateway] Route: POST /api/v2/login -> auth-service (200, 68ms)", | |
| "2026-04-06T02:10:25Z INFO [api-gateway] Route: POST /api/v2/pay -> payment-service (200, 88ms)", | |
| "2026-04-06T02:13:10Z ERROR [api-gateway] Route: POST /api/v2/login -> auth-service (503, timeout after 5000ms)", | |
| "2026-04-06T02:13:12Z WARN [api-gateway] Retrying auth-service request (attempt 2/3)", | |
| "2026-04-06T02:13:17Z ERROR [api-gateway] Route: POST /api/v2/login -> auth-service (503, timeout after 5000ms)", | |
| "2026-04-06T02:15:35Z ERROR [api-gateway] Route: POST /api/v2/login -> auth-service (503, connection refused)", | |
| "2026-04-06T02:15:36Z WARN [api-gateway] Circuit breaker OPEN for auth-service (failures=10, threshold=5)", | |
| "2026-04-06T02:15:37Z ERROR [api-gateway] Route: POST /api/v2/login -> auth-service (503, circuit breaker open)", | |
| "2026-04-06T02:15:37Z ERROR [api-gateway] Route: POST /api/v2/verify -> auth-service (503, circuit breaker open)", | |
| "2026-04-06T02:15:38Z INFO [api-gateway] Route: GET /api/v2/user/profile -> user-service (200, 28ms)", | |
| "2026-04-06T02:15:40Z INFO [api-gateway] Route: POST /api/v2/pay -> payment-service (200, 95ms)", | |
| "2026-04-06T02:15:42Z INFO [api-gateway] Route: GET /api/v2/user/settings -> user-service (200, 25ms)", | |
| ], | |
| "user-service": [ | |
| "2026-04-06T02:10:01Z INFO [user-service] GET /users/profile uid=user_4421 -> 200 (32ms)", | |
| "2026-04-06T02:10:05Z INFO [user-service] GET /users/settings uid=user_8832 -> 200 (28ms)", | |
| "2026-04-06T02:10:10Z INFO [user-service] PUT /users/profile uid=user_3310 -> 200 (85ms)", | |
| "2026-04-06T02:10:15Z INFO [user-service] GET /users/profile uid=user_1101 -> 200 (30ms)", | |
| "2026-04-06T02:10:20Z INFO [user-service] GET /users/profile uid=user_5571 -> 200 (27ms)", | |
| "2026-04-06T02:15:00Z INFO [user-service] GET /users/profile uid=user_7712 -> 200 (31ms)", | |
| "2026-04-06T02:15:30Z INFO [user-service] PUT /users/settings uid=user_2209 -> 200 (78ms)", | |
| "2026-04-06T02:15:35Z INFO [user-service] GET /users/profile uid=user_9901 -> 200 (26ms)", | |
| ], | |
| "payment-service": [ | |
| "2026-04-06T02:10:01Z INFO [payment-service] Processing payment txn=pay_8832 amount=$45.00 -> db-postgres", | |
| "2026-04-06T02:10:02Z INFO [payment-service] Payment completed txn=pay_8832 latency=85ms", | |
| "2026-04-06T02:10:10Z INFO [payment-service] Processing payment txn=pay_1120 amount=$12.99 -> db-postgres", | |
| "2026-04-06T02:10:10Z INFO [payment-service] Payment completed txn=pay_1120 latency=92ms", | |
| "2026-04-06T02:15:00Z INFO [payment-service] Processing payment txn=pay_4455 amount=$78.50 -> db-postgres", | |
| "2026-04-06T02:15:01Z INFO [payment-service] Payment completed txn=pay_4455 latency=88ms", | |
| "2026-04-06T02:15:30Z INFO [payment-service] Health check /healthz -> 200 OK", | |
| ], | |
| "db-postgres": [ | |
| "2026-04-06T02:00:00Z INFO [db-postgres] Checkpoint starting: time-based", | |
| "2026-04-06T02:00:02Z INFO [db-postgres] Checkpoint complete: wrote 842 buffers (5.7%)", | |
| "2026-04-06T02:10:00Z INFO [db-postgres] Active connections: 35/100", | |
| "2026-04-06T02:10:01Z INFO [db-postgres] Autovacuum: processing table users (dead tuples: 120)", | |
| "2026-04-06T02:15:00Z INFO [db-postgres] Active connections: 34/100", | |
| "2026-04-06T02:15:01Z INFO [db-postgres] Checkpoint starting: time-based", | |
| "2026-04-06T02:15:03Z INFO [db-postgres] Checkpoint complete: wrote 910 buffers (6.2%)", | |
| ], | |
| "cache-redis": [ | |
| "2026-04-06T02:10:00Z INFO [cache-redis] Memory usage: 1.2GB/4.0GB (30%)", | |
| "2026-04-06T02:10:01Z INFO [cache-redis] Cache hit ratio: 92% (within normal range 85-95%)", | |
| "2026-04-06T02:10:05Z INFO [cache-redis] Connected clients: 45", | |
| "2026-04-06T02:15:00Z INFO [cache-redis] Memory usage: 1.2GB/4.0GB (30%)", | |
| "2026-04-06T02:15:01Z INFO [cache-redis] Cache hit ratio: 91%", | |
| "2026-04-06T02:15:05Z INFO [cache-redis] Key evictions: 0 in last 5m", | |
| ], | |
| "notification-service": [ | |
| "2026-04-06T02:10:00Z INFO [notification-service] Email batch #4420 sent successfully (12 emails)", | |
| "2026-04-06T02:10:05Z INFO [notification-service] Auth token validated for batch #4421 (45ms)", | |
| "2026-04-06T02:15:00Z INFO [notification-service] Email batch #4425 sent successfully (8 emails)", | |
| "2026-04-06T02:15:30Z INFO [notification-service] Health check /healthz -> 200 OK", | |
| ], | |
| }, | |
| metrics={ | |
| "auth-service": [ | |
| {"timestamp": "2026-04-06T02:00:00Z", "cpu_pct": 25, "mem_pct": 60, "heap_gb": 1.2, "latency_p50": 45, "latency_p99": 120, "error_rate": 0.001, "restarts": 0, "connections": 150}, | |
| {"timestamp": "2026-04-06T02:05:00Z", "cpu_pct": 30, "mem_pct": 72, "heap_gb": 1.44, "latency_p50": 52, "latency_p99": 180, "error_rate": 0.002, "restarts": 0, "connections": 155}, | |
| {"timestamp": "2026-04-06T02:10:00Z", "cpu_pct": 45, "mem_pct": 91, "heap_gb": 1.82, "latency_p50": 310, "latency_p99": 2400, "error_rate": 0.15, "restarts": 1, "connections": 148}, | |
| {"timestamp": "2026-04-06T02:11:00Z", "cpu_pct": 35, "mem_pct": 65, "heap_gb": 1.30, "latency_p50": 65, "latency_p99": 200, "error_rate": 0.02, "restarts": 1, "connections": 140}, | |
| {"timestamp": "2026-04-06T02:13:00Z", "cpu_pct": 48, "mem_pct": 94, "heap_gb": 1.88, "latency_p50": 450, "latency_p99": 3100, "error_rate": 0.20, "restarts": 2, "connections": 130}, | |
| {"timestamp": "2026-04-06T02:15:00Z", "cpu_pct": 0, "mem_pct": 0, "heap_gb": 0, "latency_p50": 0, "latency_p99": 0, "error_rate": 1.0, "restarts": 3, "connections": 0}, | |
| ], | |
| "api-gateway": [ | |
| {"timestamp": "2026-04-06T02:00:00Z", "cpu_pct": 20, "mem_pct": 45, "latency_p50": 32, "latency_p99": 85, "error_rate": 0.001, "5xx_rate": 0.001}, | |
| {"timestamp": "2026-04-06T02:10:00Z", "cpu_pct": 22, "mem_pct": 46, "latency_p50": 35, "latency_p99": 95, "error_rate": 0.005, "5xx_rate": 0.003}, | |
| {"timestamp": "2026-04-06T02:15:00Z", "cpu_pct": 25, "mem_pct": 48, "latency_p50": 40, "latency_p99": 5200, "error_rate": 0.42, "5xx_rate": 0.40}, | |
| ], | |
| "user-service": [ | |
| {"timestamp": "2026-04-06T02:00:00Z", "cpu_pct": 15, "mem_pct": 35, "latency_p50": 28, "latency_p99": 75, "error_rate": 0.001}, | |
| {"timestamp": "2026-04-06T02:15:00Z", "cpu_pct": 15, "mem_pct": 35, "latency_p50": 29, "latency_p99": 78, "error_rate": 0.001}, | |
| ], | |
| "payment-service": [ | |
| {"timestamp": "2026-04-06T02:00:00Z", "cpu_pct": 18, "mem_pct": 40, "latency_p50": 85, "latency_p99": 150, "error_rate": 0.001}, | |
| {"timestamp": "2026-04-06T02:15:00Z", "cpu_pct": 18, "mem_pct": 40, "latency_p50": 88, "latency_p99": 155, "error_rate": 0.001}, | |
| ], | |
| "db-postgres": [ | |
| {"timestamp": "2026-04-06T02:00:00Z", "cpu_pct": 30, "mem_pct": 55, "connections": 35, "active_locks": 2, "deadlocks": 0, "write_iops": 1200, "read_iops": 3500}, | |
| {"timestamp": "2026-04-06T02:15:00Z", "cpu_pct": 32, "mem_pct": 55, "connections": 34, "active_locks": 2, "deadlocks": 0, "write_iops": 1150, "read_iops": 3400}, | |
| ], | |
| "cache-redis": [ | |
| {"timestamp": "2026-04-06T02:00:00Z", "mem_gb": 1.2, "mem_pct": 30, "hit_ratio": 0.92, "evictions_per_s": 0, "connections": 45}, | |
| {"timestamp": "2026-04-06T02:15:00Z", "mem_gb": 1.2, "mem_pct": 30, "hit_ratio": 0.91, "evictions_per_s": 0, "connections": 45}, | |
| ], | |
| }, | |
| traces={ | |
| "auth-service": [ | |
| "No recent traces available — service is down. Last successful trace:", | |
| "Trace: POST /auth/token (uid=user_4423, total=2400ms) — BEFORE CRASH", | |
| " ├─ auth-service.checkSessionCache() 5ms (cache-redis HIT)", | |
| " ├─ auth-service.generateToken() 45ms", | |
| " ├─ auth-service.GC_FULL_PAUSE 2100ms ← GC dominated total time", | |
| " └─ auth-service.writeResponse() 250ms", | |
| ], | |
| "api-gateway": [ | |
| "Trace: POST /api/v2/login (total=5005ms) — TIMEOUT", | |
| " ├─ api-gateway.parseRequest() 2ms", | |
| " ├─ api-gateway.routeToAuthService() 5000ms (TIMEOUT — auth-service unreachable)", | |
| " └─ api-gateway.returnError() 3ms (503 Service Unavailable)", | |
| ], | |
| }, | |
| deploy_history={ | |
| "auth-service": [ | |
| "v2.14.0 deployed 2026-04-01T10:00:00Z status=stable (running 5 days, no issues)", | |
| "v2.13.2 deployed 2026-03-25T14:00:00Z status=superseded", | |
| ], | |
| "api-gateway": [ | |
| "v1.12.0 deployed 2026-03-28T09:00:00Z status=stable (running 9 days)", | |
| ], | |
| "user-service": [ | |
| "v4.2.1 deployed 2026-04-05T16:00:00Z status=stable (running 10 hours)", | |
| "v4.2.0 deployed 2026-04-01T11:00:00Z status=superseded", | |
| ], | |
| "payment-service": [ | |
| "v3.8.1 deployed 2026-04-03T14:00:00Z status=stable (running 3 days)", | |
| ], | |
| }, | |
| runbooks={ | |
| "auth-service": ( | |
| "## auth-service Runbook\n" | |
| "- OOM crashes: Check heap usage trends in metrics. If memory grows linearly after\n" | |
| " restart, likely a memory leak in the token cache. Short-term fix: restart to clear\n" | |
| " cached state. Long-term: file ticket for cache eviction policy fix.\n" | |
| "- High latency: Check cache-redis connectivity. Auth-service falls back to DB lookups\n" | |
| " if cache is down, which increases latency 10x.\n" | |
| "- Connection refused: Service may be in crash loop. Check restart count and supervisor logs.\n" | |
| "- Token validation failures: Check if JWT signing key was recently rotated." | |
| ), | |
| "api-gateway": ( | |
| "## api-gateway Runbook\n" | |
| "- 503 errors: Check downstream service health. Gateway proxies to auth-service,\n" | |
| " user-service, and payment-service. Identify which downstream is failing.\n" | |
| "- Circuit breaker open: Downstream service has exceeded failure threshold.\n" | |
| " Fix the downstream service; circuit breaker will auto-close after 30s of healthy responses.\n" | |
| "- High latency: Usually caused by slow downstream. Check traces to identify bottleneck." | |
| ), | |
| }, | |
| configs={ | |
| "auth-service": { | |
| "current": "JVM_HEAP_MAX=2g\nTOKEN_CACHE_SIZE=500000\nSESSION_TTL=3600\nREDIS_POOL_SIZE=20", | |
| "previous": "JVM_HEAP_MAX=2g\nTOKEN_CACHE_SIZE=500000\nSESSION_TTL=3600\nREDIS_POOL_SIZE=20", | |
| "diff": "No changes — config has not been modified recently.", | |
| }, | |
| }, | |
| dependencies={ | |
| "api-gateway": ["auth-service", "user-service", "payment-service"], | |
| "auth-service": ["cache-redis"], | |
| "user-service": ["db-postgres"], | |
| "payment-service": ["db-postgres"], | |
| "db-postgres": [], | |
| "cache-redis": [], | |
| "notification-service": ["auth-service"], | |
| }, | |
| root_cause_services=["auth-service"], | |
| root_cause_categories=[RootCauseCategory.OOM_CRASH], | |
| required_fixes=[ | |
| RequiredFix(action="restart_service", service="auth-service"), | |
| ], | |
| diagnosis_keywords=["auth-service", "oom", "out of memory", "memory", "crash", "restart"], | |
| weights={ | |
| "correct_service": 0.30, | |
| "correct_category": 0.20, | |
| "correct_fix": 0.30, | |
| "secondary_fix": 0.00, | |
| "diagnosis_text": 0.10, | |
| "investigation": 0.10, | |
| "wrong_penalty": 0.03, | |
| }, | |
| ) | |