Spaces:
Sleeping
Sleeping
File size: 9,488 Bytes
309bdb9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 | from .base_scenario import BaseScenario
from server.simulation.topology import ENTERPRISE_TOPOLOGY
from server.models.observation import ServiceMetrics, DependencyEdge, Alert
from typing import Dict, Any, List
class Task3MultiRootCauseScenario(BaseScenario):
"""
TWO simultaneous independent root causes:
(A) redis-cache Redis cluster in eviction storm: maxmemory hit, 88% of keys evicted
- Affects: rate-limiter (throttling breaking), inventory-service (cache misses),
fraud-detection (feature store unavailable)
(B) inventory-service bad deployment 15 minutes ago with memory leak in Go code
- Affects: order-service (cannot check stock), inventory-db (hammered by retries)
Complexity: 12 active alerts, most point to downstream services
Evidence contradictory: order-service errors could be from redis OR inventory
Correct ORDER matters: fix redis first, then rollback inventory
"""
def get_root_causes(self) -> List[str]:
return ["redis-cache", "inventory-service"]
def get_initial_state(self) -> Dict[str, Any]:
service_metrics = []
for service_name, config in ENTERPRISE_TOPOLOGY.items():
service_metrics.append(ServiceMetrics(
service=service_name,
status="healthy",
error_rate=0.0,
latency_p99_ms=config.get("slo_latency_ms", 100),
cpu_percent=20.0,
memory_percent=35.0,
rps=200.0,
replica_count=config.get("replicas", 1)
))
# Dependency graph
dependency_graph = []
for service_name, config in ENTERPRISE_TOPOLOGY.items():
for dep in config["depends_on"]:
dependency_graph.append(DependencyEdge(
source=service_name,
target=dep,
protocol="HTTP",
is_healthy=True,
error_rate_on_edge=0.0
))
state = {
"service_metrics": service_metrics,
"dependency_graph": dependency_graph,
"blast_radius": 0.0,
"error_budget_burn_rate": 0.0,
"active_alerts": [],
"logs": [],
"actions_taken": 0,
"safety_violations": 0,
"episode_id": "task3_multi_root_test",
"step": 0,
"elapsed_seconds": 0.0,
"max_steps": 30,
"deploy_history_checked": {},
"cache_flushed_services": set(),
}
self.apply_scenario_effects(state)
return state
def apply_scenario_effects(self, state: Dict[str, Any]) -> None:
# Root Cause A: Redis cache in eviction storm
for sm in state["service_metrics"]:
if sm.service == "redis-cache":
sm.status = "degraded"
sm.cache_hit_rate = 0.12 # 88% eviction rate
sm.memory_percent = 99.5
break
# Root Cause B: Inventory service memory leak
for sm in state["service_metrics"]:
if sm.service == "inventory-service":
sm.status = "degraded"
sm.memory_percent = 91.0
sm.error_rate = 0.38
break
# Downstream effects of Redis degradation
for sm in state["service_metrics"]:
if sm.service == "rate-limiter":
sm.status = "degraded"
sm.error_rate = 0.15 # Throttling broken
break
for sm in state["service_metrics"]:
if sm.service == "fraud-detection":
sm.status = "critical"
sm.error_rate = 0.42
sm.queue_consumer_lag = 62000
break
# Downstream effects of Inventory degradation
for sm in state["service_metrics"]:
if sm.service == "order-service":
sm.status = "critical"
sm.error_rate = 0.65
break
for sm in state["service_metrics"]:
if sm.service == "inventory-db":
sm.status = "degraded"
sm.db_conn_pool_utilization = 0.85
break
# API gateway experiences latency from rate-limiter
for sm in state["service_metrics"]:
if sm.service == "api-gateway":
sm.latency_p99_ms = 850
break
# Red herring: HAProxy CPU spike (normal high-traffic pattern)
for sm in state["service_metrics"]:
if sm.service == "haproxy-lb":
sm.cpu_percent = 75.0
break
state["blast_radius"] = 0.75
state["active_alerts"] = [
Alert(
alert_id="redis-memory-1",
severity="SEV2",
service="redis-cache",
title="Redis eviction storm",
description="Eviction rate 88%, cache hit rate 12%",
triggered_at=0.0,
is_acknowledged=False,
firing_for_seconds=900.0
),
Alert(
alert_id="inv-service-memory-1",
severity="SEV2",
service="inventory-service",
title="Inventory service memory leak",
description="Memory 91%, growing over time",
triggered_at=0.0,
is_acknowledged=False,
firing_for_seconds=900.0
),
Alert(
alert_id="order-critical-1",
severity="SEV1",
service="order-service",
title="Order service critical - 65% error rate",
description="Checkout unavailable",
triggered_at=0.0,
is_acknowledged=False,
firing_for_seconds=600.0
),
Alert(
alert_id="fraud-critical-1",
severity="SEV1",
service="fraud-detection",
title="Fraud detection degraded",
description="42% error rate, queue lag 62k",
triggered_at=0.0,
is_acknowledged=False,
firing_for_seconds=600.0
),
Alert(
alert_id="rate-limit-err-1",
severity="SEV2",
service="rate-limiter",
title="Rate limiter errors",
description="Throttling broken, 15% failures",
triggered_at=0.0,
is_acknowledged=False,
firing_for_seconds=500.0
),
Alert(
alert_id="api-gw-latency-1",
severity="SEV3",
service="api-gateway",
title="API gateway latency spike",
description="p99 850ms (SLO 100ms)",
triggered_at=0.0,
is_acknowledged=False,
firing_for_seconds=400.0
),
Alert(
alert_id="inv-db-conn-1",
severity="SEV3",
service="inventory-db",
title="Inventory DB connection pool",
description="85% utilization",
triggered_at=0.0,
is_acknowledged=False,
firing_for_seconds=350.0
),
Alert(
alert_id="lb-cpu-1",
severity="SEV3",
service="haproxy-lb",
title="HAProxy high CPU",
description="CPU 75% (normal for high traffic)",
triggered_at=0.0,
is_acknowledged=False,
firing_for_seconds=300.0
),
Alert(
alert_id="order-error-2",
severity="SEV2",
service="order-service",
title="Order service degraded",
description="Cannot check inventory",
triggered_at=0.0,
is_acknowledged=False,
firing_for_seconds=550.0
),
Alert(
alert_id="fraud-lag-2",
severity="SEV3",
service="fraud-detection",
title="Fraud consumer lag",
description="Processing events delayed",
triggered_at=0.0,
is_acknowledged=False,
firing_for_seconds=600.0
),
Alert(
alert_id="cache-hit-1",
severity="SEV3",
service="inventory-service",
title="Cache effectiveness low",
description="Hit rate degraded",
triggered_at=0.0,
is_acknowledged=False,
firing_for_seconds=700.0
),
Alert(
alert_id="payment-margin-1",
severity="SEV4",
service="payment-service",
title="Payment service margin warning",
description="Slightly elevated latency",
triggered_at=0.0,
is_acknowledged=False,
firing_for_seconds=450.0
),
]
state["service_deployments"] = {
"inventory-service": {
"current_sha": "b9d3e7f2",
"deployed_at_minutes_ago": 15,
"stable_sha": "c1a2b3e4"
}
} |