"""Deterministic service mesh simulation for cascading incident scenarios.""" from __future__ import annotations from typing import Literal from pydantic import BaseModel, Field from models.incident import RedHerring HealthState = Literal["healthy", "degraded", "flickering", "down"] FixType = Literal[ "restart_service", "memory_increase", "config_change", "schema_migration", "data_fix", ] ProbeType = Literal["logs", "resources", "connections", "config"] Priority = Literal["low", "medium", "high"] ServiceName = Literal["auth", "database", "payments", "analytics", "notifications"] SERVICES: dict[ServiceName, dict[str, object]] = { "auth": { "description": "Authentication & token management", "dependencies": [], "failure_modes": ["rate_limiting", "token_expiry", "config_corruption"], }, "database": { "description": "Primary data store", "dependencies": [], "failure_modes": ["oom", "connection_pool_exhaustion", "replication_lag"], }, "payments": { "description": "Transaction processing", "dependencies": ["auth", "database"], "failure_modes": ["gateway_timeout", "validation_errors", "idempotency_failure"], }, "analytics": { "description": "Reporting & metrics", "dependencies": ["database"], "failure_modes": ["batch_job_runaway", "query_timeout", "stale_cache"], }, "notifications": { "description": "Email, SMS, push notifications", "dependencies": ["payments"], "failure_modes": ["queue_overflow", "template_error", "rate_exceeded"], }, } SERVICE_OBSERVABILITY: dict[ServiceName, Literal["high", "medium", "low"]] = { "auth": "high", "database": "high", "payments": "medium", "analytics": "low", "notifications": "low", } FLICKER_PATTERNS: dict[str, list[HealthState]] = { "intermittent_oom": ["healthy", "degraded", "healthy", "healthy", "degraded", "down"], "connection_flap": ["healthy", "degraded", "healthy", "degraded"], "gc_pressure": ["healthy", "healthy", "degraded", "healthy", "healthy", "degraded"], } BLAST_RADIUS: dict[FixType, dict[str, str | int | float | bool]] = { "restart_service": { "damage": "temporary_outage", "duration": 3, "cascade": False, "penalty": -0.08, "description": "Service went offline during unnecessary restart", }, "memory_increase": { "damage": "resource_starvation", "duration": 0, "cascade": True, "penalty": -0.10, "description": "Memory reallocation starved dependent services", }, "config_change": { "damage": "misconfiguration", "duration": 0, "cascade": True, "penalty": -0.12, "description": "Bad configuration cascaded through service mesh", }, "schema_migration": { "damage": "data_corruption", "duration": 0, "cascade": True, "penalty": -0.20, "description": "Schema migration on wrong table corrupted data", }, "data_fix": { "damage": "data_loss", "duration": 0, "cascade": True, "penalty": -0.25, "description": "Direct data manipulation caused data loss", }, } FAILURE_FIX_MAP: dict[str, FixType] = { "rate_limiting": "restart_service", "token_expiry": "config_change", "config_corruption": "config_change", "oom": "memory_increase", "connection_pool_exhaustion": "restart_service", "replication_lag": "schema_migration", "gateway_timeout": "restart_service", "validation_errors": "config_change", "idempotency_failure": "data_fix", "batch_job_runaway": "memory_increase", "query_timeout": "schema_migration", "stale_cache": "restart_service", "queue_overflow": "restart_service", "template_error": "config_change", "rate_exceeded": "memory_increase", } HEALTH_METRICS: dict[HealthState, tuple[float, int]] = { "healthy": (0.01, 50), "degraded": (0.45, 2000), "flickering": (0.25, 900), "down": (1.0, 10000), } class ServiceState(BaseModel): """Mutable runtime state for a single service.""" name: ServiceName health: HealthState = "healthy" root_cause: str | None = None error_rate: float = 0.0 latency_ms: int = 50 affected_by: ServiceName | None = None is_root_cause: bool = False fix_applied: bool = False fix_correct: bool = False flicker_pattern: list[HealthState] | None = None flicker_step_index: int = 0 blast_until_step: int = 0 failure_step: int = 0 class FixResult(BaseModel): """Result of a normal fix attempt.""" success: bool service: ServiceName fix_type: FixType message: str healed_services: list[ServiceName] = Field(default_factory=list) class BlastRadiusResult(BaseModel): """Result of a wrong-fix blast radius event.""" service: ServiceName fix_type: FixType damaged_services: list[ServiceName] = Field(default_factory=list) penalty: float description: str cascade: bool class MonitoringRecord(BaseModel): """Agent-visible monitoring state for one service.""" service: ServiceName health: HealthState error_rate: float latency_ms: int class MonitoringSnapshot(BaseModel): """Agent-visible monitoring snapshot.""" services: list[MonitoringRecord] = Field(default_factory=list) class ProbeResult(BaseModel): """Deterministic diagnostics for a service probe.""" service: ServiceName check_type: ProbeType observability: Literal["high", "medium", "low"] findings: list[str] = Field(default_factory=list) class Alert(BaseModel): """Single alert emitted by the service mesh.""" source: Literal["monitoring", "pagerduty"] service: ServiceName message: str is_actionable: bool priority: Priority class FlickeringBehavior: """Deterministic intermittent health state sequencer.""" def __init__(self, pattern: list[HealthState], seed: int) -> None: self.pattern = list(pattern) self.seed = seed def get_current_health(self, step: int) -> HealthState: """Return deterministic health for the given step.""" return self.pattern[step % len(self.pattern)] class ServiceMesh: """Deterministic simulation of 5 interconnected microservices.""" def __init__(self, seed: int) -> None: self.seed = seed self.step = 0 self.services: dict[ServiceName, ServiceState] = {} self.dependency_graph: dict[ServiceName, list[ServiceName]] = {} self._reverse_graph: dict[ServiceName, list[ServiceName]] = {} self._init_graph() self._init_services() def _init_graph(self) -> None: for name, config in SERVICES.items(): deps = [dep for dep in config["dependencies"] if isinstance(dep, str)] typed_deps = [dep for dep in deps if dep in SERVICES] self.dependency_graph[name] = typed_deps # type: ignore[assignment] self._reverse_graph = {name: [] for name in SERVICES} for node, deps in self.dependency_graph.items(): for dep in deps: self._reverse_graph[dep].append(node) def _init_services(self) -> None: for name in SERVICES: self.services[name] = ServiceState(name=name) def inject_failure(self, service: ServiceName, failure_mode: str) -> None: """Inject a root-cause outage and deterministically cascade it.""" state = self.services[service] state.health = "down" state.root_cause = failure_mode state.is_root_cause = True state.fix_applied = False state.fix_correct = False state.failure_step = self.step state.error_rate, state.latency_ms = HEALTH_METRICS["down"] self._cascade_from(service) def _cascade_from(self, upstream: ServiceName) -> None: for dependent in self._reverse_graph[upstream]: dep_state = self.services[dependent] if dep_state.is_root_cause: continue if dep_state.health == "down": continue dep_state.health = "degraded" dep_state.affected_by = upstream dep_state.fix_applied = False dep_state.fix_correct = False dep_state.failure_step = self.step dep_state.error_rate, dep_state.latency_ms = HEALTH_METRICS["degraded"] self._cascade_from(dependent) def set_flickering(self, service: ServiceName, pattern_name: str) -> None: """Enable deterministic flickering behavior for a service.""" pattern = FLICKER_PATTERNS[pattern_name] state = self.services[service] state.health = "flickering" state.flicker_pattern = list(pattern) state.flicker_step_index = 0 state.failure_step = self.step state.error_rate, state.latency_ms = HEALTH_METRICS["flickering"] def apply_fix(self, service: ServiceName, fix_type: FixType) -> FixResult: """Apply a fix attempt and return deterministic outcome.""" state = self.services[service] state.fix_applied = True expected = FAILURE_FIX_MAP.get(state.root_cause or "") if state.is_root_cause and expected == fix_type: state.fix_correct = True healed = self._heal_from_root(service) return FixResult( success=True, service=service, fix_type=fix_type, message="Root cause fixed. Recovery started.", healed_services=healed, ) state.fix_correct = False if state.health in ("down", "degraded", "flickering"): state.health = "healthy" state.error_rate, state.latency_ms = HEALTH_METRICS["healthy"] return FixResult( success=False, service=service, fix_type=fix_type, message="Symptom fix was temporary. Service may re-degrade.", healed_services=[], ) def _heal_from_root(self, service: ServiceName) -> list[ServiceName]: healed: list[ServiceName] = [service] root_state = self.services[service] root_state.health = "healthy" root_state.error_rate, root_state.latency_ms = HEALTH_METRICS["healthy"] root_state.root_cause = None root_state.affected_by = None root_state.is_root_cause = False for dependent in self._reverse_graph[service]: healed.extend(self._heal_chain(dependent)) return healed def _heal_chain(self, service: ServiceName) -> list[ServiceName]: state = self.services[service] healed = [service] if not state.is_root_cause: state.health = "healthy" state.error_rate, state.latency_ms = HEALTH_METRICS["healthy"] state.affected_by = None state.fix_applied = False state.fix_correct = False for dependent in self._reverse_graph[service]: healed.extend(self._heal_chain(dependent)) return healed def apply_wrong_fix(self, service: ServiceName, fix_type: FixType) -> BlastRadiusResult: """Apply blast-radius damage for an incorrect fix.""" spec = BLAST_RADIUS[fix_type] damaged: list[ServiceName] = [service] target = self.services[service] target.health = "down" target.blast_until_step = self.step + int(spec["duration"]) target.error_rate, target.latency_ms = HEALTH_METRICS["down"] cascade_enabled = bool(spec["cascade"]) if cascade_enabled: for dependent in self._reverse_graph[service]: dep = self.services[dependent] if dep.health != "down": dep.health = "degraded" dep.affected_by = service dep.error_rate, dep.latency_ms = HEALTH_METRICS["degraded"] damaged.append(dependent) return BlastRadiusResult( service=service, fix_type=fix_type, damaged_services=damaged, penalty=float(spec["penalty"]), description=str(spec["description"]), cascade=cascade_enabled, ) def tick_service_health(self, steps_since_failure: int) -> None: """Progress degradation and flickering in deterministic ticks.""" self.step += 1 for service in self.services.values(): self._tick_blast_recovery(service) self._tick_flickering(service) self._tick_degraded(service, steps_since_failure) def _tick_blast_recovery(self, service: ServiceState) -> None: if service.blast_until_step <= 0: return if self.step >= service.blast_until_step: service.blast_until_step = 0 service.health = "degraded" service.error_rate, service.latency_ms = HEALTH_METRICS["degraded"] def _tick_flickering(self, service: ServiceState) -> None: if service.flicker_pattern is None: return behavior = FlickeringBehavior(service.flicker_pattern, self.seed + service.failure_step) service.health = behavior.get_current_health(self.step) service.flicker_step_index = self.step % len(service.flicker_pattern) service.error_rate, service.latency_ms = HEALTH_METRICS[service.health] def _tick_degraded(self, service: ServiceState, steps_since_failure: int) -> None: if service.health != "degraded": return if service.fix_applied and not service.is_root_cause: if (self.step - service.failure_step) >= 2: service.health = "degraded" service.error_rate, service.latency_ms = HEALTH_METRICS["degraded"] return if steps_since_failure > 0 and steps_since_failure % 5 == 0: service.error_rate = min(1.0, service.error_rate + 0.1) service.latency_ms = min(10000, service.latency_ms + 500) if service.error_rate >= 0.9: service.health = "down" service.error_rate, service.latency_ms = HEALTH_METRICS["down"] self._cascade_from(service.name) def get_monitoring_data(self, service: ServiceName | None = None) -> MonitoringSnapshot: """Return monitoring-visible health, error rate, and latency.""" if service is not None: state = self.services[service] return MonitoringSnapshot(services=[self._record_for(state)]) records = [self._record_for(state) for state in self.services.values()] return MonitoringSnapshot(services=records) @staticmethod def _record_for(state: ServiceState) -> MonitoringRecord: return MonitoringRecord( service=state.name, health=state.health, error_rate=round(state.error_rate, 3), latency_ms=state.latency_ms, ) def probe_service(self, service: ServiceName, check_type: ProbeType) -> ProbeResult: """Return deterministic probe diagnostics with observability gaps.""" level = SERVICE_OBSERVABILITY[service] state = self.services[service] findings = self._probe_findings(state, check_type, level) return ProbeResult( service=service, check_type=check_type, observability=level, findings=findings, ) def _probe_findings( self, state: ServiceState, check_type: ProbeType, level: Literal["high", "medium", "low"], ) -> list[str]: cause = state.root_cause or "unknown" if level == "low": return [f"{state.name} responding: {'no' if state.health == 'down' else 'yes'}"] if check_type == "connections": deps = self.dependency_graph[state.name] return [f"connections to: {', '.join(deps) if deps else 'none'}"] if level == "medium": return [f"{check_type} check indicates instability around {state.name}"] return [ f"{check_type} check for {state.name}", f"observed failure signature: {cause}", f"health={state.health} error_rate={state.error_rate:.2f}", ] def get_health_summary(self) -> dict[str, str]: """Return a compact health map for all services.""" return {name: state.health for name, state in self.services.items()} def get_dependencies(self, difficulty: str) -> dict[str, list[str]]: """Return known topology by difficulty level.""" if difficulty in ("hard", "nightmare"): return {} return {name: list(deps) for name, deps in self.dependency_graph.items()} def generate_red_herrings(self, incident_id: str) -> list[RedHerring]: """Generate deterministic per-incident red herring symptoms.""" templates = [ RedHerring( service="analytics", symptom="CPU at 95%", actual_explanation="Scheduled batch processing", misleading_because="High CPU often suggests runaway workloads", ), RedHerring( service="auth", symptom="token refresh latency +200ms", actual_explanation="Expected during key rotation window", misleading_because="Latency spike resembles auth incident", ), RedHerring( service="notifications", symptom="queue depth increased", actual_explanation="Marketing campaign burst", misleading_because="Queue growth resembles delivery outage", ), ] index_base = (self.seed + sum(ord(ch) for ch in incident_id)) % len(templates) return [templates[index_base], templates[(index_base + 1) % len(templates)]] def generate_alerts(self, step: int) -> list[Alert]: """Generate deterministic actionable and noise alerts.""" alerts: list[Alert] = [] alerts.extend(self._real_alerts()) alerts.extend(self._cascade_noise()) alerts.extend(self._flicker_noise(step)) alerts.extend(self._scheduled_noise(step)) return alerts def _real_alerts(self) -> list[Alert]: alerts: list[Alert] = [] for service in self.services.values(): if service.health in ("degraded", "down"): alerts.append( Alert( source="monitoring", service=service.name, message=f"{service.name} error rate > {service.error_rate * 100:.0f}%", is_actionable=True, priority="high" if service.health == "down" else "medium", ) ) return alerts def _cascade_noise(self) -> list[Alert]: alerts: list[Alert] = [] for service in self.services.values(): if service.affected_by is None: continue alerts.append( Alert( source="monitoring", service=service.name, message=f"{service.name} latency +{service.latency_ms - 50}ms", is_actionable=False, priority="medium", ) ) return alerts def _flicker_noise(self, step: int) -> list[Alert]: alerts: list[Alert] = [] for service in self.services.values(): if service.flicker_pattern is None: continue if step % len(service.flicker_pattern) == 0: alerts.append( Alert( source="monitoring", service=service.name, message=f"{service.name} status changed: healthy ↔ degraded", is_actionable=False, priority="low", ) ) return alerts def _scheduled_noise(self, step: int) -> list[Alert]: alerts: list[Alert] = [] if (self.seed + step) % 3 == 0: alerts.append( Alert( source="monitoring", service="analytics", message="analytics batch job CPU 88% (scheduled)", is_actionable=False, priority="low", ) ) if (self.seed + step) % 4 == 0: alerts.append( Alert( source="pagerduty", service="notifications", message="Auto-recovery attempted on notifications service", is_actionable=False, priority="medium", ) ) return alerts