Add core simulation engine, environment, grader, and app wiring

- server/traces.py: distributed trace generation (Jaeger/Zipkin-style spans)
- server/simulator.py: discrete-event simulation engine with tick-based
failure evolution, propagation, action processing, SLO scoring, dense
reward computation, and pending remediation effects
- server/scenarios.py: procedural scenario generation with 3 task
definitions (easy/medium/hard) and intelligent failure placement
- server/environment.py: SevZeroEnvironment(Environment) bridging
OpenEnv SDK contract with the simulator
- server/grader.py: deterministic grading (0.0-1.0) with SLO recovery,
action efficiency, and time efficiency components
- server/app.py: FastAPI app via create_app() + custom /tasks, /grader routes
- .gitignore: added __pycache__ exclusion

.gitignore

@@ -2,4 +2,12 @@
 Docs/
 
 # OpenEnv preparatory course (dev reference only, not part of submission)
-openenv-course/
+openenv-course/
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+
+# Environment
+.env

server/app.py

@@ -0,0 +1,94 @@
+"""
+server/app.py — FastAPI application wiring.
+
+Uses OpenEnv SDK's create_app() for core endpoints (/reset, /step, /state, /ws, /health),
+then adds custom routes for /tasks, /grader, and /baseline.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from fastapi import FastAPI
+from openenv.core.env_server import create_app
+from pydantic import BaseModel
+
+from models import SevZeroAction, SevZeroObservation
+from server.environment import SevZeroEnvironment
+from server.grader import grade_episode
+from server.scenarios import TASK_DEFINITIONS
+
+
+# Create the OpenEnv app (wires /reset, /step, /state, /ws, /health, /schema, /metadata)
+app = create_app(
+    SevZeroEnvironment,
+    SevZeroAction,
+    SevZeroObservation,
+    env_name="sevzero",
+)
+
+
+# ---------------------------------------------------------------------------
+# Custom routes
+# ---------------------------------------------------------------------------
+
+
+@app.get("/tasks")
+async def list_tasks() -> List[Dict[str, Any]]:
+    """Return the 3 task definitions (easy, medium, hard)."""
+    return [
+        {
+            "task_id": t["task_id"],
+            "name": t["name"],
+            "difficulty": t["difficulty"],
+            "description": t["description"],
+            "max_steps": t["max_steps"],
+        }
+        for t in TASK_DEFINITIONS
+    ]
+
+
+class GraderRequest(BaseModel):
+    final_slo_score: float
+    steps_taken: int
+    max_steps: int
+    actions_taken: List[Dict[str, Any]]
+    terminated: bool
+    termination_reason: Optional[str] = None
+
+
+@app.post("/grader")
+async def grade(request: GraderRequest) -> Dict[str, Any]:
+    """
+    Deterministic grading endpoint.
+    Accepts episode results and returns a score 0.0–1.0 with breakdown.
+    """
+    result = grade_episode(
+        final_slo_score=request.final_slo_score,
+        steps_taken=request.steps_taken,
+        max_steps=request.max_steps,
+        actions_taken=request.actions_taken,
+        terminated=request.terminated,
+        termination_reason=request.termination_reason,
+    )
+    return {
+        "score": result.score,
+        "slo_recovery": result.slo_recovery,
+        "action_efficiency": result.action_efficiency,
+        "time_efficiency": result.time_efficiency,
+        "details": result.details,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+
+def main() -> None:
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=7860)
+
+
+if __name__ == "__main__":
+    main()

server/environment.py

@@ -0,0 +1,121 @@
+"""
+server/environment.py — SevZeroEnvironment: OpenEnv Environment subclass.
+
+Bridges the OpenEnv SDK contract (reset/step/state) with the Simulator engine.
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any, Optional
+
+from openenv.core.env_server import Environment
+from openenv.core.env_server.types import EnvironmentMetadata
+
+from models import SevZeroAction, SevZeroObservation, SevZeroState
+from server.scenarios import generate_scenario
+from server.simulator import Simulator
+
+
+class SevZeroEnvironment(Environment[SevZeroAction, SevZeroObservation, SevZeroState]):
+    """
+    SRE Incident Response Environment.
+
+    The agent observes service metrics, alerts, and logs, then issues
+    remediation commands to restore SLO compliance across a microservice cluster.
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._sim = Simulator()
+        self._episode_id: Optional[str] = None
+        self._task_id: str = "easy"
+        self._seed: Optional[int] = None
+        self._step_count: int = 0
+
+    def get_metadata(self) -> EnvironmentMetadata:
+        return EnvironmentMetadata(
+            name="sevzero",
+            description=(
+                "SRE Incident Response Environment — an autonomous on-call SRE "
+                "managing a microservice cluster undergoing cascading failures"
+            ),
+            version="1.0.0",
+        )
+
+    def reset(
+        self,
+        seed: Optional[int] = None,
+        episode_id: Optional[str] = None,
+        **kwargs: Any,
+    ) -> SevZeroObservation:
+        self._episode_id = episode_id or str(uuid.uuid4())
+        self._task_id = kwargs.get("task_id", "easy")
+        self._seed = seed if seed is not None else 42
+        self._step_count = 0
+
+        # Generate scenario and reset simulator
+        scenario = generate_scenario(self._seed, self._task_id)
+        self._sim.reset(
+            seed=self._seed,
+            difficulty=scenario.difficulty,
+            failure_specs=scenario.failure_specs,
+        )
+
+        return self._build_observation(reward=None, done=False)
+
+    def step(
+        self,
+        action: SevZeroAction,
+        timeout_s: Optional[float] = None,
+        **kwargs: Any,
+    ) -> SevZeroObservation:
+        self._step_count += 1
+
+        reward = self._sim.step(action.action_type, action.params)
+        done = self._sim.terminated
+
+        return self._build_observation(reward=reward, done=done)
+
+    @property
+    def state(self) -> SevZeroState:
+        return SevZeroState(
+            episode_id=self._episode_id,
+            step_count=self._step_count,
+            task_id=self._task_id,
+            seed=self._seed,
+            global_slo_score=self._sim.get_slo_score(),
+            terminated=self._sim.terminated,
+            termination_reason=self._sim.termination_reason,
+        )
+
+    def _build_observation(
+        self, reward: Optional[float], done: bool,
+    ) -> SevZeroObservation:
+        sim = self._sim
+        return SevZeroObservation(
+            done=done,
+            reward=reward,
+            # Episode context
+            tick=sim.tick,
+            episode_id=self._episode_id,
+            task_id=self._task_id,
+            status=sim.termination_reason or "playing",
+            max_steps=sim.max_steps,
+            # Health summary
+            global_slo_score=round(sim.get_slo_score(), 4),
+            observation_summary=sim.get_observation_summary(),
+            # Per-service state
+            services=sim.get_service_observations(),
+            # Alerts
+            alerts=sim.get_alerts(),
+            # Context
+            recent_deploys=[d for d in sim.deploys if d["ticks_ago"] <= 10],
+            actions_taken=sim.actions_taken[-10:],
+            # Action space
+            legal_actions=sim.get_legal_actions(),
+            # Diagnostics
+            logs=sim.last_logs,
+            metric_history=sim.last_metric_history,
+            traces=sim.last_traces,
+        )

server/grader.py

@@ -0,0 +1,108 @@
+"""
+server/grader.py — Deterministic grading for SevZero episodes.
+
+Score formula:
+    score = slo_recovery * 0.70 + action_efficiency * 0.15 + time_efficiency * 0.15
+
+All inputs are derived from the episode state — fully deterministic.
+Score is continuous 0.0–1.0 with partial credit.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class GradeResult:
+    """Grading result with breakdown."""
+    score: float
+    slo_recovery: float
+    action_efficiency: float
+    time_efficiency: float
+    details: Dict[str, Any]
+
+
+def grade_episode(
+    final_slo_score: float,
+    steps_taken: int,
+    max_steps: int,
+    actions_taken: List[Dict[str, Any]],
+    terminated: bool,
+    termination_reason: Optional[str],
+) -> GradeResult:
+    """
+    Grade a completed episode.
+
+    Args:
+        final_slo_score: fraction of services meeting SLO at episode end (0.0–1.0)
+        steps_taken: number of steps the agent took
+        max_steps: maximum allowed steps for this task
+        actions_taken: list of action records
+        terminated: whether the episode ended
+        termination_reason: "resolved" | "timeout" | "failed" | None
+    """
+    # --- SLO recovery (70%) ---
+    # Direct fraction of services recovered
+    slo_recovery = final_slo_score
+
+    # Bonus for full resolution
+    if termination_reason == "resolved":
+        slo_recovery = 1.0
+
+    # --- Action efficiency (15%) ---
+    # Penalize wasted actions (noops when degraded, failed actions, redundant inspects)
+    total_actions = len(actions_taken)
+    if total_actions == 0:
+        action_efficiency = 0.0
+    else:
+        successful = sum(1 for a in actions_taken if a.get("success", False))
+        remediation_actions = sum(
+            1 for a in actions_taken
+            if a.get("action") not in ("inspect_logs", "inspect_metrics", "inspect_traces", "noop")
+            and a.get("success", False)
+        )
+        inspect_actions = sum(
+            1 for a in actions_taken
+            if a.get("action") in ("inspect_logs", "inspect_metrics", "inspect_traces")
+        )
+
+        # Good ratio: some inspection + targeted remediation
+        success_rate = successful / total_actions
+        # Penalize excessive inspections (>50% of budget is too much looking, not enough doing)
+        inspect_penalty = max(0.0, (inspect_actions / total_actions) - 0.5) if total_actions > 0 else 0.0
+        action_efficiency = max(0.0, success_rate - inspect_penalty)
+
+    # --- Time efficiency (15%) ---
+    # Faster resolution = higher score
+    if max_steps == 0:
+        time_efficiency = 0.0
+    elif termination_reason == "resolved":
+        # Resolved: reward faster resolution
+        time_efficiency = max(0.1, 1.0 - (steps_taken / max_steps))
+    else:
+        # Not resolved: partial credit based on how close we got
+        time_efficiency = final_slo_score * 0.3
+
+    # --- Final score ---
+    score = (
+        slo_recovery * 0.70
+        + action_efficiency * 0.15
+        + time_efficiency * 0.15
+    )
+    score = max(0.0, min(1.0, round(score, 4)))
+
+    return GradeResult(
+        score=score,
+        slo_recovery=round(slo_recovery, 4),
+        action_efficiency=round(action_efficiency, 4),
+        time_efficiency=round(time_efficiency, 4),
+        details={
+            "final_slo_score": round(final_slo_score, 4),
+            "steps_taken": steps_taken,
+            "max_steps": max_steps,
+            "termination_reason": termination_reason,
+            "total_actions": len(actions_taken),
+        },
+    )

server/scenarios.py

@@ -0,0 +1,207 @@
+"""
+server/scenarios.py — Procedural scenario generation from seed + difficulty.
+
+Maps difficulty to graph topology, failure count, and failure placement.
+Same seed + same difficulty = identical scenario every time.
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from server.failures import (
+    FailureSpec,
+    FailureType,
+    make_failure_spec,
+    select_failure_type,
+    select_multi_root_failures,
+)
+from server.graph import ServiceGraph, generate_graph
+
+
+@dataclass
+class ScenarioConfig:
+    """Complete scenario definition for one episode."""
+    difficulty: str
+    seed: int
+    graph: ServiceGraph
+    failure_specs: List[FailureSpec]
+    max_steps: int
+    description: str
+
+
+# ---------------------------------------------------------------------------
+# Task definitions (the 3 required tasks)
+# ---------------------------------------------------------------------------
+
+TASK_DEFINITIONS = [
+    {
+        "task_id": "easy",
+        "name": "Single Service Outage",
+        "difficulty": "easy",
+        "description": (
+            "A single service in a small linear microservice chain is experiencing failures. "
+            "Diagnose the root cause and apply the correct remediation within 10 steps."
+        ),
+        "max_steps": 10,
+        "num_failures": 1,
+    },
+    {
+        "task_id": "medium",
+        "name": "Cascading Failure",
+        "difficulty": "medium",
+        "description": (
+            "A failure in a shared infrastructure service is cascading through a branching "
+            "dependency graph. Trace the root cause upstream from symptomatic services and "
+            "remediate within 20 steps."
+        ),
+        "max_steps": 20,
+        "num_failures": 1,
+    },
+    {
+        "task_id": "hard",
+        "name": "Multi-Root Sev-0 Incident",
+        "difficulty": "hard",
+        "description": (
+            "Multiple simultaneous failures across a multi-region microservice architecture. "
+            "Failures may have conflicting mitigations. Triage, diagnose, and resolve all "
+            "root causes within 50 steps."
+        ),
+        "max_steps": 50,
+        "num_failures": 3,
+    },
+]
+
+
+def get_task_definition(task_id: str) -> dict:
+    """Get a task definition by ID."""
+    for t in TASK_DEFINITIONS:
+        if t["task_id"] == task_id:
+            return t
+    raise ValueError(f"Unknown task_id: {task_id!r}. Must be one of: easy, medium, hard")
+
+
+# ---------------------------------------------------------------------------
+# Failure placement logic
+# ---------------------------------------------------------------------------
+
+
+def _pick_failure_target(
+    graph: ServiceGraph,
+    failure_type: FailureType,
+    rng: random.Random,
+    exclude: set,
+) -> Optional[str]:
+    """Pick an appropriate service to inject this failure type into."""
+    candidates = []
+
+    for node in graph.nodes:
+        if node.id in exclude:
+            continue
+
+        # Cache failures only on cache services
+        if failure_type == FailureType.CACHE_FAILURE:
+            if node.is_cache:
+                candidates.append(node.id)
+            continue
+
+        # DB degradation on infra services (postgres, etc.)
+        if failure_type == FailureType.DB_DEGRADATION:
+            if node.layer == "infra" and "postgres" in node.id:
+                candidates.append(node.id)
+            continue
+
+        # Network errors prefer non-edge services
+        if failure_type == FailureType.NETWORK_ERROR:
+            if node.layer != "edge":
+                candidates.append(node.id)
+            continue
+
+        # Config errors on any non-edge service
+        if failure_type in (FailureType.CONFIG_STARTUP, FailureType.CONFIG_RUNTIME):
+            if node.layer != "edge":
+                candidates.append(node.id)
+            continue
+
+        # Bad deploy on business or identity services
+        if failure_type == FailureType.BAD_DEPLOY:
+            if node.layer in ("business", "identity"):
+                candidates.append(node.id)
+            continue
+
+        # Resource leak on business services
+        if failure_type == FailureType.RESOURCE_LEAK:
+            if node.layer in ("business", "identity"):
+                candidates.append(node.id)
+            continue
+
+        # Crash on any non-edge service
+        if failure_type == FailureType.CRASH:
+            if node.layer != "edge":
+                candidates.append(node.id)
+            continue
+
+        # Cascading latency: prefer hotspot infra or busy business
+        if failure_type == FailureType.CASCADING_LATENCY:
+            if node.is_hotspot or node.layer == "business":
+                candidates.append(node.id)
+            continue
+
+    if not candidates:
+        # Fallback: any non-edge service
+        candidates = [n.id for n in graph.nodes if n.layer != "edge" and n.id not in exclude]
+
+    if not candidates:
+        return None
+
+    return rng.choice(candidates)
+
+
+# ---------------------------------------------------------------------------
+# Scenario generation
+# ---------------------------------------------------------------------------
+
+
+def generate_scenario(seed: int, task_id: str) -> ScenarioConfig:
+    """
+    Generate a complete scenario for the given task and seed.
+    Deterministic: same seed + same task_id = identical scenario.
+    """
+    task = get_task_definition(task_id)
+    rng = random.Random(seed)
+
+    # Generate graph
+    difficulty = task["difficulty"]
+    graph = generate_graph(difficulty, rng)
+
+    # Select and place failures
+    num_failures = task["num_failures"]
+    used_services: set = set()
+    failure_specs: List[FailureSpec] = []
+
+    if num_failures == 1:
+        ft = select_failure_type(rng)
+        target = _pick_failure_target(graph, ft, rng, used_services)
+        if target:
+            spec = make_failure_spec(target, ft, rng)
+            failure_specs.append(spec)
+            used_services.add(target)
+    else:
+        failure_types = select_multi_root_failures(rng, count=num_failures)
+        for ft in failure_types:
+            target = _pick_failure_target(graph, ft, rng, used_services)
+            if target:
+                spec = make_failure_spec(target, ft, rng)
+                failure_specs.append(spec)
+                used_services.add(target)
+
+    return ScenarioConfig(
+        difficulty=difficulty,
+        seed=seed,
+        graph=graph,
+        failure_specs=failure_specs,
+        max_steps=task["max_steps"],
+        description=task["description"],
+    )

server/simulator.py

@@ -0,0 +1,965 @@
+"""
+server/simulator.py — Core discrete-event simulation engine.
+
+Orchestrates the service graph, failure injection, metric evolution,
+propagation, log generation, and trace generation into a coherent
+per-tick simulation loop.
+
+Fully deterministic: random.Random(seed) exclusively.
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+from server.failures import (
+    FailureSpec,
+    FailureType,
+    apply_failure_to_metrics,
+    make_failure_spec,
+)
+from server.graph import ServiceGraph, ServiceNode, generate_graph
+from server.logs import generate_healthy_log, generate_log_message
+from server.propagation import (
+    CircuitBreaker,
+    ServiceRuntimeState,
+    propagate_failures,
+)
+from server.traces import generate_trace
+
+
+# ---------------------------------------------------------------------------
+# SLO targets
+# ---------------------------------------------------------------------------
+
+# Per-difficulty SLO thresholds: a service is "meeting SLO" if ALL conditions hold
+SLO_TARGETS = {
+    "easy":   {"max_error_rate": 0.05, "max_p99_ms": 500,  "max_cpu": 85, "max_memory": 90},
+    "medium": {"max_error_rate": 0.05, "max_p99_ms": 1000, "max_cpu": 90, "max_memory": 90},
+    "hard":   {"max_error_rate": 0.05, "max_p99_ms": 2000, "max_cpu": 95, "max_memory": 95},
+}
+
+
+def _service_meets_slo(state: ServiceRuntimeState, difficulty: str) -> bool:
+    targets = SLO_TARGETS[difficulty]
+    return (
+        state.error_rate <= targets["max_error_rate"]
+        and state.latency_p99_ms <= targets["max_p99_ms"]
+        and state.cpu_pct <= targets["max_cpu"]
+        and state.memory_pct <= targets["max_memory"]
+    )
+
+
+# ---------------------------------------------------------------------------
+# Pending action effects (delayed remediation)
+# ---------------------------------------------------------------------------
+
+@dataclass
+class PendingEffect:
+    """A remediation action effect that resolves after a delay."""
+    action_type: str
+    target_service: str
+    params: Dict[str, Any]
+    resolve_tick: int   # Tick at which this effect takes place
+
+
+# ---------------------------------------------------------------------------
+# Simulator
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Simulator:
+    """
+    Core simulation engine.
+
+    Usage:
+        sim = Simulator()
+        obs_data = sim.reset(seed=42, difficulty="easy")
+        obs_data = sim.step(action_type="inspect_logs", params={"service_id": "order-service"})
+    """
+
+    # --- Graph and topology ---
+    graph: Optional[ServiceGraph] = None
+    difficulty: str = "easy"
+
+    # --- Mutable per-service state ---
+    services: Dict[str, ServiceRuntimeState] = field(default_factory=dict)
+
+    # --- Failure injection ---
+    failures: List[FailureSpec] = field(default_factory=list)
+    failure_onset_tick: Dict[str, int] = field(default_factory=dict)  # service_id → tick failure started
+
+    # --- Simulation state ---
+    tick: int = 0
+    max_steps: int = 10
+    terminated: bool = False
+    termination_reason: Optional[str] = None
+
+    # --- Pending remediation effects ---
+    pending_effects: List[PendingEffect] = field(default_factory=list)
+
+    # --- Action history ---
+    actions_taken: List[Dict[str, Any]] = field(default_factory=list)
+
+    # --- Deploy history ---
+    deploys: List[Dict[str, Any]] = field(default_factory=list)
+
+    # --- Diagnostic output (from inspect_* actions, consumed by observation builder) ---
+    last_logs: Optional[str] = None
+    last_metric_history: Optional[List[Dict[str, Any]]] = None
+    last_traces: Optional[Dict[str, Any]] = None
+
+    # --- Metric history per service (for inspect_metrics) ---
+    metric_history: Dict[str, List[Dict[str, Any]]] = field(default_factory=dict)
+
+    # --- RNG ---
+    rng: random.Random = field(default_factory=random.Random)
+
+    # --- Remediation tracking ---
+    remediated_services: Dict[str, int] = field(default_factory=dict)  # service_id → tick remediated
+
+    def reset(
+        self,
+        seed: int,
+        difficulty: str,
+        failure_specs: Optional[List[FailureSpec]] = None,
+    ) -> None:
+        """Initialize a new episode. Call get_observation() after this."""
+        self.rng = random.Random(seed)
+        self.difficulty = difficulty
+        self.tick = 0
+        self.terminated = False
+        self.termination_reason = None
+        self.pending_effects = []
+        self.actions_taken = []
+        self.deploys = []
+        self.last_logs = None
+        self.last_metric_history = None
+        self.last_traces = None
+        self.metric_history = {}
+        self.remediated_services = {}
+
+        # Step budgets
+        budgets = {"easy": 10, "medium": 20, "hard": 50}
+        self.max_steps = budgets.get(difficulty, 10)
+
+        # Generate graph
+        self.graph = generate_graph(difficulty, self.rng)
+
+        # Initialize runtime state for each service
+        self.services = {}
+        for node in self.graph.nodes:
+            state = ServiceRuntimeState(
+                service_id=node.id,
+                arrival_rate=node.base_arrival_rate,
+                service_time_local=node.base_service_time_local,
+                thread_pool_size=node.thread_pool_size,
+                replicas=node.default_replicas,
+                version=node.default_version,
+                timeout_ms=node.default_timeout_ms,
+                retry_max=node.default_retry_max,
+                retry_backoff=node.default_retry_backoff,
+                pool_size=node.default_pool_size,
+            )
+            # Initialize circuit breakers for dependencies
+            for dep_id in self.graph.adjacency.get(node.id, []):
+                state.circuit_breakers[dep_id] = CircuitBreaker(
+                    error_threshold=node.default_circuit_breaker_threshold,
+                )
+            self.services[state.service_id] = state
+            self.metric_history[state.service_id] = []
+
+        # Inject failures
+        self.failures = failure_specs or []
+        self.failure_onset_tick = {}
+        for spec in self.failures:
+            self.failure_onset_tick[spec.service_id] = 0
+            svc = self.services.get(spec.service_id)
+            if svc:
+                svc.has_active_failure = True
+                # Apply bad deploy version
+                if spec.failure_type == FailureType.BAD_DEPLOY and spec.bad_version:
+                    svc.previous_version = svc.version
+                    svc.version = spec.bad_version
+                    self.deploys.append({
+                        "service": spec.service_id,
+                        "version": spec.bad_version,
+                        "ticks_ago": 0,
+                    })
+
+        # Run initial tick of failure evolution
+        self._evolve_failures()
+        self._run_propagation()
+        self._record_metrics()
+
+    def step(self, action_type: str, params: Dict[str, Any]) -> float:
+        """
+        Execute one agent action and advance the simulation by one tick.
+        Returns the step reward (dense Δ-SLO shaping).
+        """
+        if self.terminated:
+            return 0.0
+
+        prev_slo = self.get_slo_score()
+
+        # Clear diagnostic output from previous step
+        self.last_logs = None
+        self.last_metric_history = None
+        self.last_traces = None
+
+        # Process the action
+        action_record = self._process_action(action_type, params)
+        self.actions_taken.append(action_record)
+
+        # Advance tick
+        self.tick += 1
+
+        # Resolve pending effects
+        self._resolve_pending_effects()
+
+        # Evolve failures (for non-remediated services)
+        self._evolve_failures()
+
+        # Run propagation
+        self._run_propagation()
+
+        # Record metric history
+        self._record_metrics()
+
+        # Update deploy ticks_ago
+        for d in self.deploys:
+            d["ticks_ago"] += 1
+
+        # Compute reward
+        new_slo = self.get_slo_score()
+        reward = self._compute_reward(prev_slo, new_slo, action_type, action_record)
+
+        # Check termination
+        self._check_termination()
+
+        return reward
+
+    # -------------------------------------------------------------------
+    # Action processing
+    # -------------------------------------------------------------------
+
+    def _process_action(self, action_type: str, params: Dict[str, Any]) -> Dict[str, Any]:
+        """Process an agent action. Returns an action record dict."""
+        service_id = params.get("service_id")
+        record = {
+            "tick": self.tick,
+            "action": action_type,
+            "target": service_id,
+            "success": False,
+            "note": None,
+        }
+
+        if action_type == "noop":
+            record["success"] = True
+            record["note"] = "Waited and observed"
+            return record
+
+        if action_type == "inspect_logs":
+            return self._do_inspect_logs(service_id, record)
+        elif action_type == "inspect_metrics":
+            return self._do_inspect_metrics(service_id, record)
+        elif action_type == "inspect_traces":
+            return self._do_inspect_traces(service_id, record)
+        elif action_type == "restart_service":
+            return self._do_restart(service_id, record)
+        elif action_type == "rollback_service":
+            return self._do_rollback(service_id, record)
+        elif action_type == "scale_service":
+            return self._do_scale(service_id, params, record)
+        elif action_type == "tune_config":
+            return self._do_tune_config(service_id, params, record)
+        elif action_type == "clear_cache":
+            return self._do_clear_cache(params, record)
+        elif action_type == "rebalance_traffic":
+            return self._do_rebalance_traffic(params, record)
+        elif action_type == "pause_job":
+            return self._do_pause_job(params, record)
+        else:
+            record["note"] = f"Unknown action type: {action_type}"
+            return record
+
+    def _do_inspect_logs(self, service_id: Optional[str], record: Dict) -> Dict:
+        svc = self.services.get(service_id or "")
+        if not svc:
+            record["note"] = f"Service '{service_id}' not found"
+            return record
+
+        record["success"] = True
+        # Generate log output based on service state
+        logs_lines = []
+        failure = self._get_failure_for_service(service_id)
+        if failure and svc.error_rate > 0.01:
+            dep = self._get_primary_dependency(service_id)
+            for _ in range(self.rng.randint(3, 6)):
+                logs_lines.append(generate_log_message(
+                    failure.failure_type, service_id, self.rng,
+                    dependency=dep,
+                    error_rate=svc.error_rate,
+                    memory_pct=svc.memory_pct,
+                    p99_ms=svc.latency_p99_ms,
+                    pool_pct=svc.connection_pool_usage_pct,
+                    version=svc.version,
+                    config_key=failure.broken_config_key or "unknown",
+                    config_value=failure.broken_config_value or "unknown",
+                    region=self.graph.node_map[service_id].region if self.graph and service_id in self.graph.node_map else "us-east-1",
+                    throughput=svc.throughput_rps,
+                ))
+        elif svc.error_rate > 0.01:
+            # Propagated errors — show upstream dependency issues
+            dep = self._get_primary_dependency(service_id)
+            logs_lines.append(f"WARN  {service_id} Elevated error rate: {svc.error_rate*100:.1f}%. Upstream dependency {dep} may be degraded.")
+            logs_lines.append(f"ERROR {service_id} Request to {dep} failed: timeout after {svc.timeout_ms}ms. Retry 1/{svc.retry_max}.")
+        else:
+            logs_lines.append(generate_healthy_log(service_id, self.rng))
+
+        self.last_logs = "\n".join(logs_lines)
+        return record
+
+    def _do_inspect_metrics(self, service_id: Optional[str], record: Dict) -> Dict:
+        svc = self.services.get(service_id or "")
+        if not svc:
+            record["note"] = f"Service '{service_id}' not found"
+            return record
+
+        record["success"] = True
+        self.last_metric_history = self.metric_history.get(service_id, [])[-10:]
+        return record
+
+    def _do_inspect_traces(self, service_id: Optional[str], record: Dict) -> Dict:
+        svc = self.services.get(service_id or "")
+        if not svc or not self.graph:
+            record["note"] = f"Service '{service_id}' not found"
+            return record
+
+        record["success"] = True
+        errors = {sid: s.error_rate for sid, s in self.services.items()}
+        latencies = {sid: s.latency_p99_ms for sid, s in self.services.items()}
+        self.last_traces = generate_trace(
+            service_id, self.graph, errors, latencies, self.rng,
+        )
+        return record
+
+    def _do_restart(self, service_id: Optional[str], record: Dict) -> Dict:
+        svc = self.services.get(service_id or "")
+        if not svc:
+            record["note"] = f"Service '{service_id}' not found"
+            return record
+
+        failure = self._get_failure_for_service(service_id)
+        # Restart fixes: CRASH, RESOURCE_LEAK (temporarily), CONFIG_STARTUP (if config was fixed)
+        if failure and failure.failure_type in (FailureType.CRASH, FailureType.RESOURCE_LEAK):
+            delay = self.rng.randint(1, 2)
+            self.pending_effects.append(PendingEffect(
+                action_type="restart_service",
+                target_service=service_id,
+                params={},
+                resolve_tick=self.tick + delay,
+            ))
+            record["success"] = True
+            record["note"] = f"Restarting {service_id}, effect in {delay} tick(s)"
+        elif failure and failure.failure_type == FailureType.CONFIG_STARTUP:
+            # Config startup: restart alone doesn't fix it (need tune_config first)
+            record["success"] = True
+            record["note"] = f"Restarted {service_id} but config error persists — fix config first"
+        elif failure:
+            # Restart gives temporary relief for other failures
+            delay = self.rng.randint(1, 2)
+            self.pending_effects.append(PendingEffect(
+                action_type="restart_partial",
+                target_service=service_id,
+                params={},
+                resolve_tick=self.tick + delay,
+            ))
+            record["success"] = True
+            record["note"] = f"Restarting {service_id}, partial recovery expected in {delay} tick(s)"
+        else:
+            record["success"] = True
+            record["note"] = f"{service_id} is healthy, restart had no effect"
+        return record
+
+    def _do_rollback(self, service_id: Optional[str], record: Dict) -> Dict:
+        svc = self.services.get(service_id or "")
+        if not svc:
+            record["note"] = f"Service '{service_id}' not found"
+            return record
+
+        if not svc.previous_version:
+            record["note"] = f"No previous version to rollback to for {service_id}"
+            return record
+
+        failure = self._get_failure_for_service(service_id)
+        if failure and failure.failure_type == FailureType.BAD_DEPLOY:
+            delay = self.rng.randint(2, 3)
+            self.pending_effects.append(PendingEffect(
+                action_type="rollback_service",
+                target_service=service_id,
+                params={"version": svc.previous_version},
+                resolve_tick=self.tick + delay,
+            ))
+            record["success"] = True
+            record["note"] = f"Rolling back {service_id} to {svc.previous_version}, effect in {delay} tick(s)"
+        else:
+            record["success"] = True
+            record["note"] = f"Rollback queued for {service_id} but issue may not be deploy-related"
+            delay = self.rng.randint(2, 3)
+            self.pending_effects.append(PendingEffect(
+                action_type="rollback_service",
+                target_service=service_id,
+                params={"version": svc.previous_version},
+                resolve_tick=self.tick + delay,
+            ))
+        return record
+
+    def _do_scale(self, service_id: Optional[str], params: Dict, record: Dict) -> Dict:
+        svc = self.services.get(service_id or "")
+        if not svc:
+            record["note"] = f"Service '{service_id}' not found"
+            return record
+
+        target_replicas = params.get("replicas", svc.replicas + 1)
+        node = self.graph.node_map.get(service_id) if self.graph else None
+        max_r = node.max_replicas if node else 8
+        target_replicas = max(1, min(target_replicas, max_r))
+
+        delay = self.rng.randint(2, 4)
+        self.pending_effects.append(PendingEffect(
+            action_type="scale_service",
+            target_service=service_id,
+            params={"replicas": target_replicas},
+            resolve_tick=self.tick + delay,
+        ))
+        record["success"] = True
+        record["note"] = f"Scaling {service_id} to {target_replicas} replicas, effect in {delay} tick(s)"
+        return record
+
+    def _do_tune_config(self, service_id: Optional[str], params: Dict, record: Dict) -> Dict:
+        svc = self.services.get(service_id or "")
+        if not svc:
+            record["note"] = f"Service '{service_id}' not found"
+            return record
+
+        key = params.get("key", "")
+        value = params.get("value", "")
+        record["success"] = True
+        record["target"] = service_id
+
+        failure = self._get_failure_for_service(service_id)
+        if failure and failure.failure_type in (FailureType.CONFIG_STARTUP, FailureType.CONFIG_RUNTIME):
+            if key == failure.broken_config_key:
+                # Correct fix!
+                self.pending_effects.append(PendingEffect(
+                    action_type="tune_config_fix",
+                    target_service=service_id,
+                    params={"key": key, "value": value},
+                    resolve_tick=self.tick + 1,
+                ))
+                record["note"] = f"Config key '{key}' updated on {service_id}. Fix takes effect next tick."
+            else:
+                record["note"] = f"Config key '{key}' updated on {service_id}, but this may not be the broken key."
+        else:
+            # General config tune (e.g., timeout, retry)
+            self._apply_config_immediately(svc, key, value)
+            record["note"] = f"Config '{key}'={value} applied to {service_id}"
+        return record
+
+    def _do_clear_cache(self, params: Dict, record: Dict) -> Dict:
+        cache_name = params.get("cache_name") or params.get("service_id", "")
+        record["target"] = cache_name
+
+        if not self.graph or cache_name not in self.graph.cache_services:
+            record["note"] = f"'{cache_name}' is not a cache service"
+            return record
+
+        failure = self._get_failure_for_service(cache_name)
+        if failure and failure.failure_type == FailureType.CACHE_FAILURE:
+            self.pending_effects.append(PendingEffect(
+                action_type="clear_cache",
+                target_service=cache_name,
+                params={},
+                resolve_tick=self.tick + 1,
+            ))
+            record["success"] = True
+            record["note"] = f"Flushing cache {cache_name}, recovery in 1 tick"
+        else:
+            record["success"] = True
+            record["note"] = f"Cache {cache_name} flushed (was not failing)"
+        return record
+
+    def _do_rebalance_traffic(self, params: Dict, record: Dict) -> Dict:
+        from_region = params.get("from_region", "")
+        to_region = params.get("to_region", "")
+        pct = params.get("pct", 50)
+        record["target"] = f"{from_region}->{to_region}"
+
+        if not self.graph or not self.graph.has_multiple_regions:
+            record["note"] = "Traffic rebalancing only available in multi-region (hard) mode"
+            return record
+
+        delay = self.rng.randint(2, 3)
+        self.pending_effects.append(PendingEffect(
+            action_type="rebalance_traffic",
+            target_service="",
+            params={"from_region": from_region, "to_region": to_region, "pct": pct},
+            resolve_tick=self.tick + delay,
+        ))
+        record["success"] = True
+        record["note"] = f"Shifting {pct}% traffic from {from_region} to {to_region}, effect in {delay} tick(s)"
+        return record
+
+    def _do_pause_job(self, params: Dict, record: Dict) -> Dict:
+        job_name = params.get("job_name") or params.get("service_id", "")
+        record["target"] = job_name
+
+        if not self.graph or job_name not in self.graph.background_jobs:
+            record["note"] = f"'{job_name}' is not a background job service"
+            return record
+
+        svc = self.services.get(job_name)
+        if svc:
+            svc.arrival_rate *= 0.3  # Reduce load significantly
+            record["success"] = True
+            record["note"] = f"Background job on {job_name} paused, load reduced"
+        return record
+
+    # -------------------------------------------------------------------
+    # Effect resolution
+    # -------------------------------------------------------------------
+
+    def _resolve_pending_effects(self) -> None:
+        """Resolve pending effects that have reached their tick."""
+        still_pending = []
+        for effect in self.pending_effects:
+            if self.tick >= effect.resolve_tick:
+                self._apply_effect(effect)
+            else:
+                still_pending.append(effect)
+        self.pending_effects = still_pending
+
+    def _apply_effect(self, effect: PendingEffect) -> None:
+        svc = self.services.get(effect.target_service)
+
+        if effect.action_type == "restart_service":
+            # Full restart: clears crash/leak failures
+            if svc:
+                self._remediate_service(effect.target_service)
+                svc.memory_pct = 30.0  # Reset memory (leak fix)
+
+        elif effect.action_type == "restart_partial":
+            # Partial: temporary relief
+            if svc:
+                svc.error_rate *= 0.5
+                svc.memory_pct = max(30.0, svc.memory_pct * 0.7)
+
+        elif effect.action_type == "rollback_service":
+            if svc:
+                version = effect.params.get("version", svc.previous_version)
+                svc.version = version
+                svc.previous_version = None
+                self._remediate_service(effect.target_service)
+                self.deploys.append({
+                    "service": effect.target_service,
+                    "version": version,
+                    "ticks_ago": 0,
+                })
+
+        elif effect.action_type == "scale_service":
+            if svc:
+                svc.replicas = effect.params.get("replicas", svc.replicas)
+
+        elif effect.action_type == "tune_config_fix":
+            self._remediate_service(effect.target_service)
+            # If config_startup, also need a restart — but we apply partial fix
+            failure = self._get_failure_for_service(effect.target_service)
+            if failure and failure.failure_type == FailureType.CONFIG_STARTUP:
+                # Config fixed + implicit restart
+                if svc:
+                    svc.error_rate = 0.02  # Near-zero while restarting
+
+        elif effect.action_type == "clear_cache":
+            self._remediate_service(effect.target_service)
+
+        elif effect.action_type == "rebalance_traffic":
+            # Reduce arrival rate in from_region, increase in to_region
+            from_region = effect.params.get("from_region", "")
+            to_region = effect.params.get("to_region", "")
+            pct = effect.params.get("pct", 50) / 100.0
+            if self.graph:
+                for node in self.graph.nodes:
+                    s = self.services.get(node.id)
+                    if not s:
+                        continue
+                    if node.region == from_region:
+                        s.arrival_rate *= (1 - pct)
+                    elif node.region == to_region:
+                        s.arrival_rate *= (1 + pct * 0.5)  # Some traffic absorbed
+
+    def _remediate_service(self, service_id: str) -> None:
+        """Mark a service as remediated — stop failure evolution."""
+        self.remediated_services[service_id] = self.tick
+        svc = self.services.get(service_id)
+        if svc:
+            svc.has_active_failure = False
+            svc.failure_ticks = 0
+
+    def _apply_config_immediately(self, svc: ServiceRuntimeState, key: str, value: Any) -> None:
+        """Apply a config change that takes effect immediately."""
+        if key == "timeout_ms":
+            svc.timeout_ms = int(value)
+        elif key == "retry_max":
+            svc.retry_max = int(value)
+        elif key == "pool_size":
+            svc.pool_size = int(value)
+        elif key == "retry_backoff":
+            svc.retry_backoff = bool(value)
+
+    # -------------------------------------------------------------------
+    # Failure evolution
+    # -------------------------------------------------------------------
+
+    def _evolve_failures(self) -> None:
+        """Evolve all active failures by one tick."""
+        for spec in self.failures:
+            sid = spec.service_id
+            if sid in self.remediated_services:
+                # Remediated — gradually recover
+                svc = self.services.get(sid)
+                if svc:
+                    svc.error_rate = max(0.0, svc.error_rate * 0.5)
+                    svc.latency_p99_ms = max(50.0, svc.latency_p99_ms * 0.7)
+                    svc.cpu_pct = max(10.0, svc.cpu_pct * 0.8)
+                    svc.memory_pct = max(25.0, svc.memory_pct * 0.9)
+                    svc.connection_pool_usage_pct = max(5.0, svc.connection_pool_usage_pct * 0.7)
+                    svc.status = svc.compute_status()
+                continue
+
+            svc = self.services.get(sid)
+            if not svc:
+                continue
+
+            onset = self.failure_onset_tick.get(sid, 0)
+            ticks_since = self.tick - onset
+
+            node = self.graph.node_map.get(sid) if self.graph else None
+            base_p99 = 100.0
+            base_cpu = 15.0
+            base_memory = 30.0
+            base_pool = 10.0
+
+            error_rate, p99_ms, cpu_pct, memory_pct, pool_pct = apply_failure_to_metrics(
+                spec, ticks_since,
+                base_error_rate=0.0,
+                base_p99_ms=base_p99,
+                base_cpu=base_cpu,
+                base_memory=base_memory,
+                base_pool=base_pool,
+                rng=self.rng,
+            )
+
+            svc.error_rate = error_rate
+            svc.update_latency_percentiles(base_p99, p99_ms / base_p99, self.rng)
+            svc.cpu_pct = cpu_pct
+            svc.memory_pct = memory_pct
+            svc.connection_pool_usage_pct = pool_pct
+            svc.failure_ticks = ticks_since
+            svc.status = svc.compute_status()
+
+    def _run_propagation(self) -> None:
+        """Run propagation engine to cascade failures through the graph."""
+        if not self.graph:
+            return
+
+        edge_activation = {}
+        for edge in self.graph.edges:
+            edge_activation[(edge.source, edge.target)] = edge.activation_probability
+
+        propagate_failures(
+            self.services,
+            self.graph.adjacency,
+            self.graph.reverse_adjacency,
+            edge_activation,
+            self.rng,
+            current_tick=self.tick,
+        )
+
+    # -------------------------------------------------------------------
+    # Metric recording
+    # -------------------------------------------------------------------
+
+    def _record_metrics(self) -> None:
+        """Record current metrics snapshot for all services."""
+        for sid, svc in self.services.items():
+            self.metric_history[sid].append({
+                "tick": self.tick,
+                "error_rate": round(svc.error_rate, 4),
+                "latency_p99_ms": round(svc.latency_p99_ms, 1),
+                "cpu_pct": round(svc.cpu_pct, 1),
+                "memory_pct": round(svc.memory_pct, 1),
+                "pool_pct": round(svc.connection_pool_usage_pct, 1),
+                "throughput_rps": round(svc.throughput_rps, 1),
+                "status": svc.status,
+            })
+
+    # -------------------------------------------------------------------
+    # Reward computation
+    # -------------------------------------------------------------------
+
+    def _compute_reward(
+        self, prev_slo: float, new_slo: float,
+        action_type: str, record: Dict,
+    ) -> float:
+        """Dense Δ-SLO reward with action-type penalties."""
+        # Base: delta SLO (positive = improvement)
+        delta = new_slo - prev_slo
+        reward = delta * 10.0  # Scale up for signal strength
+
+        # Bonus for reaching full recovery
+        if new_slo >= 1.0:
+            reward += 5.0
+
+        # Penalty for invalid/failed actions
+        if not record.get("success", False):
+            reward -= 0.5
+
+        # Small penalty for non-diagnostic actions (encourage efficiency)
+        if action_type not in ("inspect_logs", "inspect_metrics", "inspect_traces", "noop"):
+            reward -= 0.1  # Small cost for remediation actions
+
+        # Penalty for redundant noops when system is degraded
+        if action_type == "noop" and new_slo < 0.9:
+            reward -= 0.2
+
+        return round(reward, 4)
+
+    # -------------------------------------------------------------------
+    # Termination
+    # -------------------------------------------------------------------
+
+    def _check_termination(self) -> None:
+        """Check if the episode should end."""
+        slo = self.get_slo_score()
+
+        # Success: all SLOs met
+        if slo >= 1.0:
+            self.terminated = True
+            self.termination_reason = "resolved"
+            return
+
+        # Timeout: exceeded step budget
+        if self.tick >= self.max_steps:
+            self.terminated = True
+            self.termination_reason = "timeout"
+            return
+
+        # System collapse: all services down
+        down_count = sum(1 for s in self.services.values() if s.status == "down")
+        if down_count == len(self.services) and len(self.services) > 0:
+            self.terminated = True
+            self.termination_reason = "failed"
+
+    # -------------------------------------------------------------------
+    # Observation helpers
+    # -------------------------------------------------------------------
+
+    def get_slo_score(self) -> float:
+        """Fraction of services meeting SLO targets."""
+        if not self.services:
+            return 0.0
+        meeting = sum(1 for s in self.services.values() if _service_meets_slo(s, self.difficulty))
+        return meeting / len(self.services)
+
+    def get_observation_summary(self) -> str:
+        """Generate a natural-language summary of the current state."""
+        slo = self.get_slo_score()
+        total = len(self.services)
+        healthy = sum(1 for s in self.services.values() if s.status == "healthy")
+        degraded = sum(1 for s in self.services.values() if s.status == "degraded")
+        critical = sum(1 for s in self.services.values() if s.status == "critical")
+        down = sum(1 for s in self.services.values() if s.status == "down")
+
+        parts = []
+        if down > 0:
+            parts.append(f"{down} service(s) DOWN")
+        if critical > 0:
+            parts.append(f"{critical} CRITICAL")
+        if degraded > 0:
+            parts.append(f"{degraded} degraded")
+        if healthy > 0:
+            parts.append(f"{healthy} healthy")
+
+        status_str = ", ".join(parts) if parts else "all nominal"
+        return f"Tick {self.tick}/{self.max_steps}: SLO compliance {slo*100:.0f}% ({status_str}). {total} services total."
+
+    def get_alerts(self) -> List[Dict[str, Any]]:
+        """Generate active alerts from current service states."""
+        alerts = []
+        for sid, svc in self.services.items():
+            if svc.error_rate >= 0.50:
+                alerts.append({
+                    "severity": "critical",
+                    "service": sid,
+                    "type": "error_rate_high",
+                    "message": f"{sid} error rate at {svc.error_rate*100:.0f}%",
+                    "first_seen_tick": max(0, self.tick - svc.failure_ticks),
+                })
+            elif svc.error_rate >= 0.05:
+                alerts.append({
+                    "severity": "warning",
+                    "service": sid,
+                    "type": "error_rate_high",
+                    "message": f"{sid} error rate elevated at {svc.error_rate*100:.1f}%",
+                    "first_seen_tick": max(0, self.tick - svc.failure_ticks),
+                })
+
+            if svc.latency_p99_ms >= 5000:
+                alerts.append({
+                    "severity": "critical",
+                    "service": sid,
+                    "type": "latency_high",
+                    "message": f"{sid} p99 latency {svc.latency_p99_ms:.0f}ms",
+                    "first_seen_tick": max(0, self.tick - svc.failure_ticks),
+                })
+            elif svc.latency_p99_ms >= 1000:
+                alerts.append({
+                    "severity": "warning",
+                    "service": sid,
+                    "type": "latency_high",
+                    "message": f"{sid} p99 latency elevated at {svc.latency_p99_ms:.0f}ms",
+                    "first_seen_tick": max(0, self.tick - svc.failure_ticks),
+                })
+
+            if svc.status == "down":
+                alerts.append({
+                    "severity": "critical",
+                    "service": sid,
+                    "type": "service_down",
+                    "message": f"{sid} is DOWN",
+                    "first_seen_tick": max(0, self.tick - svc.failure_ticks),
+                })
+
+            if svc.memory_pct >= 90:
+                alerts.append({
+                    "severity": "warning",
+                    "service": sid,
+                    "type": "memory_high",
+                    "message": f"{sid} memory at {svc.memory_pct:.0f}%",
+                    "first_seen_tick": max(0, self.tick - svc.failure_ticks),
+                })
+
+            if svc.connection_pool_usage_pct >= 80:
+                alerts.append({
+                    "severity": "warning",
+                    "service": sid,
+                    "type": "connection_pool_saturated",
+                    "message": f"{sid} connection pool at {svc.connection_pool_usage_pct:.0f}%",
+                    "first_seen_tick": max(0, self.tick - svc.failure_ticks),
+                })
+
+            # Circuit breaker alerts
+            for dep_id, breaker in svc.circuit_breakers.items():
+                if breaker.state.value == "OPEN":
+                    alerts.append({
+                        "severity": "warning",
+                        "service": sid,
+                        "type": "circuit_breaker_open",
+                        "message": f"{sid} circuit breaker OPEN for {dep_id}",
+                        "first_seen_tick": max(0, self.tick - breaker.ticks_in_current_state),
+                    })
+
+        # Sort by severity (critical first)
+        severity_order = {"critical": 0, "warning": 1, "info": 2}
+        alerts.sort(key=lambda a: severity_order.get(a["severity"], 9))
+        return alerts
+
+    def get_legal_actions(self) -> List[Dict[str, Any]]:
+        """Return the set of currently legal actions with valid targets."""
+        service_ids = list(self.services.keys())
+        actions = [
+            {"action_type": "noop", "valid_targets": []},
+            {"action_type": "inspect_logs", "valid_targets": service_ids},
+            {"action_type": "inspect_metrics", "valid_targets": service_ids},
+            {"action_type": "inspect_traces", "valid_targets": service_ids},
+            {"action_type": "restart_service", "valid_targets": service_ids},
+        ]
+
+        # Rollback: only services with previous versions
+        rollback_targets = [sid for sid, s in self.services.items() if s.previous_version]
+        if rollback_targets:
+            actions.append({"action_type": "rollback_service", "valid_targets": rollback_targets})
+
+        # Scale: all services
+        actions.append({"action_type": "scale_service", "valid_targets": service_ids})
+
+        # Tune config: all services
+        actions.append({"action_type": "tune_config", "valid_targets": service_ids})
+
+        # Clear cache: only cache services
+        if self.graph and self.graph.cache_services:
+            actions.append({"action_type": "clear_cache", "valid_targets": self.graph.cache_services})
+
+        # Rebalance traffic: only in multi-region
+        if self.graph and self.graph.has_multiple_regions:
+            actions.append({
+                "action_type": "rebalance_traffic",
+                "valid_targets": self.graph.regions,
+            })
+
+        # Pause job: only background job services
+        if self.graph and self.graph.background_jobs:
+            actions.append({"action_type": "pause_job", "valid_targets": self.graph.background_jobs})
+
+        return actions
+
+    def get_service_observations(self) -> List[Dict[str, Any]]:
+        """Build per-service observation dicts."""
+        result = []
+        for sid, svc in self.services.items():
+            node = self.graph.node_map.get(sid) if self.graph else None
+            deps = self.graph.adjacency.get(sid, []) if self.graph else []
+            cb_states = {
+                dep: breaker.state.value
+                for dep, breaker in svc.circuit_breakers.items()
+            }
+            result.append({
+                "id": sid,
+                "layer": node.layer if node else "unknown",
+                "status": svc.status,
+                "error_rate": round(svc.error_rate, 4),
+                "latency_p50_ms": round(svc.latency_p50_ms, 1),
+                "latency_p95_ms": round(svc.latency_p95_ms, 1),
+                "latency_p99_ms": round(svc.latency_p99_ms, 1),
+                "throughput_rps": round(svc.throughput_rps, 1),
+                "cpu_pct": round(svc.cpu_pct, 1),
+                "memory_pct": round(svc.memory_pct, 1),
+                "connection_pool_usage_pct": round(svc.connection_pool_usage_pct, 1),
+                "replicas": svc.replicas,
+                "version": svc.version,
+                "previous_version": svc.previous_version,
+                "depends_on": deps,
+                "circuit_breakers": cb_states,
+            })
+        return result
+
+    # -------------------------------------------------------------------
+    # Internal helpers
+    # -------------------------------------------------------------------
+
+    def _get_failure_for_service(self, service_id: Optional[str]) -> Optional[FailureSpec]:
+        if not service_id:
+            return None
+        for spec in self.failures:
+            if spec.service_id == service_id and service_id not in self.remediated_services:
+                return spec
+        return None
+
+    def _get_primary_dependency(self, service_id: Optional[str]) -> str:
+        if not service_id or not self.graph:
+            return "unknown"
+        deps = self.graph.adjacency.get(service_id, [])
+        return deps[0] if deps else "unknown"

server/traces.py

@@ -0,0 +1,157 @@
+"""
+server/traces.py — Distributed trace generation for inspect_traces action.
+
+Generates realistic Jaeger/Zipkin-style trace trees showing request flow
+through the service dependency graph. Healthy services show normal latencies;
+failing services show errors, timeouts, and cascading delays.
+
+Each trace is a tree of spans rooted at the inspected service.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Any, Dict, List, Optional
+
+from server.graph import ServiceGraph
+
+
+def _make_span_id(rng: random.Random) -> str:
+    return f"{rng.randint(0, 0xFFFFFFFF):08x}"
+
+
+def _make_trace_id(rng: random.Random) -> str:
+    return f"{rng.randint(0, 0xFFFFFFFFFFFFFFFF):016x}"
+
+
+def generate_trace(
+    service_id: str,
+    graph: ServiceGraph,
+    service_errors: Dict[str, float],
+    service_latencies: Dict[str, float],
+    rng: random.Random,
+    max_depth: int = 4,
+) -> Dict[str, Any]:
+    """
+    Generate a distributed trace tree rooted at service_id.
+
+    Returns a dict with trace_id, root_span, and flat spans list.
+    service_errors: service_id → error_rate (0.0–1.0)
+    service_latencies: service_id → p99_ms
+    """
+    trace_id = _make_trace_id(rng)
+    spans: List[Dict[str, Any]] = []
+
+    def _build_span(
+        svc_id: str,
+        parent_span_id: Optional[str],
+        depth: int,
+        start_offset_ms: float,
+    ) -> Dict[str, Any]:
+        span_id = _make_span_id(rng)
+        error_rate = service_errors.get(svc_id, 0.0)
+        base_latency = service_latencies.get(svc_id, rng.uniform(5, 50))
+        has_error = rng.random() < error_rate
+
+        # Span duration: base latency + noise
+        if has_error and error_rate > 0.8:
+            # Fast fail or timeout
+            duration_ms = rng.choice([
+                rng.uniform(0.5, 5),       # Fast fail
+                rng.uniform(3000, 10000),   # Timeout
+            ])
+        elif has_error:
+            duration_ms = base_latency * rng.uniform(1.5, 5.0)
+        else:
+            duration_ms = base_latency * rng.uniform(0.3, 1.2)
+
+        duration_ms = max(0.1, duration_ms)
+
+        span = {
+            "span_id": span_id,
+            "parent_span_id": parent_span_id,
+            "service": svc_id,
+            "operation": _operation_name(svc_id, rng),
+            "start_ms": round(start_offset_ms, 1),
+            "duration_ms": round(duration_ms, 1),
+            "status": "ERROR" if has_error else "OK",
+            "tags": {},
+        }
+
+        if has_error:
+            span["tags"]["error"] = True
+            span["tags"]["error.message"] = _error_message(svc_id, error_rate, rng)
+
+        node = graph.node_map.get(svc_id)
+        if node:
+            span["tags"]["service.layer"] = node.layer
+            span["tags"]["service.region"] = node.region
+
+        spans.append(span)
+
+        # Recurse into downstream dependencies
+        if depth < max_depth:
+            deps = graph.adjacency.get(svc_id, [])
+            child_offset = start_offset_ms + rng.uniform(0.1, 2.0)
+            for dep_id in deps:
+                # Check edge activation (probabilistic)
+                edge = next(
+                    (e for e in graph.edges if e.source == svc_id and e.target == dep_id),
+                    None,
+                )
+                if edge and rng.random() > edge.activation_probability:
+                    continue
+
+                child_span = _build_span(dep_id, span_id, depth + 1, child_offset)
+                child_offset += child_span["duration_ms"] + rng.uniform(0.1, 1.0)
+
+        return span
+
+    root_span = _build_span(service_id, None, 0, 0.0)
+
+    # Compute total trace duration
+    if spans:
+        total_duration = max(s["start_ms"] + s["duration_ms"] for s in spans)
+    else:
+        total_duration = 0.0
+
+    return {
+        "trace_id": trace_id,
+        "root_service": service_id,
+        "span_count": len(spans),
+        "total_duration_ms": round(total_duration, 1),
+        "spans": spans,
+    }
+
+
+def _operation_name(service_id: str, rng: random.Random) -> str:
+    """Generate a realistic operation name based on service type."""
+    if "gateway" in service_id or "bff" in service_id:
+        return rng.choice(["HTTP GET /api/v1/resource", "HTTP POST /api/v1/action", "HTTP GET /health"])
+    if "auth" in service_id or "identity" in service_id or "session" in service_id:
+        return rng.choice(["validateToken", "authenticate", "refreshSession"])
+    if "postgres" in service_id:
+        return rng.choice(["SELECT", "INSERT", "UPDATE", "pg_pool.checkout"])
+    if "redis" in service_id:
+        return rng.choice(["GET", "SET", "MGET", "EXPIRE"])
+    if "kafka" in service_id:
+        return rng.choice(["produce", "consume", "commitOffset"])
+    if "elasticsearch" in service_id:
+        return rng.choice(["search", "index", "bulk"])
+    return rng.choice(["processRequest", "handleMessage", "execute"])
+
+
+def _error_message(service_id: str, error_rate: float, rng: random.Random) -> str:
+    """Generate a trace-level error message."""
+    if error_rate > 0.8:
+        return rng.choice([
+            f"{service_id}: Connection refused",
+            f"{service_id}: Service unavailable (HTTP 503)",
+            f"{service_id}: Timeout after 5000ms",
+        ])
+    return rng.choice([
+        f"{service_id}: Internal server error (HTTP 500)",
+        f"{service_id}: Upstream dependency timeout",
+        f"{service_id}: Rate limited (HTTP 429)",
+        f"{service_id}: Bad gateway (HTTP 502)",
+    ])