Fix Phase 2: add [START]/[STEP]/[END] structured output + simulation fixes

inference.py:
- Add log_start/log_step/log_end emitting [START]/[STEP]/[END] to stdout
with flush=True — required by hackathon Phase 2 evaluator
- Strip all testing bloat: Bedrock wrapper, thinking model detection,
Kimi/Nemotron-legacy/fallback provider code, AgentAction Pydantic schema
- Standard OpenAI client path only (temperature=0, max_tokens=1024)
- Keep: system prompt, observation builder, action parser, diagnosis memory

server/simulator.py:
- CASCADING_LATENCY now remediable via restart_service and scale_service
- rebalance_traffic param aliasing (accepts region/service_id/target)
- arrival_rate floor (20% of base) prevents rebalance spam from starving a region
- tune_config diagnostic hint shows exact key and example value

server/propagation.py:
- Reset service_time_local and arrival_rate to baseline each propagation tick
for non-failing services — fixes permanent SLO ceiling after upstream heals
- Circuit breaker: cooldown_ticks 5→3, half_open_success_threshold 3→2
- Add base_arrival_rate / base_service_time_local fields to ServiceRuntimeState

server/logs.py:
- CASCADING_LATENCY templates now describe self-overload on the service itself,
not a downstream dependency — eliminates the "inspect healthy postgres" loop

server/grader.py:
- time_efficiency on timeout uses slo_factor*0.5 + speed_factor*0.5, rewarding
faster partial resolution rather than a constant slo*0.3

pyproject.toml / uv.lock:
- Remove anthropic[vertex] dependency (testing-only, not needed for submission)

inference.py

@@ -26,33 +26,15 @@ from typing import Any, Dict, List
 
 from openai import OpenAI
 
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
 API_BASE_URL = os.getenv("API_BASE_URL", "https://api.groq.com/openai/v1")
 API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
 MODEL_NAME = os.getenv("MODEL_NAME", "llama-3.3-70b-versatile")
-
-# Fallback providers tried in order if the primary hits rate limits or errors.
-# Each uses the same HF_TOKEN env var as the API key — all are OpenAI-compatible.
-_GROQ_BACKUP_KEY = os.getenv("GROQ_BACKUP_KEY")
-_FALLBACK_PROVIDERS = [
-    # Tier 1 fallback: backup Groq account, same strong model (fresh 100k TPD)
-    *([{
-        "base_url": "https://api.groq.com/openai/v1",
-        "model": "llama-3.3-70b-versatile",
-        "api_key": _GROQ_BACKUP_KEY,
-    }] if _GROQ_BACKUP_KEY else []),
-    # Tier 2 fallback: same Groq key, lighter model (14,400 RPD free)
-    {
-        "base_url": "https://api.groq.com/openai/v1",
-        "model": "llama-3.1-8b-instant",
-        "api_key": API_KEY,
-    },
-    # Tier 3 fallback: HuggingFace Inference Router
-    {
-        "base_url": "https://router.huggingface.co/v1",
-        "model": "Qwen/Qwen2.5-72B-Instruct",
-        "api_key": os.getenv("HF_INFERENCE_TOKEN") or API_KEY,
-    },
-]
+ENV_URL = os.getenv("ENV_URL", "http://localhost:7860")
+ENV_NAME = "sevzero"
 
 SYSTEM_PROMPT = textwrap.dedent("""\
     You are an expert Site Reliability Engineer (SRE) responding to a production incident.
@@ -65,9 +47,8 @@ SYSTEM_PROMPT = textwrap.dedent("""\
     2. Diagnose the root cause from log patterns:
        - OOMKilled/CrashLoopBackOff -> restart_service
        - NullPointerException/TypeError + recent deploy -> rollback_service
-       - "password authentication failed"/"config not found" -> tune_config with the broken key
-         (the logs will show: "Configuration diagnostic: key '<KEY>' has invalid value")
-       - Thread pool exhaustion/timeout from downstream -> fix the downstream dependency first
+       - "Configuration diagnostic: key '<KEY>'" -> tune_config with that exact key, value='correct'
+       - Thread pool exhaustion on THIS service -> restart_service or scale_service on THIS service
        - Memory climbing linearly -> restart_service (resource leak)
        - HikariPool exhaustion/slow queries -> scale_service or restart_service on the DB
        - CLUSTERDOWN/cache miss -> clear_cache
@@ -85,83 +66,120 @@ SYSTEM_PROMPT = textwrap.dedent("""\
         {"action_type": "tune_config", "params": {"service_id": "order-service", "key": "api_endpoint", "value": "correct"}}
     - clear_cache:
         {"action_type": "clear_cache", "params": {"cache_name": "redis-cache"}}
+    - rebalance_traffic:
+        {"action_type": "rebalance_traffic", "params": {"from_region": "us-east-1", "to_region": "us-west-2"}}
     - noop:
         {"action_type": "noop", "params": {}}
 """)
 
+# ---------------------------------------------------------------------------
+# Structured logging — required by hackathon evaluator
+# ---------------------------------------------------------------------------
+
+
+def log_start(task: str, env: str, model: str) -> None:
+    print(f"[START] task={task} env={env} model={model}", flush=True)
+
+
+def log_step(step: int, action: str, reward: float, done: bool, error: Any = None) -> None:
+    print(
+        f"[STEP] step={step} action={action} reward={reward:.4f} "
+        f"done={str(done).lower()} error={error}",
+        flush=True,
+    )
+
+
+def log_end(task: str, success: bool, steps: int, score: float, rewards: List[float]) -> None:
+    print(
+        f"[END] task={task} success={str(success).lower()} steps={steps} "
+        f"score={score:.4f} rewards={rewards}",
+        flush=True,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Token tracking
+# ---------------------------------------------------------------------------
+
+_token_usage: Dict[str, int] = {"prompt": 0, "completion": 0}
+
+
+def _track_usage(completion: Any) -> None:
+    usage = getattr(completion, "usage", None)
+    if not usage:
+        return
+    _token_usage["prompt"] += getattr(usage, "prompt_tokens", 0)
+    _token_usage["completion"] += getattr(usage, "completion_tokens", 0)
 
-def _call_llm(
-    messages: List[Dict[str, Any]],
-    primary_client: OpenAI,
-    primary_model: str,
-) -> str:
-    """Call LLM with automatic fallback on rate limit or error."""
-    providers = [{"client": primary_client, "model": primary_model}] + [
-        {
-            "client": OpenAI(base_url=p["base_url"], api_key=p["api_key"]),
-            "model": p["model"],
-        }
-        for p in _FALLBACK_PROVIDERS
-    ]
-
-    last_err = None
-    for i, provider in enumerate(providers):
+
+# ---------------------------------------------------------------------------
+# LLM call — standard OpenAI client, retry on transient errors
+# ---------------------------------------------------------------------------
+
+
+def _call_llm(messages: List[Dict[str, Any]], client: OpenAI) -> str:
+    """Call the LLM with exponential backoff retry. Returns raw response text."""
+    attempt = 0
+    while True:
         try:
-            completion = provider["client"].chat.completions.create(
-                model=provider["model"],
+            completion = client.chat.completions.create(
+                model=MODEL_NAME,
                 messages=messages,
-                temperature=0.2,
-                max_tokens=4000,  # Thinking models (Gemini 3.1 Pro, o3) use tokens for reasoning
+                temperature=0,
+                max_tokens=1024,
             )
-            # content can be None for thinking models if limit was too low;
-            # 4000 ensures thinking budget + response both fit
+            _track_usage(completion)
             return completion.choices[0].message.content or ""
         except Exception as e:
-            last_err = e
-            is_rate_limit = any(x in str(e).lower() for x in ("429", "rate_limit", "quota", "credits", "402"))
-            label = "fallback" if i > 0 else "primary"
-            print(f"  [{label} {provider['model']}] error: {e}")
-            if is_rate_limit and i < len(providers) - 1:
-                time.sleep(3)
-                continue
-            if i < len(providers) - 1:
-                continue
-    print(f"  All providers failed. Last error: {last_err}")
-    return '{"action_type": "noop", "params": {}}'
+            attempt += 1
+            wait = min(10 * (2 ** (attempt - 1)), 60)
+            print(f"  [attempt {attempt}] {MODEL_NAME} error: {e}", flush=True)
+            print(f"  [retry] waiting {wait}s...", flush=True)
+            time.sleep(wait)
+
+
+# ---------------------------------------------------------------------------
+# Observation → prompt
+# ---------------------------------------------------------------------------
 
 
 def build_observation_prompt(obs: Dict[str, Any]) -> str:
-    """Build a concise prompt from the observation."""
     parts = [f"## Incident Status\n{obs.get('observation_summary', 'N/A')}"]
 
-    # Alerts (most important)
     alerts = obs.get("alerts", [])
     if alerts:
         alert_lines = [f"  [{a['severity'].upper()}] {a['message']}" for a in alerts[:10]]
         parts.append("## Active Alerts\n" + "\n".join(alert_lines))
 
-    # Service states (condensed — degraded only)
     services = obs.get("services", [])
     degraded = [s for s in services if s.get("status") in ("degraded", "critical", "down")]
     if degraded:
-        svc_lines = [
-            f"  {s['id']} [{s['status']}]: error={s['error_rate']:.1%}, "
-            f"p99={s['latency_p99_ms']:.0f}ms, cpu={s['cpu_pct']:.0f}%, "
-            f"mem={s['memory_pct']:.0f}%, pool={s['connection_pool_usage_pct']:.0f}%"
-            for s in degraded
-        ]
+        # Identify root causes: services that have OPEN circuit breakers pointing at them
+        # from callers, but do not themselves have OPEN outgoing breakers
+        breaker_targets: set = set()
+        for s in services:
+            for dep, state in s.get("circuit_breakers", {}).items():
+                if state == "OPEN":
+                    breaker_targets.add(dep)
+
+        svc_lines = []
+        for s in degraded:
+            sid = s["id"]
+            own_open = any(v == "OPEN" for v in s.get("circuit_breakers", {}).values())
+            is_root = sid in breaker_targets and not own_open
+            label = " [ROOT CAUSE]" if is_root else " [propagation victim]" if sid not in breaker_targets else ""
+            svc_lines.append(
+                f"  {sid} [{s['status']}]{label}: error={s['error_rate']:.1%}, "
+                f"p99={s['latency_p99_ms']:.0f}ms, cpu={s['cpu_pct']:.0f}%, "
+                f"mem={s['memory_pct']:.0f}%"
+            )
         parts.append("## Degraded Services\n" + "\n".join(svc_lines))
 
-    # Recent deploys
     deploys = obs.get("recent_deploys", [])
     if deploys:
-        dep_lines = [
-            f"  {d['service']} -> {d['version']} ({d['ticks_ago']} ticks ago)"
-            for d in deploys
-        ]
+        dep_lines = [f"  {d['service']} -> {d['version']} ({d['ticks_ago']} ticks ago)" for d in deploys]
         parts.append("## Recent Deploys\n" + "\n".join(dep_lines))
 
-    # Actions taken
     actions = obs.get("actions_taken", [])
     if actions:
         act_lines = [
@@ -170,12 +188,10 @@ def build_observation_prompt(obs: Dict[str, Any]) -> str:
         ]
         parts.append("## Recent Actions\n" + "\n".join(act_lines))
 
-    # Logs (if available from inspect)
     logs = obs.get("logs")
     if logs:
         parts.append(f"## Logs\n{logs}")
 
-    # Traces (if available)
     traces = obs.get("traces")
     if traces:
         error_spans = [s for s in traces.get("spans", []) if s.get("status") == "ERROR"]
@@ -186,7 +202,6 @@ def build_observation_prompt(obs: Dict[str, Any]) -> str:
             ]
             parts.append("## Trace Errors\n" + "\n".join(trace_lines))
 
-    # Legal actions
     legal = obs.get("legal_actions", [])
     if legal:
         legal_strs = [f"  {la['action_type']}: targets={la['valid_targets'][:5]}" for la in legal]
@@ -195,17 +210,17 @@ def build_observation_prompt(obs: Dict[str, Any]) -> str:
     return "\n\n".join(parts)
 
 
+# ---------------------------------------------------------------------------
+# Action parsing
+# ---------------------------------------------------------------------------
+
+
 def parse_action(response_text: str) -> Dict[str, Any]:
-    """Parse the model's JSON response into an action dict."""
     text = response_text.strip()
-
-    # Strip markdown code blocks
     if "```json" in text:
         text = text.split("```json")[1].split("```")[0].strip()
     elif "```" in text:
         text = text.split("```")[1].split("```")[0].strip()
-
-    # Extract JSON object
     start = text.find("{")
     end = text.rfind("}") + 1
     if start >= 0 and end > start:
@@ -213,22 +228,24 @@ def parse_action(response_text: str) -> Dict[str, Any]:
             return json.loads(text[start:end])
         except json.JSONDecodeError:
             pass
-
     return {"action_type": "noop", "params": {}}
 
 
+# ---------------------------------------------------------------------------
+# Episode runner
+# ---------------------------------------------------------------------------
+
+
 def run_episode(
     client: OpenAI,
-    env_url: str,
     task_id: str,
-    seed: int = 42,
+    seed: int,
 ) -> Dict[str, Any]:
-    """Run one episode using the OpenEnv HTTP API."""
     import httpx
 
-    base = env_url.rstrip("/")
+    base = ENV_URL.rstrip("/")
 
-    # Reset
+    # Reset environment
     reset_resp = httpx.post(
         f"{base}/reset",
         json={"seed": seed, "task_id": task_id},
@@ -238,58 +255,88 @@ def run_episode(
     obs = resp_data.get("observation", resp_data)
 
     max_steps = obs.get("max_steps", 10)
-    total_reward = 0.0
     done = resp_data.get("done", False)
+    rewards: List[float] = []
 
-    # Rolling conversation: system prompt + last 6 messages (3 turns).
-    # Prevents context explosion on hard tasks (50 steps x ~800 tokens/step).
+    # Persistent episode memory — survives rolling context truncation
     conversation_history: List[Dict[str, Any]] = []
-
-    for step_num in range(max_steps):
+    tried_actions: Dict[str, List[str]] = {}
+    resolved_services: List[str] = []
+
+    def _build_memory() -> str:
+        if not tried_actions and not resolved_services:
+            return ""
+        lines = ["## Episode Memory (do not repeat failed approaches)"]
+        if resolved_services:
+            lines.append(f"  Resolved: {', '.join(resolved_services)}")
+        for act, targets in tried_actions.items():
+            lines.append(f"  {act}: {'; '.join(targets)}")
+        return "\n".join(lines)
+
+    log_start(task=task_id, env=ENV_NAME, model=MODEL_NAME)
+
+    steps_taken = 0
+    for step_num in range(1, max_steps + 1):
         if done:
             break
 
         user_msg = build_observation_prompt(obs)
         conversation_history.append({"role": "user", "content": user_msg})
 
-        # Keep only last 6 messages (3 user+assistant turns) to bound context size
+        # Rolling window of last 6 messages + persistent memory in system prompt
         trimmed = conversation_history[-6:]
-        messages_to_send = [{"role": "system", "content": SYSTEM_PROMPT}] + trimmed
+        memory = _build_memory()
+        system_content = SYSTEM_PROMPT + ("\n\n" + memory if memory else "")
+        messages_to_send = [{"role": "system", "content": system_content}] + trimmed
 
-        response_text = _call_llm(messages_to_send, client, MODEL_NAME)
+        response_text = _call_llm(messages_to_send, client)
         action = parse_action(response_text)
         conversation_history.append({"role": "assistant", "content": response_text})
 
-        print(f"  Step {step_num + 1}: {action.get('action_type', 'noop')}({action.get('params', {})})")
+        act_type = action.get("action_type", "noop")
+        act_params = action.get("params", {})
+        target = act_params.get("service_id") or act_params.get("cache_name") or act_params.get("from_region") or ""
 
-        # Step the environment
-        params = action.get("params", {})
-        # Coerce replicas to int if model sends a string
-        if "replicas" in params:
+        # Coerce replicas to int
+        if "replicas" in act_params:
             try:
-                params["replicas"] = int(params["replicas"])
+                act_params["replicas"] = int(act_params["replicas"])
             except (ValueError, TypeError):
-                params["replicas"] = 2
+                act_params["replicas"] = 2
+
+        print(f"  Step {step_num}: {act_type}({act_params})", flush=True)
 
         step_resp = httpx.post(
             f"{base}/step",
-            json={"action": {
-                "action_type": action.get("action_type", "noop"),
-                "params": params,
-            }},
+            json={"action": {"action_type": act_type, "params": act_params}},
             timeout=30.0,
         )
         try:
             resp_data = step_resp.json()
         except Exception:
-            # Empty or non-JSON response (server error) — treat as noop
             resp_data = {}
+
         obs = resp_data.get("observation", resp_data)
         done = resp_data.get("done", False)
-        reward = obs.get("reward") or resp_data.get("reward") or 0.0
-        total_reward += reward if reward else 0.0
-
-    # Final state + grade
+        reward = float(obs.get("reward") or resp_data.get("reward") or 0.0)
+        rewards.append(reward)
+        steps_taken = step_num
+
+        log_step(step=step_num, action=act_type, reward=reward, done=done)
+
+        # Update persistent memory
+        if act_type not in ("inspect_logs", "inspect_metrics", "inspect_traces", "noop") and target:
+            new_slo = obs.get("global_slo_score", 0.0)
+            for svc in obs.get("services", []):
+                if svc["id"] == target and svc["status"] == "healthy":
+                    if target not in resolved_services:
+                        resolved_services.append(target)
+            entry = f"{target} (slo={new_slo:.0%})"
+            tried_actions.setdefault(act_type, [])
+            if entry not in tried_actions[act_type]:
+                tried_actions[act_type].append(entry)
+
+    # Grade the episode
     final_state = httpx.get(f"{base}/state", timeout=10.0).json()
     grade = httpx.post(
         f"{base}/grader",
@@ -304,55 +351,68 @@ def run_episode(
         timeout=10.0,
     ).json()
 
+    score = grade.get("score", 0.0)
+    outcome = final_state.get("termination_reason", "timeout")
+    success = outcome == "resolved"
+
+    log_end(task=task_id, success=success, steps=steps_taken, score=score, rewards=rewards)
+
     return {
         "task_id": task_id,
         "seed": seed,
-        "total_reward": total_reward,
-        "score": grade.get("score", 0.0),
+        "score": score,
         "slo_recovery": grade.get("slo_recovery", 0.0),
         "action_efficiency": grade.get("action_efficiency", 0.0),
         "time_efficiency": grade.get("time_efficiency", 0.0),
         "steps_taken": final_state.get("step_count", 0),
-        "termination_reason": final_state.get("termination_reason"),
+        "termination_reason": outcome,
+        "rewards": rewards,
     }
 
 
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+
 def main() -> None:
     client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
-    env_url = os.getenv("ENV_URL", "http://localhost:7860")
 
-    tasks = ["easy", "medium", "hard"]
-    seeds = [42, 123, 7]
+    all_tasks = {"easy": 42, "medium": 123, "hard": 7}
+    task_filter = os.getenv("TASKS", "").strip()
+    selected = [t.strip() for t in task_filter.split(",")] if task_filter else list(all_tasks)
+    tasks = [(t, all_tasks[t]) for t in selected if t in all_tasks]
 
-    print("=" * 60)
-    print("SevZero Baseline Inference")
-    print("=" * 60)
-    print(f"Model:       {MODEL_NAME}")
-    print(f"API:         {API_BASE_URL}")
-    print(f"Environment: {env_url}")
-    print()
+    print("=" * 60, flush=True)
+    print("SevZero Baseline Inference", flush=True)
+    print("=" * 60, flush=True)
+    print(f"Model:       {MODEL_NAME}", flush=True)
+    print(f"API:         {API_BASE_URL}", flush=True)
+    print(f"Environment: {ENV_URL}", flush=True)
+    print(flush=True)
 
     results = []
-    for task_id, seed in zip(tasks, seeds):
-        print(f"--- Task: {task_id} (seed={seed}) ---")
-        result = run_episode(client, env_url, task_id, seed)
+    for task_id, seed in tasks:
+        print(f"--- Task: {task_id} (seed={seed}) ---", flush=True)
+        result = run_episode(client, task_id, seed)
         results.append(result)
         print(
             f"  Score: {result['score']:.4f} | SLO: {result['slo_recovery']:.4f} | "
             f"AE: {result['action_efficiency']:.4f} | TE: {result['time_efficiency']:.4f} | "
-            f"Steps: {result['steps_taken']} | Outcome: {result['termination_reason']}"
+            f"Steps: {result['steps_taken']} | Outcome: {result['termination_reason']}",
+            flush=True,
         )
-        print()
+        print(flush=True)
 
-    print("=" * 60)
-    print("Summary")
-    print("=" * 60)
+    print("=" * 60, flush=True)
+    print("Summary", flush=True)
+    print("=" * 60, flush=True)
     for r in results:
-        print(f"  {r['task_id']:8s} score={r['score']:.4f}  slo={r['slo_recovery']:.4f}  steps={r['steps_taken']}")
+        print(f"  {r['task_id']:8s} score={r['score']:.4f}  slo={r['slo_recovery']:.4f}  steps={r['steps_taken']}", flush=True)
     avg_score = sum(r["score"] for r in results) / len(results) if results else 0.0
-    print(f"\n  Average score: {avg_score:.4f}")
+    print(f"\n  Average score: {avg_score:.4f}", flush=True)
 
-    # Save results to outputs/
+    # Save results
     outputs_dir = Path(__file__).parent / "outputs"
     outputs_dir.mkdir(exist_ok=True)
     run_ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
@@ -364,10 +424,15 @@ def main() -> None:
         "results": results,
     }
     out_file = outputs_dir / f"baseline_{run_ts}.json"
-    latest_file = outputs_dir / "baseline_latest.json"
+    (outputs_dir / "baseline_latest.json").write_text(json.dumps(payload, indent=2))
     out_file.write_text(json.dumps(payload, indent=2))
-    latest_file.write_text(json.dumps(payload, indent=2))
-    print(f"\n  Results saved -> {out_file.name}")
+    print(f"\n  Results saved -> {out_file.name}", flush=True)
+
+    total = _token_usage["prompt"] + _token_usage["completion"]
+    print(f"\n  Token usage:", flush=True)
+    print(f"    prompt:     {_token_usage['prompt']:,}", flush=True)
+    print(f"    completion: {_token_usage['completion']:,}", flush=True)
+    print(f"    total:      {total:,}", flush=True)
 
 
 if __name__ == "__main__":

outputs/baseline_latest.json

@@ -1,8 +1,8 @@
 {
-  "run_at": "20260330_144414",
-  "model": "llama-3.3-70b-versatile",
-  "api_base_url": "https://api.groq.com/openai/v1",
-  "average_score": 0.8844,
+  "run_at": "20260401_165311",
+  "model": "us.anthropic.claude-sonnet-4-6",
+  "api_base_url": "https://bedrock-runtime.us-east-1.amazonaws.com",
+  "average_score": 0.9187,
   "results": [
     {
       "task_id": "easy",
@@ -18,10 +18,10 @@
     {
       "task_id": "medium",
       "seed": 123,
-      "total_reward": 7.1222,
-      "score": 0.9325,
+      "total_reward": 7.022200000000001,
+      "score": 0.97,
       "slo_recovery": 1.0,
-      "action_efficiency": 0.75,
+      "action_efficiency": 1.0,
       "time_efficiency": 0.8,
       "steps_taken": 4,
       "termination_reason": "resolved"
@@ -29,11 +29,11 @@
     {
       "task_id": "hard",
       "seed": 7,
-      "total_reward": -3.3000000000000007,
-      "score": 0.7906,
-      "slo_recovery": 0.88,
-      "action_efficiency": 0.9,
-      "time_efficiency": 0.264,
+      "total_reward": -2.8000000000000016,
+      "score": 0.8561,
+      "slo_recovery": 0.92,
+      "action_efficiency": 1.0,
+      "time_efficiency": 0.414,
       "steps_taken": 50,
       "termination_reason": "timeout"
     }

server/grader.py

@@ -82,8 +82,14 @@ def grade_episode(
         # Resolved: reward faster resolution
         time_efficiency = max(0.1, 1.0 - (steps_taken / max_steps))
     else:
-        # Not resolved: partial credit based on how close we got
-        time_efficiency = final_slo_score * 0.3
+        # Not resolved: combine SLO progress with how quickly it was reached.
+        # slo_factor: how much of the system was recovered
+        # speed_factor: steps remaining as a fraction of budget (rewards using fewer steps)
+        # 0.9 discount ensures a resolved episode always scores higher than a
+        # timed-out one under equivalent conditions.
+        slo_factor = final_slo_score
+        speed_factor = max(0.0, 1.0 - (steps_taken / max_steps))
+        time_efficiency = (slo_factor * 0.5 + speed_factor * 0.5) * 0.9
 
     # --- Final score ---
     score = (

server/logs.py

@@ -59,12 +59,17 @@ _TEMPLATES: Dict[FailureType, List[str]] = {
     ],
 
     FailureType.CASCADING_LATENCY: [
-        "WARN  {service} Downstream {dependency} timeout after {timeout_ms}ms. Circuit breaker OPEN for {cooldown}s. {queued} requests queued.",
-        "WARN  {service} Thread pool exhaustion: {active}/{pool_size} threads active. Queue depth: {queue_depth}. Avg wait: {wait_ms}ms.",
-        "ERROR {service} gRPC deadline exceeded: remaining_ms={remaining_ms}. Upstream deadline propagated through {hop_count} hops.",
-        "WARN  {service} Connection pool to {dependency}: active={active}/{pool_size}, pending={pending}. Avg checkout time: {checkout_ms}ms (threshold: {threshold_ms}ms).",
-        "ERROR {service} Request timeout: {dependency} did not respond within {timeout_ms}ms. Retry {retry_count}/{retry_max}.",
-        "WARN  {service} p99 latency spike: {p99_ms}ms (baseline: {baseline_ms}ms). {dependency} response time degrading.",
+        "WARN  {service} Thread pool self-saturation: {active}/{pool_size} worker threads active. Queue depth: {queue_depth}. Avg wait: {wait_ms}ms. "
+        "This service is the bottleneck — scale or rebalance traffic away from this service.",
+        "WARN  {service} Worker thread exhaustion: arrival rate {throughput}rps exceeds processing capacity. "
+        "Active threads: {active}/{pool_size}. Queued: {queue_depth}. Fix: scale_service or rebalance_traffic.",
+        "ERROR {service} Request queue overflow: {queue_depth} requests waiting for worker threads ({active}/{pool_size} busy). "
+        "p99={p99_ms}ms. Root cause is this service's own capacity — restart to clear threads or scale to add capacity.",
+        "WARN  {service} Internal latency spiral: p99={p99_ms}ms (baseline: {baseline_ms}ms). Thread pool utilisation critical. "
+        "Retry amplification causing {throughput}rps effective load. This service needs to be restarted or scaled.",
+        "CRIT  {service} Capacity overload: {active}/{pool_size} threads saturated, {queue_depth} requests pending. "
+        "All downstream timeouts are a symptom of THIS service being overwhelmed. "
+        "Run: restart_service or scale_service on {service}.",
     ],
 
     FailureType.RESOURCE_LEAK: [

server/propagation.py

@@ -38,8 +38,8 @@ class CircuitBreaker:
 
     # Config (tunable by agent via tune_config)
     error_threshold: float = 0.5      # Error rate to trip OPEN
-    cooldown_ticks: int = 5           # Ticks to stay OPEN before half-open
-    half_open_success_threshold: int = 3  # Successes needed to close
+    cooldown_ticks: int = 3           # Ticks to stay OPEN before half-open
+    half_open_success_threshold: int = 2  # Successes needed to close
 
     # Runtime state
     ticks_in_current_state: int = 0

server/simulator.py

@@ -314,8 +314,8 @@ class Simulator:
             # Guarantee the broken config key is always visible in logs for config failures
             if failure.failure_type in (FailureType.CONFIG_STARTUP, FailureType.CONFIG_RUNTIME) and failure.broken_config_key:
                 logs_lines.append(
-                    f"ERROR {service_id} Configuration diagnostic: key '{failure.broken_config_key}' has invalid value. "
-                    f"Run: tune_config(service_id='{service_id}', key='{failure.broken_config_key}', value=<correct_value>)"
+                    f"ERROR {service_id} Configuration diagnostic: key '{failure.broken_config_key}' has invalid value '{failure.broken_config_value}'. "
+                    f"Run: tune_config(service_id='{service_id}', key='{failure.broken_config_key}', value='correct') to restore."
                 )
         elif svc.error_rate > 0.01:
             # Propagated errors — show upstream dependency issues
@@ -359,8 +359,14 @@ class Simulator:
             return record
 
         failure = self._get_failure_for_service(service_id)
-        # Restart fixes: CRASH, RESOURCE_LEAK (temporarily), CONFIG_STARTUP (if config was fixed)
-        if failure and failure.failure_type in (FailureType.CRASH, FailureType.RESOURCE_LEAK):
+        # Restart fixes: CRASH, RESOURCE_LEAK, CASCADING_LATENCY (clears thread pool),
+        # DB_DEGRADATION (resets connection pool state)
+        if failure and failure.failure_type in (
+            FailureType.CRASH,
+            FailureType.RESOURCE_LEAK,
+            FailureType.CASCADING_LATENCY,
+            FailureType.DB_DEGRADATION,
+        ):
             delay = self.rng.randint(1, 2)
             self.pending_effects.append(PendingEffect(
                 action_type="restart_service",
@@ -434,9 +440,15 @@ class Simulator:
         max_r = node.max_replicas if node else 8
         target_replicas = max(1, min(target_replicas, max_r))
 
+        failure = self._get_failure_for_service(service_id)
+        # Scaling resolves CASCADING_LATENCY: more capacity drops utilisation below saturation threshold
+        action = "scale_remediate" if (
+            failure and failure.failure_type == FailureType.CASCADING_LATENCY
+        ) else "scale_service"
+
         delay = self.rng.randint(2, 4)
         self.pending_effects.append(PendingEffect(
-            action_type="scale_service",
+            action_type=action,
             target_service=service_id,
             params={"replicas": target_replicas},
             resolve_tick=self.tick + delay,
@@ -502,15 +514,32 @@ class Simulator:
         return record
 
     def _do_rebalance_traffic(self, params: Dict, record: Dict) -> Dict:
-        from_region = params.get("from_region", "")
-        to_region = params.get("to_region", "")
+        # Accept the varied param names models actually send
+        from_region = (
+            params.get("from_region")
+            or params.get("region")
+            or params.get("service_id")
+            or ""
+        )
+        to_region = params.get("to_region") or params.get("target") or ""
         pct = params.get("pct", 50)
+
+        # If only one region given, infer the other from the graph's region list
+        if from_region and not to_region and self.graph:
+            others = [r for r in self.graph.regions if r != from_region]
+            to_region = others[0] if others else ""
+
         record["target"] = f"{from_region}->{to_region}"
 
         if not self.graph or not self.graph.has_multiple_regions:
             record["note"] = "Traffic rebalancing only available in multi-region (hard) mode"
             return record
 
+        if not from_region:
+            record["success"] = False
+            record["note"] = "rebalance_traffic requires 'from_region' (or 'region') param"
+            return record
+
         delay = self.rng.randint(2, 3)
         self.pending_effects.append(PendingEffect(
             action_type="rebalance_traffic",
@@ -578,9 +607,11 @@ class Simulator:
                     "ticks_ago": 0,
                 })
 
-        elif effect.action_type == "scale_service":
+        elif effect.action_type in ("scale_service", "scale_remediate"):
             if svc:
                 svc.replicas = effect.params.get("replicas", svc.replicas)
+            if effect.action_type == "scale_remediate":
+                self._remediate_service(effect.target_service)
 
         elif effect.action_type == "tune_config_fix":
             self._remediate_service(effect.target_service)
@@ -605,10 +636,20 @@ class Simulator:
                     if not s:
                         continue
                     if node.region == from_region:
-                        s.arrival_rate *= (1 - pct)
+                        floor = node.base_arrival_rate * 0.2
+                        s.arrival_rate = max(floor, s.arrival_rate * (1 - pct))
                     elif node.region == to_region:
                         s.arrival_rate *= (1 + pct * 0.5)  # Some traffic absorbed
 
+                # If a CASCADING_LATENCY failure exists in from_region and traffic is
+                # significantly shifted away (>= 40%), the load reduction resolves it
+                if pct >= 0.4:
+                    for spec in self.failures:
+                        if spec.failure_type == FailureType.CASCADING_LATENCY:
+                            node = self.graph.node_map.get(spec.service_id)
+                            if node and node.region == from_region:
+                                self._remediate_service(spec.service_id)
+
     def _remediate_service(self, service_id: str) -> None:
         """Mark a service as remediated — stop failure evolution."""
         self.remediated_services[service_id] = self.tick

uv.lock

@@ -2134,6 +2134,7 @@ version = "1.0.0"
 source = { editable = "." }
 dependencies = [
     { name = "fastapi" },
+    { name = "httpx" },
     { name = "openai" },
     { name = "openenv-core" },
     { name = "pydantic" },
@@ -2155,6 +2156,7 @@ dev = [
 [package.metadata]
 requires-dist = [
     { name = "fastapi", specifier = ">=0.104.0" },
+    { name = "httpx", specifier = ">=0.24.0" },
     { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.24.0" },
     { name = "openai", specifier = ">=1.0.0" },
     { name = "openenv-core", specifier = ">=0.2.2" },