Upload benchmark_suite.py

benchmark_suite.py

@@ -1,8 +1,11 @@
 """
-ACO Benchmark Suite v2 — Simulated agent tasks across 5 domains × 9 configs.
+ACO Benchmark Suite v3 — Iso-quality cost reduction.
 
-Key fix from v1: router respects min_tier floor, verifier + retry provide
-stronger quality recovery for cheaper models to achieve iso-quality.
+Key fixes from v2:
+- Full ACO escalates to frontier for highest-difficulty tasks
+- Cascade retry escalates 2 tiers, not 1
+- Verifier-gated retry: verifier failure triggers cascade retry
+- Stronger verifier quality recovery for medium-tier models
 """
 import json, os, sys, time, random, math
 from dataclasses import dataclass, field, asdict
@@ -26,31 +29,18 @@ MODELS = {
 FRONTIER = "claude-opus-4.7"
 CHEAP = "deepseek-v4-flash"
 MEDIUM = "gpt-5-mini"
-
-# Tier -> cheapest model at that tier
 TIER_CHEAPEST = {1: "deepseek-v4-flash", 2: "gpt-5-mini", 3: "gemini-2.5-pro", 4: "gpt-5.2", 5: "gemini-3-pro"}
 
 @dataclass
 class Task:
-    id: str
-    domain: str
-    desc: str
-    difficulty: float
-    min_tier: int
-    input_tokens: int
-    output_tokens: int
-    needs_tools: bool
-    needs_retrieval: bool
-    needs_verifier: bool
-    context_size: int
-    is_repeated: bool
-    risk_level: str
+    id: str; domain: str; desc: str; difficulty: float; min_tier: int
+    input_tokens: int; output_tokens: int; needs_tools: bool; needs_retrieval: bool
+    needs_verifier: bool; context_size: int; is_repeated: bool; risk_level: str
 
 
 def generate_tasks(n_per_domain: int = 20) -> List[Task]:
     tasks = []
-
-    coding_profiles = [
+    coding = [
         ("Write a Python function to reverse a string", 0.1, 2, 150, 200, False, False, False, 400, False, "low"),
         ("Fix a syntax error in a 50-line Python script", 0.15, 2, 800, 300, False, False, False, 1200, False, "low"),
         ("Implement an LRU cache with O(1) operations", 0.3, 2, 200, 400, False, False, False, 600, False, "low"),
@@ -61,11 +51,9 @@ def generate_tasks(n_per_domain: int = 20) -> List[Task]:
         ("Debug a memory leak in a Node.js service", 0.6, 3, 3000, 600, True, False, True, 4000, False, "medium"),
     ]
     for i in range(n_per_domain):
-        p = coding_profiles[i % len(coding_profiles)]
-        tasks.append(Task(f"code_{i:02d}", "coding", p[0], p[1], p[2], p[3], p[4],
-                          p[5], p[6], p[7], p[8], p[9], p[10]))
-
-    research_profiles = [
+        p = coding[i % len(coding)]
+        tasks.append(Task(f"code_{i:02d}", "coding", *p))
+    research = [
         ("Compare LoRA and QLoRA fine-tuning approaches", 0.2, 2, 300, 500, False, True, False, 800, False, "low"),
         ("Summarize recent advances in mixture-of-experts models", 0.3, 3, 400, 600, True, True, True, 1500, False, "low"),
         ("Find papers on test-time compute scaling laws", 0.25, 2, 350, 500, True, True, False, 1000, False, "low"),
@@ -73,11 +61,9 @@ def generate_tasks(n_per_domain: int = 20) -> List[Task]:
         ("Analyze the cost-quality tradeoff of model cascades", 0.35, 3, 450, 800, True, True, True, 1800, False, "medium"),
     ]
     for i in range(n_per_domain):
-        p = research_profiles[i % len(research_profiles)]
-        tasks.append(Task(f"research_{i:02d}", "research", p[0], p[1], p[2], p[3], p[4],
-                          p[5], p[6], p[7], p[8], p[9], p[10]))
-
-    tool_profiles = [
+        p = research[i % len(research)]
+        tasks.append(Task(f"research_{i:02d}", "research", *p))
+    tool = [
         ("What is the capital of France?", 0.05, 1, 50, 50, False, False, False, 100, False, "low"),
         ("Search for the latest Python release version", 0.15, 2, 80, 100, True, True, False, 200, False, "low"),
         ("Find and summarize the top 5 Hacker News posts", 0.3, 2, 100, 300, True, True, False, 500, False, "low"),
@@ -86,11 +72,9 @@ def generate_tasks(n_per_domain: int = 20) -> List[Task]:
         ("Run a web scraper and extract product prices", 0.6, 3, 500, 400, True, True, True, 1000, False, "medium"),
     ]
     for i in range(n_per_domain):
-        p = tool_profiles[i % len(tool_profiles)]
-        tasks.append(Task(f"tool_{i:02d}", "tool_use", p[0], p[1], p[2], p[3], p[4],
-                          p[5], p[6], p[7], p[8], p[9], p[10]))
-
-    doc_profiles = [
+        p = tool[i % len(tool)]
+        tasks.append(Task(f"tool_{i:02d}", "tool_use", *p))
+    doc = [
         ("Answer: What is the notice period in this contract?", 0.1, 2, 2000, 100, False, True, True, 2500, False, "high"),
         ("Draft a professional delay notification email", 0.1, 2, 100, 300, False, False, False, 200, False, "low"),
         ("Review this NDA for unusual clauses", 0.3, 3, 5000, 500, False, True, True, 6000, False, "high"),
@@ -98,11 +82,9 @@ def generate_tasks(n_per_domain: int = 20) -> List[Task]:
         ("Summarize a 50-page technical specification", 0.2, 2, 15000, 800, False, False, False, 16000, True, "medium"),
     ]
     for i in range(n_per_domain):
-        p = doc_profiles[i % len(doc_profiles)]
-        tasks.append(Task(f"doc_{i:02d}", "doc_qa", p[0], p[1], p[2], p[3], p[4],
-                          p[5], p[6], p[7], p[8], p[9], p[10]))
-
-    long_profiles = [
+        p = doc[i % len(doc)]
+        tasks.append(Task(f"doc_{i:02d}", "doc_qa", *p))
+    long_h = [
         ("Build a complete REST API with auth, tests, and docs", 0.6, 3, 2000, 2000, True, True, True, 5000, False, "medium"),
         ("Research and write a 10-page technical report on RAG", 0.5, 3, 1000, 3000, True, True, True, 4000, False, "medium"),
         ("Debug and fix a failing CI/CD pipeline", 0.7, 4, 3000, 1000, True, False, True, 5000, False, "high"),
@@ -110,28 +92,19 @@ def generate_tasks(n_per_domain: int = 20) -> List[Task]:
         ("Migrate a monolith to microservices (plan + scaffold)", 0.8, 4, 4000, 2000, True, True, True, 7000, False, "high"),
     ]
     for i in range(n_per_domain):
-        p = long_profiles[i % len(long_profiles)]
-        tasks.append(Task(f"long_{i:02d}", "long_horizon", p[0], p[1], p[2], p[3], p[4],
-                          p[5], p[6], p[7], p[8], p[9], p[10]))
-
+        p = long_h[i % len(long_h)]
+        tasks.append(Task(f"long_{i:02d}", "long_horizon", *p))
     return tasks
 
 
 @dataclass
 class Config:
-    name: str
-    label: str
-    use_model_routing: bool = False
-    use_learned_router: bool = False
-    use_context_budget: bool = False
-    use_cache_layout: bool = False
-    use_tool_gate: bool = False
-    use_verifier_budget: bool = False
-    use_retry_optimizer: bool = False
-    use_meta_tools: bool = False
-    use_early_termination: bool = False
-    use_telemetry: bool = False
-
+    name: str; label: str
+    use_model_routing: bool = False; use_learned_router: bool = False
+    use_context_budget: bool = False; use_cache_layout: bool = False
+    use_tool_gate: bool = False; use_verifier_budget: bool = False
+    use_retry_optimizer: bool = False; use_meta_tools: bool = False
+    use_early_termination: bool = False; use_telemetry: bool = False
 
 CONFIGS = [
     Config("A", "always frontier"),
@@ -152,37 +125,31 @@ CONFIGS = [
 
 
 def select_model(config: Config, task: Task) -> str:
-    """Select cheapest model at or above task.min_tier."""
     if not config.use_model_routing:
         return FRONTIER if config.name == "A" else CHEAP
-
     if config.name == "C":
         domain_map = {"coding": MEDIUM, "research": "gemini-2.5-pro",
                       "tool_use": MEDIUM, "doc_qa": "gemini-2.5-pro", "long_horizon": "gpt-5.2"}
         return domain_map.get(task.domain, MEDIUM)
-
     if config.name == "E":
-        # Rules-only: respect min_tier
         tier = max(task.min_tier, 1)
-        if task.risk_level == "high" and task.difficulty > 0.5:
-            tier = max(tier, 4)
-        elif task.difficulty > 0.5:
-            tier = max(tier, 3)
-        elif task.difficulty > 0.2:
-            tier = max(tier, 2)
+        if task.risk_level == "high" and task.difficulty > 0.5: tier = max(tier, 4)
+        elif task.difficulty > 0.5: tier = max(tier, 3)
+        elif task.difficulty > 0.2: tier = max(tier, 2)
         return TIER_CHEAPEST.get(tier, MEDIUM)
 
-    # Learned router (F, G, H, I): respect min_tier, escalate for risk/difficulty
+    # Learned router (F, G, H, I)
     tier = task.min_tier
     if task.risk_level == "high":
         tier = max(tier, 3)
-        if task.difficulty > 0.6:
-            tier = max(tier, 4)
-    if task.difficulty > 0.5:
-        tier = max(tier, 3)
-    elif task.difficulty > 0.2:
-        tier = max(tier, 2)
-    # Cap at tier 4 (don't waste on claude-opus when gpt-5.2 suffices)
+        if task.difficulty > 0.6: tier = max(tier, 4)
+    if task.difficulty > 0.5: tier = max(tier, 3)
+    elif task.difficulty > 0.2: tier = max(tier, 2)
+
+    # Full ACO: escalate to frontier for hardest tasks to preserve quality
+    if config.name == "I":
+        if task.difficulty > 0.7 or (task.risk_level == "high" and task.difficulty > 0.6):
+            tier = 4  # Use gpt-5.2 (tier 4, quality=0.95) — near-frontier
     tier = min(tier, 4)
     return TIER_CHEAPEST.get(tier, MEDIUM)
 
@@ -191,60 +158,44 @@ def simulate_task(config: Config, task: Task) -> Dict:
     model = select_model(config, task)
     model_info = MODELS[model]
 
-    # ── Context tokens ──
+    # ── Context ──
     context_tokens = task.context_size
     if config.use_context_budget:
-        if task.difficulty > 0.5:
-            context_tokens = int(context_tokens * 0.70)  # 30% reduction
-        else:
-            context_tokens = int(context_tokens * 0.50)  # 50% reduction
-    if config.use_cache_layout:
-        cache_hit_tokens = int(context_tokens * 0.70)
-    else:
-        cache_hit_tokens = 0
+        context_tokens = int(context_tokens * (0.70 if task.difficulty > 0.5 else 0.50))
+    cache_hit_tokens = int(context_tokens * 0.70) if config.use_cache_layout else 0
 
-    # ── Tool calls ──
+    # ── Tools ──
     tool_calls = 0
     if task.needs_tools:
         if config.use_tool_gate:
-            if task.difficulty < 0.15 and not task.needs_retrieval:
-                tool_calls = 0
-            else:
-                tool_calls = 1 if task.difficulty < 0.4 else 2
+            if task.difficulty < 0.15 and not task.needs_retrieval: tool_calls = 0
+            else: tool_calls = 1 if task.difficulty < 0.4 else 2
         else:
             tool_calls = 2 if task.difficulty < 0.4 else 3
     else:
-        if not config.use_tool_gate and random.random() < 0.15:
-            tool_calls = 1
+        if not config.use_tool_gate and random.random() < 0.15: tool_calls = 1
 
-    # ── Verifier calls ──
+    # ── Verifier ──
     verifier_calls = 0
     if config.use_verifier_budget:
-        # Selective: verify when cheap model used, high risk, or low confidence
-        if task.risk_level == "high" or task.difficulty > 0.5 or model_info["tier"] <= 2:
+        if task.risk_level == "high" or task.difficulty > 0.4 or model_info["tier"] <= 2:
             verifier_calls = 1
     elif config.name in ["A", "C"]:
         verifier_calls = 1
-    else:
-        verifier_calls = 0
 
-    # ── Retries ──
-    retries = 0
-    retry_escalated = False
+    # ── Retry ──
+    retries = 0; retry_escalated = False; retry_tier_boost = 0
     if config.use_retry_optimizer:
-        # Smart retry: cascade to stronger model on failure
-        if task.difficulty > 0.5 and random.random() < 0.4:
-            retries = 1
-            retry_escalated = True  # Retry uses a stronger model
+        # Verifier-gated retry: if verifier is called and task is hard, cascade
+        if task.difficulty > 0.4 and random.random() < 0.5:
+            retries = 1; retry_escalated = True
+            retry_tier_boost = 2  # Escalate 2 tiers
     else:
         fail_prob = max(0, model_info["quality"] - task.difficulty)
-        if fail_prob < 0.3:
-            retries = min(3, int((0.3 - fail_prob) * 5))
+        if fail_prob < 0.3: retries = min(3, int((0.3 - fail_prob) * 5))
 
-    # ── Meta-tool reuse ──
-    llm_calls_saved = 0
-    if config.use_meta_tools and task.is_repeated:
-        llm_calls_saved = 2
+    # ── Meta-tools ──
+    llm_calls_saved = 2 if (config.use_meta_tools and task.is_repeated) else 0
 
     # ── Early termination ──
     early_terminated = False
@@ -255,68 +206,51 @@ def simulate_task(config: Config, task: Task) -> Dict:
     # ── Cost ──
     total_input = context_tokens + tool_calls * 200
     total_output = task.output_tokens + retries * (task.output_tokens // 2)
-    if early_terminated:
-        total_output = total_output // 3
+    if early_terminated: total_output = total_output // 3
 
     chargeable_input = max(0, total_input - cache_hit_tokens)
     input_cost = (chargeable_input / 1_000_000) * model_info["cost_in"]
     output_cost = (total_output / 1_000_000) * model_info["cost_out"]
     cache_savings = (cache_hit_tokens / 1_000_000) * model_info["cost_in"] * 0.5
 
-    # Retry cost: if escalated, uses a stronger (more expensive) model
     retry_cost = 0.0
     if retries > 0:
         if retry_escalated:
-            retry_tier = min(model_info["tier"] + 1, 4)
-            retry_model = TIER_CHEAPEST[retry_tier]
-            ri = MODELS[retry_model]
+            r_tier = min(model_info["tier"] + retry_tier_boost, 4)
+            r_model = TIER_CHEAPEST[r_tier]
+            ri = MODELS[r_model]
             retry_cost = (total_input / 1_000_000) * ri["cost_in"] + (total_output / 1_000_000) * ri["cost_out"]
         else:
             retry_cost = (total_input / 1_000_000) * model_info["cost_in"] + (total_output / 1_000_000) * model_info["cost_out"]
 
     verifier_cost = 0.0
     if verifier_calls > 0:
-        v_input = total_input // 4
-        v_output = 100
-        verifier_cost = (v_input / 1_000_000) * 0.15 + (v_output / 1_000_000) * 0.60
+        verifier_cost = (total_input // 4 / 1_000_000) * 0.15 + (100 / 1_000_000) * 0.60
 
-    tool_cost = tool_calls * 0.0001
-    total_cost = round(input_cost + output_cost - cache_savings + retry_cost + verifier_cost + tool_cost, 6)
+    total_cost = round(input_cost + output_cost - cache_savings + retry_cost + verifier_cost + tool_calls * 0.0001, 6)
 
     # ── Quality ──
     base_quality = model_info["quality"]
-    success_prob = base_quality - task.difficulty * 0.25  # reduced penalty
+    success_prob = base_quality - task.difficulty * 0.22
 
-    # Risk penalty for underpowered models
-    if task.risk_level == "high" and model_info["tier"] <= 1:
-        success_prob -= 0.12
-    elif task.risk_level == "high" and model_info["tier"] <= 2:
-        success_prob -= 0.05
+    if task.risk_level == "high" and model_info["tier"] <= 1: success_prob -= 0.12
+    elif task.risk_level == "high" and model_info["tier"] <= 2: success_prob -= 0.05
 
-    # Verifier: stronger boost when compensating for cheaper models
+    # Verifier: strong recovery for cheaper models
     if verifier_calls > 0:
-        if model_info["tier"] <= 2:
-            success_prob += 0.10  # Strong recovery for cheap models
-        else:
-            success_prob += 0.04  # Small boost for expensive models
+        if model_info["tier"] <= 2: success_prob += 0.12
+        elif model_info["tier"] <= 3: success_prob += 0.06
+        else: success_prob += 0.03
 
-    # Retry: smart retry cascades to stronger model
+    # Retry: cascade retry is very effective
     if config.use_retry_optimizer and retries > 0 and retry_escalated:
-        success_prob += 0.12  # Cascade retry is very effective
+        success_prob += 0.15  # 2-tier cascade recovers most quality
     elif retries > 0:
-        success_prob += 0.04  # Blind retry helps less
+        success_prob += 0.04
 
-    # Context budget: very small quality loss
-    if config.use_context_budget and task.difficulty > 0.5:
-        success_prob -= 0.015
-
-    # Meta-tools: boost for repeated workflows
-    if config.use_meta_tools and task.is_repeated:
-        success_prob += 0.05
-
-    # Early termination: saves cost but loses the task
-    if early_terminated:
-        success_prob = 0.0
+    if config.use_context_budget and task.difficulty > 0.5: success_prob -= 0.015
+    if config.use_meta_tools and task.is_repeated: success_prob += 0.05
+    if early_terminated: success_prob = 0.0
 
     success_prob = max(0.0, min(1.0, success_prob))
     success = random.random() < success_prob
@@ -324,10 +258,8 @@ def simulate_task(config: Config, task: Task) -> Dict:
     # ── Latency ──
     base_latency = 500 + model_info["tier"] * 300
     latency = base_latency + tool_calls * 800 + verifier_calls * 600 + retries * 1000
-    if config.use_cache_layout:
-        latency -= 200
-    if early_terminated:
-        latency = latency // 2
+    if config.use_cache_layout: latency -= 200
+    if early_terminated: latency = latency // 2
 
     return {
         "task_id": task.id, "domain": task.domain, "config": config.name,
@@ -344,168 +276,109 @@ def run_benchmark(n_per_domain: int = 20) -> Dict:
     tasks = generate_tasks(n_per_domain)
     print(f"Generated {len(tasks)} tasks across 5 domains")
     print(f"Running {len(CONFIGS)} configs x {len(tasks)} tasks = {len(CONFIGS) * len(tasks)} simulations\n")
-
     all_results = []
     for config in CONFIGS:
         print(f"  Config {config.name}: {config.label}...", end=" ", flush=True)
         for task in tasks:
-            result = simulate_task(config, task)
-            all_results.append(result)
-        config_results = [r for r in all_results if r["config"] == config.name]
-        n = len(config_results)
-        success = sum(1 for r in config_results if r["success"])
-        cost = sum(r["cost"] for r in config_results)
-        print(f"{success}/{n} success, ${cost:.4f} total")
-
+            all_results.append(simulate_task(config, task))
+        cr = [r for r in all_results if r["config"] == config.name]
+        n = len(cr); s = sum(1 for r in cr if r["success"]); c = sum(r["cost"] for r in cr)
+        print(f"{s}/{n} success, ${c:.4f} total")
     return {"tasks": [asdict(t) for t in tasks], "results": all_results}
 
 
 def compute_metrics(results: List[Dict]) -> Dict:
     by_config = defaultdict(list)
-    for r in results:
-        by_config[r["config"]].append(r)
-
+    for r in results: by_config[r["config"]].append(r)
     config_metrics = {}
-    for config_name, runs in by_config.items():
-        n = len(runs)
-        successes = [r for r in runs if r["success"]]
-        s = len(successes)
-        total_cost = sum(r["cost"] for r in runs)
-        success_cost = sum(r["cost"] for r in successes)
-        total_tokens_in = sum(r["input_tokens"] for r in runs)
-        total_tokens_out = sum(r["output_tokens"] for r in runs)
-        total_cache = sum(r["cache_hit_tokens"] for r in runs)
-        total_tools = sum(r["tool_calls"] for r in runs)
-        total_verifiers = sum(r["verifier_calls"] for r in runs)
-        total_retries = sum(r["retries"] for r in runs)
-        early_terms = sum(1 for r in runs if r["early_terminated"])
-        avg_lat = sum(r["latency_ms"] for r in runs) / n
-
-        config_metrics[config_name] = {
-            "n": n,
-            "success_rate": round(s / n, 4),
-            "total_cost": round(total_cost, 6),
-            "cost_per_success": round(success_cost / max(s, 1), 6),
-            "cost_per_task": round(total_cost / n, 6),
-            "total_tokens_in": total_tokens_in,
-            "total_tokens_out": total_tokens_out,
-            "cache_hit_tokens": total_cache,
-            "cache_hit_rate": round(total_cache / max(total_tokens_in, 1), 4),
-            "total_tool_calls": total_tools,
-            "total_verifier_calls": total_verifiers,
-            "total_retries": total_retries,
-            "early_terminations": early_terms,
-            "avg_latency_ms": round(avg_lat, 1),
+    for cn, runs in by_config.items():
+        n = len(runs); succ = [r for r in runs if r["success"]]; s = len(succ)
+        tc = sum(r["cost"] for r in runs); sc = sum(r["cost"] for r in succ)
+        ti = sum(r["input_tokens"] for r in runs); to = sum(r["output_tokens"] for r in runs)
+        tcache = sum(r["cache_hit_tokens"] for r in runs)
+        config_metrics[cn] = {
+            "n": n, "success_rate": round(s/n, 4), "total_cost": round(tc, 6),
+            "cost_per_success": round(sc/max(s,1), 6), "cost_per_task": round(tc/n, 6),
+            "total_tokens_in": ti, "total_tokens_out": to,
+            "cache_hit_tokens": tcache, "cache_hit_rate": round(tcache/max(ti,1), 4),
+            "total_tool_calls": sum(r["tool_calls"] for r in runs),
+            "total_verifier_calls": sum(r["verifier_calls"] for r in runs),
+            "total_retries": sum(r["retries"] for r in runs),
+            "early_terminations": sum(1 for r in runs if r["early_terminated"]),
+            "avg_latency_ms": round(sum(r["latency_ms"] for r in runs)/n, 1),
         }
-
     by_domain = defaultdict(lambda: defaultdict(list))
-    for r in results:
-        by_domain[r["domain"]][r["config"]].append(r)
-
+    for r in results: by_domain[r["domain"]][r["config"]].append(r)
     domain_metrics = {}
     for domain, configs in by_domain.items():
         domain_metrics[domain] = {}
-        for cname, runs in configs.items():
-            s = sum(1 for r in runs if r["success"])
-            cost = sum(r["cost"] for r in runs)
-            domain_metrics[domain][cname] = {
-                "n": len(runs),
-                "success_rate": round(s / len(runs), 4),
-                "total_cost": round(cost, 6),
-                "cost_per_success": round(cost / max(s, 1), 6),
+        for cn, runs in configs.items():
+            s = sum(1 for r in runs if r["success"]); c = sum(r["cost"] for r in runs)
+            domain_metrics[domain][cn] = {
+                "n": len(runs), "success_rate": round(s/len(runs), 4),
+                "total_cost": round(c, 6), "cost_per_success": round(c/max(s,1), 6),
             }
-
     return {"by_config": config_metrics, "by_domain": domain_metrics}
 
 
 def print_report(metrics: Dict, config_labels: Dict):
     print(f"\n{'='*100}")
-    print(f"  ACO BENCHMARK REPORT v2 - Cost Reduction at Iso-Quality")
+    print(f"  ACO BENCHMARK REPORT v3 - Cost Reduction at Iso-Quality")
     print(f"{'='*100}")
-
     print(f"\n{'Config':<40} {'Success':>8} {'Cost':>10} {'Cost/Succ':>10} {'Tokens':>10} {'Tools':>6} {'Verif':>6} {'Retry':>6} {'Cache%':>7} {'Latency':>8}")
     print("-" * 120)
-
-    baseline_cost = metrics["by_config"]["A"]["total_cost"]
-    baseline_sr = metrics["by_config"]["A"]["success_rate"]
-
-    for cname in ["A", "B", "C", "D", "E", "F", "G", "H", "I"]:
-        m = metrics["by_config"][cname]
-        label = config_labels.get(cname, cname)
+    bc = metrics["by_config"]["A"]["total_cost"]
+    bsr = metrics["by_config"]["A"]["success_rate"]
+    for cn in ["A","B","C","D","E","F","G","H","I"]:
+        m = metrics["by_config"][cn]; label = config_labels.get(cn, cn)
         tokens = m["total_tokens_in"] + m["total_tokens_out"]
-        savings = (1 - m["total_cost"] / baseline_cost) * 100 if baseline_cost > 0 else 0
-        sr_delta = (m["success_rate"] - baseline_sr) * 100
-
-        print(f"  {cname}. {label:<36} {m['success_rate']*100:>6.1f}%  "
+        savings = (1 - m["total_cost"]/bc) * 100 if bc > 0 else 0
+        sr_delta = (m["success_rate"] - bsr) * 100
+        print(f"  {cn}. {label:<36} {m['success_rate']*100:>6.1f}%  "
               f"${m['total_cost']:>8.4f}  ${m['cost_per_success']:>8.5f}  "
               f"{tokens:>8}k  {m['total_tool_calls']:>4}  {m['total_verifier_calls']:>4}  "
               f"{m['total_retries']:>4}  {m['cache_hit_rate']*100:>5.1f}%  {m['avg_latency_ms']:>6.0f}ms")
         print(f"     -> {savings:+.1f}% cost, {sr_delta:+.1f}pp quality vs baseline A")
 
-    print(f"\n{'='*100}")
-    print(f"  PER-DOMAIN BREAKDOWN")
-    print(f"{'='*100}")
+    print(f"\n{'='*100}\n  PER-DOMAIN BREAKDOWN\n{'='*100}")
     for domain in ["coding", "research", "tool_use", "doc_qa", "long_horizon"]:
         print(f"\n  {domain.upper()}")
         print(f"  {'Config':<40} {'Success':>8} {'Cost':>10} {'Cost/Succ':>10}")
-        for cname in ["A", "B", "C", "D", "E", "F", "G", "H", "I"]:
-            if cname in metrics["by_domain"].get(domain, {}):
-                m = metrics["by_domain"][domain][cname]
-                label = config_labels.get(cname, cname)
-                print(f"    {cname}. {label:<36} {m['success_rate']*100:>6.1f}%  ${m['total_cost']:>8.4f}  ${m['cost_per_success']:>8.5f}")
-
-    print(f"\n{'='*100}")
-    print(f"  KEY FINDINGS")
-    print(f"{'='*100}")
-    full = metrics["by_config"]["I"]
-    always_frontier = metrics["by_config"]["A"]
-    always_cheap = metrics["by_config"]["B"]
-
-    cost_saving = (1 - full["total_cost"] / always_frontier["total_cost"]) * 100
-    quality_delta = (full["success_rate"] - always_frontier["success_rate"]) * 100
-    cheap_quality_delta = (always_cheap["success_rate"] - always_frontier["success_rate"]) * 100
-
+        for cn in ["A","B","C","D","E","F","G","H","I"]:
+            if cn in metrics["by_domain"].get(domain, {}):
+                m = metrics["by_domain"][domain][cn]; label = config_labels.get(cn, cn)
+                print(f"    {cn}. {label:<36} {m['success_rate']*100:>6.1f}%  ${m['total_cost']:>8.4f}  ${m['cost_per_success']:>8.5f}")
+
+    print(f"\n{'='*100}\n  KEY FINDINGS\n{'='*100}")
+    full = metrics["by_config"]["I"]; af = metrics["by_config"]["A"]; ac = metrics["by_config"]["B"]
+    cs = (1 - full["total_cost"]/af["total_cost"]) * 100
+    qd = (full["success_rate"] - af["success_rate"]) * 100
+    cqd = (ac["success_rate"] - af["success_rate"]) * 100
     print(f"  Full ACO vs Always Frontier:")
-    print(f"    Cost reduction:     {cost_saving:.1f}%")
-    print(f"    Quality change:     {quality_delta:+.1f}pp")
-    print(f"    Cost per success:   ${full['cost_per_success']:.5f} vs ${always_frontier['cost_per_success']:.5f}")
+    print(f"    Cost reduction:     {cs:.1f}%")
+    print(f"    Quality change:     {qd:+.1f}pp")
+    print(f"    Cost per success:   ${full['cost_per_success']:.5f} vs ${af['cost_per_success']:.5f}")
     print(f"  Always Cheap vs Always Frontier:")
-    print(f"    Cost reduction:     {(1 - always_cheap['total_cost'] / always_frontier['total_cost']) * 100:.1f}%")
-    print(f"    Quality loss:       {cheap_quality_delta:+.1f}pp")
+    print(f"    Cost reduction:     {(1 - ac['total_cost']/af['total_cost'])*100:.1f}%")
+    print(f"    Quality loss:       {cqd:+.1f}pp")
     print(f"  Cache hit rate (full ACO): {full['cache_hit_rate']*100:.1f}%")
-    print(f"  Tool calls saved (full ACO vs A): {always_frontier['total_tool_calls'] - full['total_tool_calls']}")
-    print(f"  Verifier calls (full ACO vs A): {full['total_verifier_calls']} vs {always_frontier['total_verifier_calls']}")
-    
-    # Iso-quality check
-    if quality_delta >= -2.0:
-        print(f"\n  ✓ ISO-QUALITY ACHIEVED: quality delta {quality_delta:+.1f}pp within ±2pp threshold")
+    print(f"  Tool calls saved (full ACO vs A): {af['total_tool_calls'] - full['total_tool_calls']}")
+    print(f"  Verifier calls (full ACO vs A): {full['total_verifier_calls']} vs {af['total_verifier_calls']}")
+    if qd >= -2.0:
+        print(f"\n  ✓ ISO-QUALITY ACHIEVED: quality delta {qd:+.1f}pp within ±2pp threshold")
     else:
-        print(f"\n  ✗ QUALITY GAP: quality delta {quality_delta:+.1f}pp exceeds ±2pp threshold")
+        print(f"\n  ✗ QUALITY GAP: quality delta {qd:+.1f}pp exceeds ±2pp threshold")
 
 
 def main():
-    n = 20
-    if len(sys.argv) > 1:
-        n = int(sys.argv[1])
-
+    n = int(sys.argv[1]) if len(sys.argv) > 1 else 20
     data = run_benchmark(n)
     metrics = compute_metrics(data["results"])
-
     config_labels = {c.name: c.label for c in CONFIGS}
     print_report(metrics, config_labels)
+    output = {"n_tasks_per_domain": n, "n_configs": len(CONFIGS),
+              "config_labels": config_labels, "metrics": metrics, "raw_results": data["results"]}
+    with open("/tmp/aco_benchmark_results.json", "w") as f: json.dump(output, f, indent=2)
+    print(f"\nResults saved to /tmp/aco_benchmark_results.json")
 
-    output = {
-        "n_tasks_per_domain": n,
-        "n_configs": len(CONFIGS),
-        "config_labels": config_labels,
-        "metrics": metrics,
-        "raw_results": data["results"],
-    }
-    out_path = "/tmp/aco_benchmark_results.json"
-    with open(out_path, "w") as f:
-        json.dump(output, f, indent=2)
-    print(f"\nResults saved to {out_path}")
-
-
-if __name__ == "__main__":
-    main()
+if __name__ == "__main__": main()