Delete eval/benchmark_v9_results.json, eval/bert_vs_xgboost_results.json/*, eval/e2e_standalone_results.json, training/e2e_*.py

training/e2e_benchmark.py

@@ -1,429 +0,0 @@
-"""End-to-End ACO Evaluation on SWE-bench Agent Traces.
-
-Simulates a coding agent running on SWE-bench tasks, with the ACO optimizer
-intercepting every decision point. Uses SWE-Router execution data for
-ground-truth model costs and outcomes.
-
-Measures:
-  - Total cost with optimizer vs without
-  - Per-module cost impact (ablation study)
-  - Which optimizer decisions actually saved money
-  - Which optimizer decisions wasted money
-
-Architecture:
-  For each SWE-bench task:
-    1. ACO predicts task type, difficulty, optimal tier
-    2. ACO routes to a model (cascade)
-    3. Agent "runs" using SWE-Router ground truth (did it resolve? how much did it cost?)
-    4. ACO decides: verify? retry? escalate? stop?
-    5. Measure: actual cost path vs baseline
-
-The "agent" is simulated: we use SWE-Router traces to know what each model
-actually did (cost, api_calls, success). ACO makes the decisions, we look up
-the real outcomes, ACO reacts, etc.
-"""
-import json, sys, os, copy, random, hashlib
-from collections import defaultdict
-from typing import Dict, List, Optional, Tuple
-import numpy as np
-
-# ── Load ACO package ────────────────────────────────────────────
-sys.path.insert(0, "/app")
-from aco.optimizer import ACOOptimizer
-from aco.config import ACOConfig, RoutingPolicy
-from aco.router import RoutingDecision
-from aco.router_v10 import V10Router
-from datasets import load_dataset
-
-REPO = "narcolepticchicken/agent-cost-optimizer"
-
-print("=" * 70)
-print("ACO END-TO-END BENCHMARK ON SWE-BENCH AGENT TRACES")
-print("=" * 70)
-
-# ── 1. Load SWE-Router ground truth ─────────────────────────────
-print("\n[1] Loading SWE-Router execution data (8 models × 500 tasks)...")
-
-MODELS = [
-    "deepseek-v4-flash", "gpt-5-nano", "gpt-5-mini", "deepseek-v3.2",
-    "gemini-2.5-pro", "claude-opus-4.7", "gpt-5.2", "gemini-3-pro",
-]
-
-TIER = {
-    "deepseek-v4-flash": 1, "gpt-5-nano": 1,
-    "gpt-5-mini": 2, "deepseek-v3.2": 2,
-    "gemini-2.5-pro": 3,
-    "claude-opus-4.7": 4, "gpt-5.2": 4,
-    "gemini-3-pro": 5,
-}
-
-TIER_MODEL_MAP = {
-    1: "deepseek-v4-flash", 2: "gpt-5-mini",
-    3: "gemini-2.5-pro", 4: "claude-opus-4.7", 5: "gemini-3-pro",
-}
-
-# Load all SWE-Router traces
-traces = defaultdict(dict)
-for model_name in MODELS:
-    ds = load_dataset(f"SWE-Router/swebench-verified-{model_name}", split="test")
-    for row in ds:
-        iid = row["instance_id"]
-        traces[iid][model_name] = {
-            "resolved": row["resolved"],
-            "cost": float(row["instance_cost"]),
-            "api_calls": int(row["api_calls"]),
-            "problem": row.get("problem_statement", row.get("problem", "")),
-        }
-print(f"  Loaded {len(traces)} tasks × {len(MODELS)} models = {len(traces) * len(MODELS)} outcomes")
-
-# ── 2. Set up ACO optimizer ─────────────────────────────────────
-print("\n[2] Setting up ACO optimizer...")
-
-config = ACOConfig()
-config.routing_policy = RoutingPolicy(
-    routing_mode="cascade",
-    feedback_escalation=True,
-    max_retries=2,
-    max_cost_per_task=1.0,
-    safety_threshold=0.5,
-)
-config.enable_meta_tools = False  # no trace data to mine yet
-config.enable_doom_detector = True
-config.router_model_path = "/tmp/router_models/router_bundle_v10_fixed.pkl"
-
-# Download the router bundle
-from huggingface_hub import hf_hub_download
-try:
-    bundle_path = hf_hub_download(
-        repo_id=REPO, 
-        filename="router_models/router_bundle_v10_fixed.pkl",
-        cache_dir="/tmp/router_models"
-    )
-    config.router_model_path = bundle_path
-    print(f"  Router model: {bundle_path}")
-except Exception as e:
-    print(f"  Router bundle download failed: {e}")
-    config.router_model_path = None
-
-optimizer = ACOOptimizer(config)
-
-# ── 3. Simulate agent runs ──────────────────────────────────────
-print("\n[3] Running end-to-end simulation...")
-
-def simulate_agent_run(task_id: str, task_problem: str, model_name: str,
-                       optimizer: ACOOptimizer) -> Dict:
-    """Simulate one agent run with ACO intercepting decisions."""
-    result = {
-        "task_id": task_id,
-        "problem": task_problem[:200],
-        "steps": [],
-        "total_cost": 0.0,
-        "total_api_calls": 0,
-        "success": False,
-        "routed_model": model_name,
-        "routed_tier": TIER.get(model_name, 4),
-        "escalated": False,
-        "terminated_early": False,
-        "verifier_calls": 0,
-        "tool_gate_skips": 0,
-        "doom_stops": 0,
-    }
-    
-    # Step 1: ACO routes to a model
-    start = optimizer.start_run(task_problem)
-    routed_model = start["routing"]["model_id"]
-    routed_tier = start["routing"]["tier"]
-    
-    result["routed_model"] = routed_model
-    result["routed_tier"] = routed_tier
-    result["aco_start"] = start
-    
-    # Step 2: Simulate agent execution with this model
-    model_trace = traces[task_id].get(routed_model, {})
-    resolved = model_trace.get("resolved", False)
-    cost = model_trace.get("cost", 0.30)
-    api_calls = model_trace.get("api_calls", 10)
-    
-    # Simulate 3 agent steps (model call + tool calls per step)
-    step_names = ["analyze_problem", "generate_patch", "verify_patch"]
-    for i, step_name in enumerate(step_names):
-        step_cost = cost / len(step_names)
-        step_api = max(1, api_calls // len(step_names))
-        
-        # ACO: gate tools (simulate 2-3 tool calls per step)
-        tool_decisions = []
-        for tool_idx in range(random.randint(2, 4)):
-            td = optimizer.gate_tool(f"tool_{tool_idx}", {"step": step_name})
-            tool_decisions.append(td)
-            if td.action == "skip":
-                result["tool_gate_skips"] += 1
-        
-        # ACO: should we verify?
-        is_final = (i == len(step_names) - 1)
-        vd = optimizer.should_verify(
-            is_irreversible=is_final,
-            has_prior_failures=(not resolved and i > 0),
-        )
-        if vd.should_verify:
-            result["verifier_calls"] += 1
-            # Verifier adds ~20% to cost
-            step_cost *= 1.2
-        
-        # ACO: check for doom
-        total_so_far = result["total_cost"] + step_cost
-        doom = optimizer.check_doom(current_cost=total_so_far)
-        
-        if doom.doomed:
-            result["terminated_early"] = True
-            result["doom_stops"] += 1
-            result["doom_reason"] = doom.reasoning
-            break
-        
-        # Record step
-        optimizer.record_step(
-            model_call={
-                "model_id": routed_model,
-                "input_tokens": 5000 // len(step_names),
-                "output_tokens": 500,
-                "cost": step_cost,
-                "latency_ms": 2000,
-            },
-            tool_calls=[{
-                "tool_name": td.tool_name,
-                "args": td.args,
-                "success": td.action != "skip",
-                "cost": td.estimated_cost,
-                "latency_ms": td.estimated_latency,
-            } for td in tool_decisions],
-            context_size=5000 + (i * 1000),
-            retry_num=0,
-        )
-        
-        result["total_cost"] += step_cost
-        result["total_api_calls"] += step_api
-        result["steps"].append(step_name)
-    
-    # If escalated, try next tier
-    if not resolved and not result["terminated_early"]:
-        for escalate_tier in range(routed_tier + 1, 6):
-            escalate_model = TIER_MODEL_MAP[escalate_tier]
-            esc_trace = traces[task_id].get(escalate_model, {})
-            if esc_trace.get("resolved", False):
-                result["total_cost"] += esc_trace.get("cost", 0.30)
-                result["escalated"] = True
-                result["final_model_used"] = escalate_model
-                resolved = True
-                break
-            else:
-                # Escalation failed, add cost and continue
-                result["total_cost"] += esc_trace.get("cost", 0.30)
-    
-    result["success"] = resolved
-    
-    # End run
-    outcome = "completed" if resolved else ("terminated" if result["terminated_early"] else "abandoned")
-    trace = optimizer.end_run(
-        success=resolved,
-        outcome=outcome,
-        artifacts=["patch.diff"] if resolved else [],
-        failure_tags=[] if resolved else ["not_resolved"],
-    )
-    
-    return result
-
-
-def compute_baseline_cost(task_id: str) -> Dict:
-    """Get baseline metrics for a task."""
-    best = None
-    for model_name, mt in traces[task_id].items():
-        # Frontier: claude-opus-4.7
-        if model_name == "claude-opus-4.7":
-            frontier = mt
-        # Cheapest that solves it
-        if mt["resolved"]:
-            if best is None or mt["cost"] < best["cost"]:
-                best = mt
-    
-    return {
-        "frontier_cost": traces[task_id].get("claude-opus-4.7", {}).get("cost", 0.317),
-        "frontier_resolved": traces[task_id].get("claude-opus-4.7", {}).get("resolved", False),
-        "cheapest_success_cost": best["cost"] if best else None,
-        "cheapest_resolved": best is not None,
-        "always_cheap_cost": traces[task_id].get("deepseek-v4-flash", {}).get("cost", 0.014),
-        "always_cheap_resolved": traces[task_id].get("deepseek-v4-flash", {}).get("resolved", False),
-    }
-
-# Run on all 500 tasks
-all_results = []
-for idx, (task_id, task_traces) in enumerate(traces.items()):
-    problem = next(iter(task_traces.values()))["problem"]
-    result = simulate_agent_run(task_id, problem, "deepseek-v4-flash", optimizer)
-    baseline = compute_baseline_cost(task_id)
-    result["baseline"] = baseline
-    all_results.append(result)
-    
-    if (idx + 1) % 100 == 0:
-        n = idx + 1
-        resolved = sum(1 for r in all_results if r["success"])
-        total_cost = sum(r["total_cost"] for r in all_results)
-        print(f"  [{n}/500] resolved={resolved} ({resolved/n*100:.1f}%), "
-              f"avg_cost=${total_cost/n:.4f}")
-
-# ── 4. Compute metrics ──────────────────────────────────────────
-print(f"\n[4] Computing metrics...")
-
-n = len(all_results)
-resolved = sum(1 for r in all_results if r["success"])
-total_cost = sum(r["total_cost"] for r in all_results)
-baseline_frontier_cost = sum(r["baseline"]["frontier_cost"] for r in all_results)
-baseline_cheap_cost = sum(r["baseline"]["always_cheap_cost"] for r in all_results)
-escalated = sum(1 for r in all_results if r["escalated"])
-terminated = sum(1 for r in all_results if r["terminated_early"])
-verifier_calls = sum(r["verifier_calls"] for r in all_results)
-tool_skips = sum(r["tool_gate_skips"] for r in all_results)
-
-# Baseline metrics
-frontier_resolved = sum(1 for r in all_results if r["baseline"]["frontier_resolved"])
-cheap_resolved = sum(1 for r in all_results if r["baseline"]["always_cheap_resolved"])
-
-print(f"""
-{'='*70}
-ACO END-TO-END RESULTS (500 SWE-bench tasks)
-{'='*70}
-
-TASKS: {n}
-
-{'Policy':<25} {'Resolved':>10} {'Rate':>10} {'TotalCost':>12} {'AvgCost':>10} {'CostRed':>10}
-{'-'*77}
-{'ACO Optimized':<25} {resolved:>10} {resolved/n*100:>9.1f}% ${total_cost:>10.2f} ${total_cost/n:>9.4f} {(1-total_cost/baseline_frontier_cost)*100:>9.1f}%
-{'Frontier (always)':<25} {frontier_resolved:>10} {frontier_resolved/n*100:>9.1f}% ${baseline_frontier_cost:>10.2f} ${baseline_frontier_cost/n:>9.4f} {'--':>10}
-{'Always Cheap':<25} {cheap_resolved:>10} {cheap_resolved/n*100:>9.1f}% ${baseline_cheap_cost:>10.2f} ${baseline_cheap_cost/n:>9.4f} {(1-baseline_cheap_cost/baseline_frontier_cost)*100:>9.1f}%
-
-PER-MODULE IMPACT:
-  Escalations triggered: {escalated} ({escalated/n*100:.1f}%)
-  Early terminations:    {terminated} ({terminated/n*100:.1f}%)
-  Verifier calls:        {verifier_calls} ({verifier_calls/n*100:.1f} avg/task)
-  Tool gate skips:       {tool_skips} ({tool_skips/n*100:.1f} avg/task)
-
-COST BREAKDOWN:
-  Avg cost per run:     ${total_cost/n:.4f}
-  Avg cost per resolved: ${total_cost/max(resolved,1):.4f}
-  Waste on failed runs: ${sum(r['total_cost'] for r in all_results if not r['success']):.2f}
-""")
-
-# ── 5. Ablation study ───────────────────────────────────────────
-print(f"[5] Running ablation study...")
-
-def run_ablation(disable_modules: List[str]) -> Dict:
-    """Run with specific modules disabled."""
-    ab_config = ACOConfig()
-    ab_config.routing_policy = RoutingPolicy(
-        routing_mode="cascade" if "router" not in disable_modules else "frontier",
-        feedback_escalation="feedback_escalation" not in disable_modules,
-        max_retries=2,
-        max_cost_per_task=1.0,
-        safety_threshold=0.5,
-    )
-    ab_config.enable_meta_tools = "meta_tools" not in disable_modules
-    ab_config.enable_doom_detector = "doom_detector" not in disable_modules
-    ab_config.router_model_path = config.router_model_path
-    
-    ab_opt = ACOOptimizer(ab_config)
-    
-    results = []
-    for task_id, task_traces in list(traces.items())[:100]:  # Use 100 for ablation
-        problem = next(iter(task_traces.values()))["problem"]
-        result = simulate_agent_run(task_id, problem, "deepseek-v4-flash", ab_opt)
-        # Override with ablation behavior
-        if "verifier" in disable_modules:
-            result["verifier_calls"] = 0
-        if "tool_gate" in disable_modules:
-            result["tool_gate_skips"] = 0
-        if "context" in disable_modules:
-            pass  # mock agent doesn't use context
-        results.append(result)
-    
-    return {
-        "modules_disabled": disable_modules,
-        "n": len(results),
-        "resolved": sum(1 for r in results if r["success"]),
-        "total_cost": sum(r["total_cost"] for r in results),
-        "avg_cost": sum(r["total_cost"] for r in results) / max(len(results), 1),
-        "verifier_calls": sum(r["verifier_calls"] for r in results),
-        "escalated": sum(1 for r in results if r["escalated"]),
-        "terminated": sum(1 for r in results if r["terminated_early"]),
-    }
-
-ablations = {
-    "full_aco": run_ablation([]),
-    "no_router": run_ablation(["router", "feedback_escalation"]),
-    "no_feedback": run_ablation(["feedback_escalation"]),
-    "no_verifier": run_ablation(["verifier"]),
-    "no_doom_detector": run_ablation(["doom_detector"]),
-    "no_tool_gate": run_ablation(["tool_gate"]),
-    "frontier_only": run_ablation(["router", "feedback_escalation", "verifier", "doom_detector", "tool_gate"]),
-}
-
-print(f"\n{'='*70}")
-print("ABLATION STUDY (100 tasks)")
-print(f"{'='*70}")
-print(f"\n{'Configuration':<25} {'Resolved':>10} {'Rate':>8} {'AvgCost':>10} {'vsFull':>10}")
-print("-" * 63)
-
-full_cost = ablations["full_aco"]["avg_cost"]
-full_resolved = ablations["full_aco"]["resolved"]
-
-for name, ab in sorted(ablations.items(), key=lambda x: x[1]["avg_cost"]):
-    delta_cost = (ab["avg_cost"] - full_cost) / max(full_cost, 0.0001) * 100
-    delta_resolved = ab["resolved"] - full_resolved
-    flag = ""
-    if delta_cost > 0 and ab["resolved"] < full_resolved:
-        flag = " ← REGRESSION"
-    elif delta_cost < 0 and ab["resolved"] >= full_resolved:
-        flag = " ← IMPROVEMENT"
-    print(f"  {name:<23} {ab['resolved']:>10} {ab['resolved']/10:>7.1f}% ${ab['avg_cost']:>9.4f} {delta_cost:>+9.1f}%{flag}")
-
-# ── 6. Save results ─────────────────────────────────────────────
-output = {
-    "end_to_end": {
-        "n": n,
-        "resolved": resolved,
-        "success_rate": round(resolved / n, 4),
-        "total_cost": round(total_cost, 2),
-        "avg_cost": round(total_cost / n, 4),
-        "avg_cost_per_resolved": round(total_cost / max(resolved, 1), 4),
-        "cost_reduction_vs_frontier": round((1 - total_cost / baseline_frontier_cost) * 100, 1),
-        "frontier_cost": round(baseline_frontier_cost, 2),
-        "frontier_resolved": frontier_resolved,
-        "escalations": escalated,
-        "early_terminations": terminated,
-        "verifier_calls": verifier_calls,
-        "tool_gate_skips": tool_skips,
-    },
-    "ablations": {
-        name: {
-            "resolved": a["resolved"],
-            "avg_cost": round(a["avg_cost"], 4),
-            "cost_delta_vs_full": round((a["avg_cost"] - full_cost) / max(full_cost, 0.0001) * 100, 1),
-        }
-        for name, a in ablations.items()
-    },
-}
-
-with open("/tmp/aco_e2e_results.json", "w") as f:
-    json.dump(output, f, indent=2)
-
-# Upload
-from huggingface_hub import HfApi
-api = HfApi()
-api.upload_file(
-    path_or_fileobj="/tmp/aco_e2e_results.json",
-    path_in_repo="eval/e2e_benchmark_results.json",
-    repo_id=REPO,
-    repo_type="model",
-)
-
-print(f"\n  ✓ Results uploaded to eval/e2e_benchmark_results.json")
-print(f"\n{'='*70}")
-print("E2E BENCHMARK COMPLETE")
-print("=" * 70)

training/e2e_benchmark_v2.py

@@ -1,355 +0,0 @@
-"""End-to-End ACO Evaluation on SWE-bench Agent Traces.
-
-FIXED: Clones the ACO repo first for imports.
-"""
-import json, sys, os, copy, random, subprocess
-from collections import defaultdict
-from typing import Dict, List, Optional, Tuple
-import numpy as np
-
-# Clone repo for ACO imports
-repo_url = "https://huggingface.co/narcolepticchicken/agent-cost-optimizer"
-if not os.path.exists("/tmp/aco_repo"):
-    print(f"Cloning {repo_url}...")
-    subprocess.run(["git", "clone", repo_url, "/tmp/aco_repo"], check=True)
-    subprocess.run(["pip", "install", "pyyaml"], check=True, capture_output=True)
-
-sys.path.insert(0, "/tmp/aco_repo")
-from aco.optimizer import ACOOptimizer
-from aco.config import ACOConfig, RoutingPolicy
-from datasets import load_dataset
-
-REPO = "narcolepticchicken/agent-cost-optimizer"
-
-print("=" * 70)
-print("ACO END-TO-END BENCHMARK ON SWE-BENCH AGENT TRACES (V2)")
-print("=" * 70)
-
-# ── 1. Load SWE-Router ground truth ─────────────────────────────
-print("\n[1] Loading SWE-Router execution data...")
-
-MODELS = [
-    "deepseek-v4-flash", "gpt-5-nano", "gpt-5-mini", "deepseek-v3.2",
-    "gemini-2.5-pro", "claude-opus-4.7", "gpt-5.2", "gemini-3-pro",
-]
-
-TIER = {
-    "deepseek-v4-flash": 1, "gpt-5-nano": 1,
-    "gpt-5-mini": 2, "deepseek-v3.2": 2,
-    "gemini-2.5-pro": 3,
-    "claude-opus-4.7": 4, "gpt-5.2": 4,
-    "gemini-3-pro": 5,
-}
-
-TIER_MODEL_MAP = {
-    1: "deepseek-v4-flash", 2: "gpt-5-mini",
-    3: "gemini-2.5-pro", 4: "claude-opus-4.7", 5: "gemini-3-pro",
-}
-
-# Load all SWE-Router traces
-traces = defaultdict(dict)
-for model_name in MODELS:
-    ds = load_dataset(f"SWE-Router/swebench-verified-{model_name}", split="test")
-    for row in ds:
-        iid = row["instance_id"]
-        traces[iid][model_name] = {
-            "resolved": row["resolved"],
-            "cost": float(row["instance_cost"]),
-            "api_calls": int(row["api_calls"]),
-            "problem": row.get("problem_statement", row.get("problem", "")),
-        }
-print(f"  Loaded {len(traces)} tasks × {len(MODELS)} models")
-
-# ── 2. Set up ACO optimizer ─────────────────────────────────────
-print("\n[2] Setting up ACO optimizer...")
-
-config = ACOConfig()
-config.routing_policy = RoutingPolicy(
-    routing_mode="cascade",
-    feedback_escalation=True,
-    max_retries=2,
-    max_cost_per_task=1.0,
-    safety_threshold=0.5,
-)
-config.enable_meta_tools = False
-config.enable_doom_detector = True
-
-# Download router bundle
-from huggingface_hub import hf_hub_download
-try:
-    bundle_path = hf_hub_download(
-        repo_id=REPO,
-        filename="router_models/router_bundle_v10_fixed.pkl",
-    )
-    config.router_model_path = bundle_path
-    print(f"  Router model loaded")
-except Exception as e:
-    print(f"  Router not available: {e}")
-    config.router_model_path = None
-
-optimizer = ACOOptimizer(config)
-
-# ── 3. Simulate agent runs ──────────────────────────────────────
-print("\n[3] Running end-to-end simulation...")
-
-def simulate_agent_run(task_id: str, task_problem: str,
-                       optimizer: ACOOptimizer) -> Dict:
-    """Simulate one agent run with ACO intercepting decisions."""
-    result = {
-        "task_id": task_id,
-        "total_cost": 0.0,
-        "success": False,
-        "escalated": False,
-        "terminated_early": False,
-        "verifier_calls": 0,
-        "tool_gate_skips": 0,
-        "doom_stops": 0,
-    }
-    
-    # Step 1: ACO routes to a model
-    start = optimizer.start_run(task_problem)
-    routed_model = start["routing"]["model_id"]
-    routed_tier = start["routing"]["tier"]
-    
-    result["routed_model"] = routed_model
-    result["routed_tier"] = routed_tier
-    
-    # Step 2: Simulate agent execution
-    model_trace = traces[task_id].get(routed_model, {})
-    resolved = model_trace.get("resolved", False)
-    cost = model_trace.get("cost", 0.30)
-    api_calls = model_trace.get("api_calls", 10)
-    
-    step_names = ["analyze", "generate", "verify"]
-    n_steps = min(len(step_names), api_calls)
-    
-    for i, step_name in enumerate(step_names[:n_steps]):
-        step_cost = cost / max(n_steps, 1)
-        
-        # Gate tools
-        for tool_idx in range(random.randint(1, 3)):
-            td = optimizer.gate_tool(f"tool_{tool_idx}", {"step": step_name})
-            if td.action == "skip":
-                result["tool_gate_skips"] += 1
-        
-        # Should we verify?
-        is_final = (i == n_steps - 1)
-        vd = optimizer.should_verify(
-            is_irreversible=is_final,
-            has_prior_failures=(not resolved and i > 0),
-        )
-        if vd.should_verify:
-            result["verifier_calls"] += 1
-            step_cost *= 1.2
-        
-        # Doom check
-        total_so_far = result["total_cost"] + step_cost
-        doom = optimizer.check_doom(current_cost=total_so_far)
-        if doom.doomed:
-            result["terminated_early"] = True
-            result["doom_stops"] += 1
-            break
-        
-        # Record step
-        optimizer.record_step(
-            model_call={
-                "model_id": routed_model,
-                "input_tokens": 5000 // max(n_steps, 1),
-                "output_tokens": 500,
-                "cost": step_cost,
-                "latency_ms": 2000,
-            },
-            context_size=5000 + (i * 1000),
-            retry_num=0,
-        )
-        
-        result["total_cost"] += step_cost
-    
-    # Escalate if needed
-    if not resolved and not result["terminated_early"]:
-        for escalate_tier in range(routed_tier + 1, 6):
-            escalate_model = TIER_MODEL_MAP[escalate_tier]
-            esc_trace = traces[task_id].get(escalate_model, {})
-            if esc_trace.get("resolved", False):
-                result["total_cost"] += esc_trace.get("cost", 0.30)
-                result["escalated"] = True
-                resolved = True
-                break
-            else:
-                result["total_cost"] += esc_trace.get("cost", 0.30)
-
-    result["success"] = resolved
-    
-    optimizer.end_run(
-        success=resolved,
-        outcome="completed" if resolved else "abandoned",
-        artifacts=["fix"] if resolved else [],
-        failure_tags=[] if resolved else ["not_resolved"],
-    )
-    
-    return result
-
-def compute_baseline(task_id: str) -> Dict:
-    ft = traces[task_id].get("claude-opus-4.7", {})
-    ch = traces[task_id].get("deepseek-v4-flash", {})
-    best = None
-    for m, mt in traces[task_id].items():
-        if mt["resolved"] and (best is None or mt["cost"] < best["cost"]):
-            best = mt
-    return {
-        "frontier_cost": ft.get("cost", 0.317),
-        "frontier_resolved": ft.get("resolved", False),
-        "always_cheap_cost": ch.get("cost", 0.014),
-        "always_cheap_resolved": ch.get("resolved", False),
-        "oracle_cost": best["cost"] if best else ft.get("cost", 0.317),
-        "oracle_resolved": best is not None,
-    }
-
-# Run on all 500 tasks
-all_results = []
-for idx, (task_id, task_traces) in enumerate(traces.items()):
-    problem = next(iter(task_traces.values()))["problem"]
-    result = simulate_agent_run(task_id, problem, optimizer)
-    baseline = compute_baseline(task_id)
-    result["baseline"] = baseline
-    all_results.append(result)
-    
-    if (idx + 1) % 100 == 0:
-        n = idx + 1
-        resolved = sum(1 for r in all_results if r["success"])
-        total_cost = sum(r["total_cost"] for r in all_results)
-        print(f"  [{n}/500] resolved={resolved} ({resolved/n*100:.1f}%), "
-              f"avg_cost=${total_cost/n:.4f}")
-
-# ── 4. Compute metrics ──────────────────────────────────────────
-print(f"\n[4] Computing metrics...")
-
-n = len(all_results)
-resolved = sum(1 for r in all_results if r["success"])
-total_cost = sum(r["total_cost"] for r in all_results)
-baseline_frontier_cost = sum(r["baseline"]["frontier_cost"] for r in all_results)
-baseline_cheap_cost = sum(r["baseline"]["always_cheap_cost"] for r in all_results)
-baseline_oracle_cost = sum(r["baseline"]["oracle_cost"] for r in all_results)
-frontier_resolved = sum(1 for r in all_results if r["baseline"]["frontier_resolved"])
-cheap_resolved = sum(1 for r in all_results if r["baseline"]["always_cheap_resolved"])
-oracle_resolved = sum(1 for r in all_results if r["baseline"]["oracle_resolved"])
-escalated = sum(1 for r in all_results if r["escalated"])
-terminated = sum(1 for r in all_results if r["terminated_early"])
-verifier_calls = sum(r["verifier_calls"] for r in all_results)
-
-print(f"""
-{'='*70}
-ACO END-TO-END RESULTS ({n} SWE-bench tasks)
-{'='*70}
-
-{'Policy':<25} {'Resolved':>10} {'Rate':>10} {'TotalCost':>12} {'AvgCost':>10} {'vsFront':>10}
-{'-'*77}
-{'ACO Optimized':<25} {resolved:>10} {resolved/n*100:>9.1f}% ${total_cost:>10.2f} ${total_cost/n:>9.4f} {(1-total_cost/baseline_frontier_cost)*100:>9.1f}%
-{'Frontier (always)':<25} {frontier_resolved:>10} {frontier_resolved/n*100:>9.1f}% ${baseline_frontier_cost:>10.2f} ${baseline_frontier_cost/n:>9.4f} {'--':>10}
-{'Always Cheap':<25} {cheap_resolved:>10} {cheap_resolved/n*100:>9.1f}% ${baseline_cheap_cost:>10.2f} ${baseline_cheap_cost/n:>9.4f} {(1-baseline_cheap_cost/baseline_frontier_cost)*100:>9.1f}%
-{'Oracle':<25} {oracle_resolved:>10} {oracle_resolved/n*100:>9.1f}% ${baseline_oracle_cost:>10.2f} ${baseline_oracle_cost/n:>9.4f} {(1-baseline_oracle_cost/baseline_frontier_cost)*100:>9.1f}%
-
-MODULE IMPACT:
-  Escalations: {escalated} ({escalated/n*100:.1f}%)
-  Early stops: {terminated} ({terminated/n*100:.1f}%)
-  Verifier calls: {verifier_calls} ({verifier_calls/max(n,1):.2f}/task)
-  Avg cost/resolved: ${total_cost/max(resolved,1):.4f}
-  Waste on failures: ${sum(r['total_cost'] for r in all_results if not r['success']):.2f}
-""")
-
-# ── 5. Ablation study (100 tasks) ───────────────────────────────
-print(f"\n[5] Ablation study (100 tasks)...")
-
-def run_ablation(disable_modules: List[str], n_tasks: int = 100) -> Dict:
-    ab_config = ACOConfig()
-    ab_config.routing_policy = RoutingPolicy(
-        routing_mode="cascade" if "router" not in disable_modules else "frontier",
-        feedback_escalation="feedback" not in disable_modules,
-        max_retries=2, max_cost_per_task=1.0, safety_threshold=0.5,
-    )
-    ab_config.enable_doom_detector = "doom" not in disable_modules
-    ab_config.router_model_path = config.router_model_path
-    
-    ab_opt = ACOOptimizer(ab_config)
-    results = []
-    task_items = list(traces.items())[:n_tasks]
-    
-    for task_id, task_traces in task_items:
-        problem = next(iter(task_traces.values()))["problem"]
-        result = simulate_agent_run(task_id, problem, ab_opt)
-        if "verifier" in disable_modules:
-            result["verifier_calls"] = 0
-        if "tool_gate" in disable_modules:
-            result["tool_gate_skips"] = 0
-        results.append(result)
-    
-    return {
-        "n": len(results),
-        "resolved": sum(1 for r in results if r["success"]),
-        "total_cost": sum(r["total_cost"] for r in results),
-        "avg_cost": sum(r["total_cost"] for r in results) / max(len(results), 1),
-    }
-
-ablations = {
-    "full_aco": run_ablation([]),
-    "no_router": run_ablation(["router", "feedback"]),
-    "no_feedback": run_ablation(["feedback"]),
-    "no_verifier": run_ablation(["verifier"]),
-    "no_doom": run_ablation(["doom"]),
-    "no_tool_gate": run_ablation(["tool_gate"]),
-    "frontier_only": run_ablation(["router", "feedback", "verifier", "doom", "tool_gate"]),
-}
-
-full_cost = ablations["full_aco"]["avg_cost"]
-full_resolved = ablations["full_aco"]["resolved"]
-
-print(f"\n{'Configuration':<25} {'Resolved':>10} {'Rate':>8} {'AvgCost':>10} {'vsFull':>10}")
-print("-" * 63)
-
-for name in ["full_aco", "no_feedback", "no_doom", "no_tool_gate", "no_verifier", "no_router", "frontier_only"]:
-    ab = ablations[name]
-    dc = (ab["avg_cost"] - full_cost) / max(full_cost, 0.0001) * 100
-    dr = ab["resolved"] - full_resolved
-    flag = ""
-    if dc > 0 and dr < 0: flag = " ← REGRESSION"
-    elif dc < 0 and dr >= 0: flag = " ← BETTER"
-    print(f"  {name:<23} {ab['resolved']:>10} {ab['resolved']/ab['n']*100:>7.1f}% ${ab['avg_cost']:>9.4f} {dc:>+9.1f}%{flag}")
-
-# ── 6. Save ─────────────────────────────────────────────────────
-output = {
-    "end_to_end": {
-        "n": n, "resolved": resolved,
-        "success_rate": round(resolved/n, 4),
-        "total_cost": round(total_cost, 2),
-        "avg_cost": round(total_cost/n, 4),
-        "cost_reduction_vs_frontier": round((1 - total_cost/baseline_frontier_cost) * 100, 1),
-        "frontier_cost": round(baseline_frontier_cost, 2),
-        "frontier_resolved": frontier_resolved,
-        "oracle_cost": round(baseline_oracle_cost, 2),
-        "oracle_resolved": oracle_resolved,
-        "escalations": escalated,
-        "early_terminations": terminated,
-        "verifier_calls": verifier_calls,
-    },
-    "ablations": {
-        name: {"resolved": a["resolved"], "avg_cost": round(a["avg_cost"], 4),
-               "cost_delta_vs_full": round((a["avg_cost"] - full_cost) / max(full_cost, 0.0001) * 100, 1)}
-        for name, a in ablations.items()
-    },
-}
-
-with open("/tmp/aco_e2e_v2.json", "w") as f:
-    json.dump(output, f, indent=2)
-
-from huggingface_hub import HfApi
-api = HfApi()
-api.upload_file(
-    path_or_fileobj="/tmp/aco_e2e_v2.json",
-    path_in_repo="eval/e2e_benchmark_v2.json",
-    repo_id=REPO, repo_type="model",
-)
-
-print(f"\n✓ Results uploaded: eval/e2e_benchmark_v2.json")
-print(f"\n{'='*70}")
-print("E2E BENCHMARK V2 COMPLETE")
-print("=" * 70)

training/e2e_standalone.py

@@ -1,397 +0,0 @@
-"""Standalone ACO E2E Benchmark — no external aco/ imports.
-
-Evaluates the Agent Cost Optimizer's model cascade routing against simulated
-SWE-bench agent runs using SWE-Router execution traces as ground truth.
-
-Measures per-module impact through ablation: disable each module one at a time.
-"""
-import json, sys, os, random, pickle
-from collections import defaultdict
-from typing import Dict, List, Optional
-import numpy as np
-
-# ── Core Router (inlined from aco/router_v10.py) ─────────────────
-CODE_KW = ["python","javascript","code","function","bug","debug","refactor",
-    "implement","test","compile","runtime","segfault","thread","async","class",
-    "module","import","error","traceback"]
-CRITICAL_KW = ["critical","production","urgent","emergency","live","deployed","safety","security"]
-SIMPLE_KW = ["typo","simple","quick","brief","minor","small","easy","trivial","just"]
-RESEARCH_KW = ["research","investigate","compare","analyze","survey","paper"]
-TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
-LONG_KW = ["plan","project","roadmap","orchestrate","migrate","pipeline","deploy","architecture"]
-
-FEAT_KEYS = sorted([
-    'req_len','num_words','has_code','n_code','has_legal','has_research',
-    'has_tool','has_critical','has_simple','has_long','has_math',
-    'has_error_msg','has_file_path','n_lines','has_version','has_add',
-    'has_fix','has_change','has_remove','has_test','has_doc',
-    'has_see_also','has_steps_to_reproduce',
-])
-
-TIER_TO_MODEL = {
-    1: 'deepseek-v4-flash', 2: 'gpt-5-mini',
-    3: 'gemini-2.5-pro', 4: 'claude-opus-4.7', 5: 'gemini-3-pro',
-}
-
-TIER_COST = {1:0.01, 2:0.05, 3:0.15, 4:0.30, 5:0.50}
-
-MODELS = ["deepseek-v4-flash","gpt-5-nano","gpt-5-mini","deepseek-v3.2",
-           "gemini-2.5-pro","claude-opus-4.7","gpt-5.2","gemini-3-pro"]
-MODEL_TIER = {"deepseek-v4-flash":1,"gpt-5-nano":1,"gpt-5-mini":2,"deepseek-v3.2":2,
-              "gemini-2.5-pro":3,"claude-opus-4.7":4,"gpt-5.2":4,"gemini-3-pro":5}
-
-def extract_features(text: str) -> np.ndarray:
-    r = text.lower()
-    feats = {
-        'req_len': len(text), 'num_words': len(text.split()),
-        'has_code': int(any(k in r for k in CODE_KW)),
-        'n_code': sum(1 for k in CODE_KW if k in r),
-        'has_legal': int(any(k in r for k in ["contract","legal","compliance"])),
-        'has_research': int(any(k in r for k in RESEARCH_KW)),
-        'has_tool': int(any(k in r for k in TOOL_KW)),
-        'has_critical': int(any(k in r for k in CRITICAL_KW)),
-        'has_simple': int(any(k in r for k in SIMPLE_KW)),
-        'has_long': int(any(k in r for k in LONG_KW)),
-        'has_math': int(any(k in r for k in ["calculate","compute","solve","equation"])),
-        'has_error_msg': int('error' in r or 'traceback' in r or 'exception' in r),
-        'has_file_path': int('/' in r),
-        'n_lines': text.count('\n') + 1,
-        'has_version': int('version' in r or 'update' in r),
-        'has_add': int('add' in r or 'new' in r or 'create' in r),
-        'has_fix': int('fix' in r or 'bug' in r or 'issue' in r),
-        'has_change': int('change' in r or 'modify' in r),
-        'has_remove': int('remove' in r or 'delete' in r),
-        'has_test': int('test' in r or 'spec' in r),
-        'has_doc': int('doc' in r or 'readme' in r),
-        'has_see_also': int('see also' in r or 'related' in r),
-        'has_steps_to_reproduce': int('reproduce' in r or 'steps' in r),
-    }
-    return np.array([float(feats.get(k,0.0)) for k in FEAT_KEYS], dtype=np.float32)
-
-
-# ── Load Router Bundle ──────────────────────────────────────────
-from huggingface_hub import hf_hub_download
-from sklearn.isotonic import IsotonicRegression
-
-print("Loading router model...")
-try:
-    bundle_path = hf_hub_download(
-        repo_id="narcolepticchicken/agent-cost-optimizer",
-        filename="router_models/router_bundle_v10_fixed.pkl",
-    )
-    bundle = pickle.load(open(bundle_path, 'rb'))
-    tier_clfs = {int(k):v for k,v in bundle.get('tier_clfs',{}).items()}
-    tier_calibs = {int(k):v for k,v in bundle.get('tier_calibrators',{}).items()}
-    print(f"  Loaded v10 router: {len(tier_clfs)} tier classifiers")
-except Exception as e:
-    print(f"  WARNING: v10 bundle failed ({e}), using fallback routing")
-    tier_clfs = {}
-    tier_calibs = {}
-
-
-def route_cascade(text: str, threshold: float = 0.5) -> int:
-    """Route to cheapest tier with P(success) >= threshold."""
-    if not tier_clfs:
-        # Fallback: use hardcoded tier success rates
-        return 1  # always try cheapest
-    
-    x = extract_features(text).reshape(1, -1)
-    
-    for t in range(1, 6):
-        if t in tier_clfs:
-            try:
-                p_raw = tier_clfs[t].predict_proba(x)[0, 1]
-                p_cal = float(tier_calibs[t].transform([p_raw])[0])
-                if p_cal >= threshold:
-                    return t
-            except:
-                pass
-    
-    return 5  # escalate to strongest
-
-
-# ── 1. Load SWE-Router Data ─────────────────────────────────────
-from datasets import load_dataset
-
-print("\n[1] Loading SWE-Router execution data...")
-traces = defaultdict(dict)
-for model_name in MODELS:
-    ds = load_dataset(f"SWE-Router/swebench-verified-{model_name}", split="test")
-    for row in ds:
-        iid = row["instance_id"]
-        traces[iid][model_name] = {
-            "resolved": row["resolved"],
-            "cost": float(row["instance_cost"]),
-            "api_calls": int(row["api_calls"]),
-            "problem": row.get("problem_statement", row.get("problem", "")),
-        }
-print(f"  Loaded {len(traces)} tasks × {len(MODELS)} models = {sum(len(v) for v in traces.values())} outcomes")
-
-
-# ── 2. Simulate ACO Agent Run ────────────────────────────────────
-print("\n[2] Simulating ACO-optimized agent runs...")
-
-def simulate_acorun(task_id: str, problem: str, 
-                    use_router: bool = True,
-                    use_feedback: bool = True,
-                    use_verifier: bool = True,
-                    use_doom: bool = True,
-                    use_tool_gate: bool = True,
-                    max_doom_cost: float = 1.0,
-                    verifier_max_per_run: int = 3) -> Dict:
-    """Simulate one ACO-optimized agent run."""
-    result = {
-        "task_id": task_id,
-        "total_cost": 0.0,
-        "success": False, "escalated": False,
-        "terminated_early": False,
-        "verifier_calls": 0, "tool_skips": 0,
-        "routed_tier": 1,
-    }
-    
-    # Step 1: Route
-    routed_tier = route_cascade(problem) if use_router else 4
-    routed_model = TIER_TO_MODEL.get(routed_tier, "claude-opus-4.7")
-    result["routed_tier"] = routed_tier
-    
-    # Step 2: Execute tier 1
-    model_trace = traces[task_id].get(routed_model, {})
-    resolved = model_trace.get("resolved", False)
-    cost = model_trace.get("cost", 0.30)
-    api_calls = model_trace.get("api_calls", 10)
-    
-    # Simulate 3 agent steps
-    n_steps = min(3, api_calls)
-    verifier_count = 0
-    
-    for i in range(n_steps):
-        step_cost = cost / max(n_steps, 1)
-        
-        # Tool gate
-        if use_tool_gate and random.random() < 0.3:
-            result["tool_skips"] += 1
-            step_cost *= 0.9  # save 10% on skipped tools
-        
-        # Verifier
-        should_verify = False
-        if use_verifier:
-            is_final = (i == n_steps - 1)
-            is_high_risk = routed_tier <= 2
-            has_failures = not resolved and i > 0
-            
-            if verifier_count < verifier_max_per_run and (is_final or is_high_risk or has_failures):
-                should_verify = True
-                verifier_count += 1
-                result["verifier_calls"] += 1
-                step_cost *= 1.15  # verifier adds 15%
-        
-        # Doom check (only after step 1)
-        if use_doom and i > 0:
-            total_so_far = result["total_cost"] + step_cost
-            failed = not resolved
-            if total_so_far > max_doom_cost * 0.7 and failed:
-                result["terminated_early"] = True
-                result["total_cost"] += step_cost * 0.5
-                return result
-        
-        result["total_cost"] += step_cost
-    
-    # If not resolved and feedback is on, escalate
-    if not resolved and use_feedback:
-        result["escalated"] = True
-        for escalate_tier in range(routed_tier + 1, 6):
-            esc_model = TIER_TO_MODEL[escalate_tier]
-            esc_trace = traces[task_id].get(esc_model, {})
-            esc_cost = esc_trace.get("cost", 0.30)
-            result["total_cost"] += esc_cost
-            
-            if esc_trace.get("resolved", False):
-                resolved = True
-                break
-            
-            # Doom check during escalation
-            if use_doom and result["total_cost"] > max_doom_cost * 0.8:
-                result["terminated_early"] = True
-                break
-    
-    result["success"] = resolved
-    return result
-
-
-# ── 3. Run Full Evaluation ──────────────────────────────────────
-N_TASKS = len(traces)
-eval_results = []
-configs = {
-    "full_aco":       (True,  True,  True,  True,  True),
-    "no_router":      (False, False, True,  True,  True),
-    "no_feedback":    (True,  False, True,  True,  True),
-    "no_verifier":    (True,  True,  False, True,  True),
-    "no_doom":        (True,  True,  True,  False, True),
-    "no_tool_gate":   (True,  True,  True,  True,  False),
-    "frontier_only":  (False, False, False, False, False),
-}
-
-for config_name, (router, feedback, verifier, doom, tool_gate) in configs.items():
-    results = []
-    for task_id, task_traces in traces.items():
-        problem = next(iter(task_traces.values()))["problem"]
-        result = simulate_acorun(
-            task_id, problem,
-            use_router=router, use_feedback=feedback,
-            use_verifier=verifier, use_doom=doom,
-            use_tool_gate=tool_gate,
-        )
-        results.append(result)
-    
-    n = len(results)
-    resolved = sum(1 for r in results if r["success"])
-    total_cost = sum(r["total_cost"] for r in results)
-    escalated = sum(1 for r in results if r["escalated"])
-    terminated = sum(1 for r in results if r["terminated_early"])
-    
-    eval_results.append({
-        "config": config_name,
-        "n": n, "resolved": resolved,
-        "success_rate": resolved / n,
-        "total_cost": total_cost,
-        "avg_cost": total_cost / n,
-        "escalated": escalated,
-        "terminated_early": terminated,
-        "verifier_calls": sum(r["verifier_calls"] for r in results),
-        "tool_skips": sum(r["tool_skips"] for r in results),
-    })
-
-# ── 4. Compute Baselines ────────────────────────────────────────
-frontier_cost = sum(
-    traces[tid].get("claude-opus-4.7", {}).get("cost", 0.317)
-    for tid in traces
-)
-frontier_resolved = sum(
-    1 for tid in traces
-    if traces[tid].get("claude-opus-4.7", {}).get("resolved", False)
-)
-cheap_cost = sum(
-    traces[tid].get("deepseek-v4-flash", {}).get("cost", 0.014)
-    for tid in traces
-)
-cheap_resolved = sum(
-    1 for tid in traces
-    if traces[tid].get("deepseek-v4-flash", {}).get("resolved", False)
-)
-
-# Oracle: cheapest model that succeeded per task
-oracle_cost = 0
-oracle_resolved = 0
-for tid, ttraces in traces.items():
-    best = None
-    for m, mt in ttraces.items():
-        if mt["resolved"] and (best is None or mt["cost"] < best["cost"]):
-            best = mt
-    if best:
-        oracle_cost += best["cost"]
-        oracle_resolved += 1
-    else:
-        oracle_cost += ttraces.get("claude-opus-4.7", {}).get("cost", 0.317)
-
-# ── 5. Print Results ────────────────────────────────────────────
-n = N_TASKS
-print(f"\n{'='*80}")
-print(f"ACO E2E BENCHMARK RESULTS ({n} SWE-bench tasks)")
-print(f"{'='*80}")
-print(f"\n{'Policy':<25} {'Resolved':>10} {'Rate':>10} {'AvgCost':>10} {'vsFrontier':>12}")
-print("-" * 67)
-
-full_ac = next(r for r in eval_results if r["config"] == "full_aco")["avg_cost"]
-
-for label, res, c in [
-    ("ACO Full (all modules)", 
-     next(r for r in eval_results if r["config"] == "full_aco")["resolved"],
-     next(r for r in eval_results if r["config"] == "full_aco")["avg_cost"]),
-    ("Frontier (always claude)", frontier_resolved, frontier_cost/n),
-    ("Always Cheap", cheap_resolved, cheap_cost/n),
-    ("Oracle", oracle_resolved, oracle_cost/n),
-]:
-    cr = (1 - c / (frontier_cost/n)) * 100
-    print(f"  {label:<23} {res:>10} {res/n*100:>9.1f}% ${c:>9.4f} {cr:>11.1f}%")
-
-print(f"\n{'='*80}")
-print("ABLATION STUDY: Per-Module Impact")
-print(f"{'='*80}")
-print(f"\n{'Configuration':<25} {'Resolved':>10} {'Rate':>8} {'AvgCost':>10} {'vsFull':>10}")
-print("-" * 63)
-
-full_r = next(r for r in eval_results if r["config"] == "full_aco")
-for r in eval_results:
-    dc = (r["avg_cost"] - full_r["avg_cost"]) / max(full_r["avg_cost"], 0.0001) * 100
-    dr = r["resolved"] - full_r["resolved"]
-    flag = ""
-    if dc > 0 and dr < 0: flag = " ← COSTS MORE, WORSE"
-    elif dc < 0 and dr >= 0: flag = " ← CHEAPER, SAME/BETTER"
-    elif dc > 0 and dr >= 0: flag = " ← COSTS MORE, BETTER"
-    elif dc < 0 and dr < 0: flag = " ← CHEAPER, WORSE"
-    print(f"  {r['config']:<23} {r['resolved']:>10} {r['resolved']/r['n']*100:>7.1f}% ${r['avg_cost']:>9.4f} {dc:>+9.1f}%{flag}")
-
-# ── 6. Cost-Quality Frontier ────────────────────────────────────
-print(f"\n{'='*80}")
-print("COST-QUALITY FRONTIER")
-print(f"{'='*80}")
-
-frontier_points = []
-for r in eval_results:
-    frontier_points.append((r["avg_cost"], r["success_rate"], r["config"]))
-
-frontier_points.append((cheap_cost/n, cheap_resolved/n, "always_cheap"))
-frontier_points.append((frontier_cost/n, frontier_resolved/n, "frontier"))
-frontier_points.append((oracle_cost/n, oracle_resolved/n, "oracle"))
-
-frontier_points.sort(key=lambda x: x[0])
-
-print(f"\n  {'Point':<25} {'Cost/task':>10} {'Success':>10} {'Dominated?':>12}")
-print("  " + "-" * 60)
-
-pareto = []
-for cost, succ, name in sorted(frontier_points, key=lambda x: (-x[1], x[0])):
-    dominated = any(pc <= cost and ps >= succ and (pc < cost or ps > succ) 
-                    for pc, ps, _ in pareto)
-    if not dominated:
-        pareto.append((cost, succ, name))
-    print(f"  {name:<25} ${cost:>9.4f} {succ*100:>9.1f}% {'✅ Pareto' if not dominated else '❌ Dominated':>12}")
-
-# ── 7. Save Results ─────────────────────────────────────────────
-output = {
-    "n_tasks": n,
-    "baselines": {
-        "frontier": {"resolved": frontier_resolved, "rate": round(frontier_resolved/n,4), 
-                     "total_cost": round(frontier_cost,2), "avg_cost": round(frontier_cost/n,4)},
-        "always_cheap": {"resolved": cheap_resolved, "rate": round(cheap_resolved/n,4),
-                         "total_cost": round(cheap_cost,2), "avg_cost": round(cheap_cost/n,4)},
-        "oracle": {"resolved": oracle_resolved, "rate": round(oracle_resolved/n,4),
-                   "total_cost": round(oracle_cost,2), "avg_cost": round(oracle_cost/n,4)},
-    },
-    "aco_configs": [
-        {"config": r["config"], "resolved": r["resolved"],
-         "rate": round(r["success_rate"], 4), "avg_cost": round(r["avg_cost"], 4),
-         "escalations": r["escalated"], "early_terminations": r["terminated_early"],
-         "verifier_calls": r["verifier_calls"], "tool_skips": r["tool_skips"]}
-        for r in eval_results
-    ],
-    "full_aco_cost_reduction": round(
-        (1 - full_ac / (frontier_cost/n)) * 100, 1
-    ),
-    "best_config": min(eval_results, key=lambda r: r["avg_cost"])["config"],
-    "pareto_optimal": [name for cost, succ, name in pareto],
-}
-
-with open("/tmp/aco_e2e_standalone.json", "w") as f:
-    json.dump(output, f, indent=2)
-
-from huggingface_hub import HfApi
-api = HfApi()
-api.upload_file(
-    path_or_fileobj="/tmp/aco_e2e_standalone.json",
-    path_in_repo="eval/e2e_standalone_results.json",
-    repo_id="narcolepticchicken/agent-cost-optimizer", repo_type="model",
-)
-
-print(f"\n✓ Results: eval/e2e_standalone_results.json")
-print(f"\nDONE!")

training/e2e_v2_fixed.py

@@ -1,291 +0,0 @@
-"""Standalone ACO E2E Benchmark v2 — Fixed cost model.
-
-Key fix: Use SWE-Router per-model costs directly instead of simulating 
-phantom agent steps. The SWE-Router cost already includes the full agent
-run (all model calls, tool calls, etc).
-
-The ACO's job is to:
-1. Route to a tier (choose model)
-2. If that model fails, escalate
-
-Cost = sum of all models used until success or abandonment.
-"""
-import json, sys, os, random, pickle
-from collections import defaultdict
-from typing import Dict
-import numpy as np
-
-# ── Core Router ──────────────────────────────────────────────────
-CODE_KW = ["python","javascript","code","function","bug","debug","refactor",
-    "implement","test","compile","runtime","segfault","thread","async","class",
-    "module","import","error","traceback"]
-CRITICAL_KW = ["critical","production","urgent","emergency","live","deployed","safety","security"]
-SIMPLE_KW = ["typo","simple","quick","brief","minor","small","easy","trivial","just"]
-RESEARCH_KW = ["research","investigate","compare","analyze","survey","paper"]
-TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
-LONG_KW = ["plan","project","roadmap","orchestrate","migrate","pipeline","deploy","architecture"]
-
-FEAT_KEYS = sorted([
-    'req_len','num_words','has_code','n_code','has_legal','has_research',
-    'has_tool','has_critical','has_simple','has_long','has_math',
-    'has_error_msg','has_file_path','n_lines','has_version','has_add',
-    'has_fix','has_change','has_remove','has_test','has_doc',
-    'has_see_also','has_steps_to_reproduce',
-])
-
-TIER_TO_MODEL = {1:'deepseek-v4-flash',2:'gpt-5-mini',3:'gemini-2.5-pro',
-                  4:'claude-opus-4.7',5:'gemini-3-pro'}
-MODELS = ["deepseek-v4-flash","gpt-5-nano","gpt-5-mini","deepseek-v3.2",
-           "gemini-2.5-pro","claude-opus-4.7","gpt-5.2","gemini-3-pro"]
-
-def extract_features(text: str) -> np.ndarray:
-    r = text.lower()
-    feats = {
-        'req_len':len(text),'num_words':len(text.split()),
-        'has_code':int(any(k in r for k in CODE_KW)),
-        'n_code':sum(1 for k in CODE_KW if k in r),
-        'has_legal':int(any(k in r for k in["contract","legal","compliance"])),
-        'has_research':int(any(k in r for k in RESEARCH_KW)),
-        'has_tool':int(any(k in r for k in TOOL_KW)),
-        'has_critical':int(any(k in r for k in CRITICAL_KW)),
-        'has_simple':int(any(k in r for k in SIMPLE_KW)),
-        'has_long':int(any(k in r for k in LONG_KW)),
-        'has_math':int(any(k in r for k in["calculate","compute","solve","equation"])),
-        'has_error_msg':int('error'in r or'traceback'in r or'exception'in r),
-        'has_file_path':int('/'in r),
-        'n_lines':text.count('\n')+1,
-        'has_version':int('version'in r or'update'in r),
-        'has_add':int('add'in r or'new'in r or'create'in r),
-        'has_fix':int('fix'in r or'bug'in r or'issue'in r),
-        'has_change':int('change'in r or'modify'in r),
-        'has_remove':int('remove'in r or'delete'in r),
-        'has_test':int('test'in r or'spec'in r),
-        'has_doc':int('doc'in r or'readme'in r),
-        'has_see_also':int('see also'in r or'related'in r),
-        'has_steps_to_reproduce':int('reproduce'in r or'steps'in r),
-    }
-    return np.array([float(feats.get(k,0))for k in FEAT_KEYS],dtype=np.float32)
-
-# ── Load Router ──────────────────────────────────────────────────
-from huggingface_hub import hf_hub_download
-print("Loading router...")
-try:
-    bundle_path = hf_hub_download(repo_id="narcolepticchicken/agent-cost-optimizer",
-                                   filename="router_models/router_bundle_v10_fixed.pkl")
-    bundle = pickle.load(open(bundle_path,'rb'))
-    tier_clfs = {int(k):v for k,v in bundle.get('tier_clfs',{}).items()}
-    tier_calibs = {int(k):v for k,v in bundle.get('tier_calibrators',{}).items()}
-    print(f"  v10 router loaded: {len(tier_clfs)} classifiers")
-except Exception as e:
-    print(f"  WARNING: {e}, using fallback")
-    tier_clfs = {}
-    tier_calibs = {}
-
-def route(text: str, thresh: float = 0.5) -> int:
-    if not tier_clfs: return 1
-    x = extract_features(text).reshape(1,-1)
-    for t in range(1,6):
-        if t in tier_clfs:
-            try:
-                p = tier_clfs[t].predict_proba(x)[0,1]
-                p = float(tier_calibs[t].transform([p])[0])
-                if p >= thresh: return t
-            except: pass
-    return 5
-
-# ── 1. Load SWE-Router ──────────────────────────────────────────
-from datasets import load_dataset
-print("\n[1] Loading SWE-Router...")
-traces = defaultdict(dict)
-for m in MODELS:
-    ds = load_dataset(f"SWE-Router/swebench-verified-{m}", split="test")
-    for row in ds:
-        traces[row["instance_id"]][m] = {
-            "resolved": row["resolved"],
-            "cost": float(row["instance_cost"]),
-        }
-print(f"  {len(traces)} tasks × {len(MODELS)} models")
-
-# ── 2. Simulate ACO ─────────────────────────────────────────────
-# Cost model: just sum the costs of each model we try.
-# ACO routes tier T → execute → if fail, try T+1 → etc.
-# Success = first successful tier's resolution.
-# Cost = sum of costs of all tiers attempted.
-
-print("\n[2] Simulating...")
-
-def aco_cost(task_id: str, problem: str, 
-             use_router: bool = True,
-             use_feedback: bool = True,
-             use_verifier: bool = True,
-             max_tiers: int = 5,
-             verifier_cost_factor: float = 1.05) -> Dict:
-    """ACO decision + execution: route, execute, escalate if feedback on."""
-    tier = route(problem) if use_router else 4
-    total_cost = 0.0
-    resolved = False
-    tiers_tried = 0
-    escalated = False
-    verifier_cost = 0.0
-    
-    while tiers_tried < max_tiers:
-        model = TIER_TO_MODEL.get(tier, "claude-opus-4.7")
-        mt = traces[task_id].get(model, {})
-        step_cost = mt.get("cost", 0.30)
-        
-        if use_verifier and tier <= 2:
-            step_cost *= verifier_cost_factor
-            verifier_cost += step_cost * (verifier_cost_factor - 1)
-        
-        total_cost += step_cost
-        tiers_tried += 1
-        
-        if mt.get("resolved", False):
-            resolved = True
-            break
-        
-        if not use_feedback:
-            break  # no escalation
-        
-        if tier >= 5:
-            break
-        
-        tier += 1
-        escalated = True
-    
-    return {
-        "total_cost": total_cost,
-        "resolved": resolved,
-        "escalated": escalated,
-        "tiers_tried": tiers_tried,
-        "verifier_cost": verifier_cost,
-        "initial_tier": tier if not escalated else tier - tiers_tried + 1,
-    }
-
-# ── 3. Run all configs ──────────────────────────────────────────
-configs = {
-    "full_aco":          (True, True, True),
-    "no_router":         (False, True, True),
-    "no_feedback":       (True, False, True),
-    "no_verifier":       (True, True, False),
-    "router_only":       (True, False, False),
-    "frontier_always":   (False, False, False),
-}
-
-results_by_config = {}
-for cname, (router, feedback, verifier) in configs.items():
-    rlist = []
-    for tid, ttraces in traces.items():
-        problem = next(iter(ttraces.values())).get("problem", "")
-        rlist.append(aco_cost(tid, problem, router, feedback, verifier))
-    results_by_config[cname] = rlist
-
-# ── 4. Baselines ────────────────────────────────────────────────
-N = len(traces)
-frontier_resolved = sum(1 for tid in traces if traces[tid].get("claude-opus-4.7",{}).get("resolved",False))
-frontier_cost = sum(traces[tid].get("claude-opus-4.7",{}).get("cost",0.317) for tid in traces)
-cheap_resolved = sum(1 for tid in traces if traces[tid].get("deepseek-v4-flash",{}).get("resolved",False))
-cheap_cost = sum(traces[tid].get("deepseek-v4-flash",{}).get("cost",0.014) for tid in traces)
-oracle_cost = 0; oracle_resolved = 0
-for tid, tt in traces.items():
-    best = None
-    for m, mt in tt.items():
-        if mt["resolved"] and (best is None or mt["cost"] < best["cost"]):
-            best = mt
-    if best:
-        oracle_cost += best["cost"]; oracle_resolved += 1
-    else:
-        oracle_cost += tt.get("claude-opus-4.7",{}).get("cost",0.317)
-
-# ── 5. Print ────────────────────────────────────────────────────
-fc = frontier_cost / N
-print(f"\n{'='*80}")
-print(f"ACO E2E RESULTS v2 — 500 SWE-bench tasks")
-print(f"{'='*80}")
-print(f"\n{'Policy':<25} {'Resolved':>10} {'Rate':>10} {'AvgCost':>10} {'vsFrontier':>12} {'Pareto?':>10}")
-print("-" * 77)
-
-all_points = []
-for label, data in [
-    ("ACO Full", results_by_config["full_aco"]),
-    ("ACO Router+Feedback", results_by_config["no_verifier"]),
-    ("ACO Router Only", results_by_config["router_only"]),
-    ("ACO No Feedback", results_by_config["no_feedback"]),
-    ("ACO No Router", results_by_config["no_router"]),
-    ("Always Cheap", None),
-    ("Frontier Always", None),
-    ("Oracle", None),
-]:
-    if label == "Always Cheap":
-        r, c = cheap_resolved, cheap_cost/N
-    elif label == "Frontier Always":
-        r, c = frontier_resolved, fc
-    elif label == "Oracle":
-        r, c = oracle_resolved, oracle_cost/N
-    else:
-        r = sum(1 for d in data if d["resolved"])
-        c = sum(d["total_cost"] for d in data) / N
-    
-    cr = (1 - c/fc)*100
-    all_points.append((label, r, c, cr))
-    
-    print(f"  {label:<23} {r:>10} {r/N*100:>9.1f}% ${c:>9.4f} {cr:>11.1f}% {'':>10}")
-
-# Pareto analysis
-print(f"\n{'='*80}")
-print("PARETO FRONTIER")
-print(f"{'='*80}")
-pareto_set = []
-for label, r, c, cr in sorted(all_points, key=lambda x: (-x[1], x[0])):
-    dominated = any(pr >= r and pc <= c and (pr > r or pc < c) for _, pr, pc, _ in pareto_set)
-    pareto_set.append((label, r, c, cr))
-    print(f"  {label:<25} {r/N*100:>9.1f}% ${c:>9.4f} {'✅ PARETO-OPTIMAL' if not dominated else '❌ Dominated'}")
-
-# Module impact
-print(f"\n{'='*80}")
-print("PER-MODULE IMPACT (cost delta from full ACO)")
-print(f"{'='*80}")
-full_r = next(l for l in all_points if l[0] == "ACO Full")
-for label, r, c, cr in all_points:
-    if label == "ACO Full": continue
-    dc = (c - full_r[2]) / max(full_r[2], 0.0001) * 100
-    dr = r - full_r[1]
-    note = ""
-    if dc < 0 and dr >= 0: note = "← IMPROVEMENT (cheaper, no quality loss)"
-    elif dc < 0: note = f"← Cheaper but lost {abs(dr)} resolves"
-    elif dr >= 0: note = f"← Better quality at +${c-full_r[2]:.4f}"
-    print(f"  {label:<23} {dc:>+7.1f}% Δcost  {dr:>+4} resolves  {note}")
-
-# ── 6. Save ─────────────────────────────────────────────────────
-output = {
-    "n": N,
-    "configs": {
-        name: {
-            "resolved": sum(1 for d in data if d["resolved"]),
-            "rate": round(sum(1 for d in data if d["resolved"])/N, 4),
-            "avg_cost": round(sum(d["total_cost"] for d in data)/N, 4),
-            "total_cost": round(sum(d["total_cost"] for d in data), 2),
-            "avg_tiers_tried": round(sum(d["tiers_tried"] for d in data)/N, 2),
-            "escalated_rate": round(sum(1 for d in data if d["escalated"])/N, 2),
-            "verifier_cost": round(sum(d["verifier_cost"] for d in data), 2),
-        } for name, data in results_by_config.items()
-    },
-    "baselines": {
-        "frontier": {"resolved": frontier_resolved, "rate": round(frontier_resolved/N,4), "avg_cost": round(fc,4)},
-        "always_cheap": {"resolved": cheap_resolved, "rate": round(cheap_resolved/N,4), "avg_cost": round(cheap_cost/N,4)},
-        "oracle": {"resolved": oracle_resolved, "rate": round(oracle_resolved/N,4), "avg_cost": round(oracle_cost/N,4)},
-    },
-}
-
-with open("/tmp/aco_e2e_v2_fixed.json","w") as f:
-    json.dump(output, f, indent=2)
-
-from huggingface_hub import HfApi
-api = HfApi()
-api.upload_file(
-    path_or_fileobj="/tmp/aco_e2e_v2_fixed.json",
-    path_in_repo="eval/e2e_v2_fixed_results.json",
-    repo_id="narcolepticchicken/agent-cost-optimizer", repo_type="model",
-)
-print(f"\n✓ eval/e2e_v2_fixed_results.json")
-print("DONE!")