Delete eval/ablation_results.json, eval/baarbundle.json, eval/benchmark_results.json, eval/benchmark_v9_final.json, eval/bfcl_results.json, eval/v8_*, eval/e2e_standalone_results.json, eval/e2e_v2_fixed_results.json, eval/repro_v2.json, eval/launcher.py, eval/run_bert_eval_launcher.py, docs/model_card.md, docs/deployment_guide.md, docs/ROADMAP.md, docs/technical_blog.md, docs/conformal_report.md, docs/FINAL_COMPREHENSIVE_REPORT.md, docs/final_report_v2.md, eval_runner.py, app.py, dashboard.py, examples/end_to_end_demo.py

app.py

@@ -1,183 +0,0 @@
-"""Gradio Space for Agent Cost Optimizer Dashboard.
-
-This app visualizes cost-quality frontiers from ACO benchmark runs.
-If no benchmark data exists, it runs the benchmark on first load.
-"""
-
-import json
-import subprocess
-import sys
-from pathlib import Path
-from typing import Dict, List, Any
-
-import gradio as gr
-
-
-def ensure_data_exists():
-    """Run benchmark if data doesn't exist."""
-    results_path = Path("eval_results_v2/baseline_results.json")
-    report_path = Path("eval_results_v2/report.txt")
-    
-    if not results_path.exists() or not report_path.exists():
-        print("Benchmark data not found. Running benchmark...")
-        try:
-            # Run the benchmark generator
-            subprocess.run(
-                [sys.executable, "standalone_eval_v2.py", "--tasks", "2000", "--output", "eval_results_v2"],
-                capture_output=True, text=True, timeout=120
-            )
-            print("Benchmark complete.")
-        except Exception as e:
-            print(f"Benchmark failed: {e}")
-    
-    return results_path, report_path
-
-
-def load_results(path: str) -> Dict[str, Any]:
-    with open(path) as f:
-        return json.load(f)
-
-
-def parse_report(report_path: str) -> str:
-    with open(report_path) as f:
-        return f.read()
-
-
-def create_frontier_plot(results: Dict[str, Any]):
-    points = []
-    for name, data in results.items():
-        success = (data.get("num_success", 0) + data.get("num_partial", 0)) / max(data.get("num_tasks", 1), 1)
-        cost = data.get("avg_cost_success", 0)
-        points.append({"baseline": name, "success_rate": success, "cost_per_success": cost})
-    points.sort(key=lambda p: (-p["success_rate"], p["cost_per_success"]))
-    frontier = []
-    min_cost = float("inf")
-    for p in points:
-        if p["cost_per_success"] <= min_cost:
-            frontier.append(p)
-            min_cost = p["cost_per_success"]
-    return points, frontier
-
-
-def build_dashboard():
-    results_path, report_path = ensure_data_exists()
-    
-    if not results_path.exists() or not report_path.exists():
-        with gr.Blocks(title="Agent Cost Optimizer Dashboard") as demo:
-            gr.Markdown("# Agent Cost Optimizer Dashboard")
-            gr.Markdown("## Benchmark data not available")
-            gr.Markdown("Run `python standalone_eval_v2.py --tasks 2000 --output eval_results_v2` to generate data.")
-        return demo
-    
-    results = load_results(str(results_path))
-    report_text = parse_report(str(report_path))
-    points, frontier = create_frontier_plot(results)
-    
-    with gr.Blocks(title="Agent Cost Optimizer Dashboard") as demo:
-        gr.Markdown("# Agent Cost Optimizer - Cost-Quality Dashboard")
-        gr.Markdown("Visualize cost-quality tradeoffs across routing strategies and ablations.")
-        
-        with gr.Row():
-            with gr.Column(scale=2):
-                gr.Markdown("## Cost-Quality Frontier")
-                gr.Markdown("**X-axis**: Average cost per successful task | **Y-axis**: Success rate")
-                
-                scatter_data = [
-                    [p["baseline"], f"{p['success_rate']:.1%}", f"${p['cost_per_success']:.4f}"]
-                    for p in points
-                ]
-                gr.Dataframe(
-                    headers=["Baseline", "Success Rate", "Cost per Success"],
-                    value=scatter_data,
-                    label="All Baselines",
-                )
-                
-                frontier_data = [
-                    [p["baseline"], f"{p['success_rate']:.1%}", f"${p['cost_per_success']:.4f}"]
-                    for p in frontier
-                ]
-                gr.Dataframe(
-                    headers=["Baseline", "Success Rate", "Cost per Success"],
-                    value=frontier_data,
-                    label="Pareto Frontier",
-                )
-            
-            with gr.Column(scale=1):
-                gr.Markdown("## Pareto Frontier Baselines")
-                pareto_names = [p["baseline"] for p in frontier]
-                for name in pareto_names:
-                    gr.Markdown(f"- **{name}**")
-        
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("## Baseline Comparison")
-                comparison_data = []
-                for name, data in results.items():
-                    comparison_data.append([
-                        name,
-                        f"{(data.get('num_success',0)+data.get('num_partial',0))/max(data.get('num_tasks',1),1):.1%}",
-                        f"${data.get('avg_cost_success',0):.4f}",
-                        f"${data.get('total_cost',0):.2f}",
-                        f"{data.get('cost_reduction_vs_frontier',0):.1%}",
-                        f"{data.get('false_done_rate',0):.1%}",
-                        f"{data.get('unsafe_cheap_miss_rate',0):.1%}",
-                        f"{data.get('regression_rate',0):.1%}",
-                    ])
-                gr.Dataframe(
-                    headers=["Baseline", "Success", "Cost/Success", "Total Cost", "Cost Reduction", "False-DONE", "Cheap Miss", "Regression"],
-                    value=comparison_data,
-                )
-        
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("## Per-Scenario Breakdown (Full Optimizer)")
-                full_data = results.get("full_optimizer", {})
-                scenario_stats = full_data.get("per_scenario_stats", {})
-                if scenario_stats:
-                    scenario_data = []
-                    for scenario, stats in scenario_stats.items():
-                        count = stats.get("count", 0)
-                        success = stats.get("success", 0)
-                        cost = stats.get("cost", 0)
-                        scenario_data.append([
-                            scenario,
-                            str(count),
-                            f"{success/max(count,1):.1%}",
-                            f"${cost:.2f}",
-                        ])
-                    gr.Dataframe(
-                        headers=["Scenario", "Count", "Success Rate", "Total Cost"],
-                        value=scenario_data,
-                    )
-        
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("## Ablation Impact")
-                gr.Markdown("Cost impact when removing each module (vs full_optimizer)")
-                
-                full_cost = results.get("full_optimizer", {}).get("total_cost", 0)
-                ablation_data = []
-                for name, data in results.items():
-                    if name.startswith("no_"):
-                        delta = data.get("total_cost", 0) - full_cost
-                        pct = (delta / max(full_cost, 0.001)) * 100
-                        ablation_data.append([name, f"${delta:.2f}", f"{pct:.1f}%"])
-                
-                if ablation_data:
-                    ablation_data.sort(key=lambda x: float(x[1].replace("$", "")), reverse=True)
-                    gr.Dataframe(
-                        headers=["Module Removed", "Cost Increase", "% Increase"],
-                        value=ablation_data,
-                    )
-        
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("## Full Report")
-                gr.Textbox(report_text, lines=40, label="Benchmark Report", interactive=False)
-    
-    return demo
-
-
-if __name__ == "__main__":
-    demo = build_dashboard()
-    demo.launch(server_name="0.0.0.0", server_port=7860)

dashboard.py

@@ -1,176 +0,0 @@
-"""Gradio Dashboard for Agent Cost Optimizer.
-
-Visualizes:
-- Cost-quality frontier (scatter plot: success rate vs avg cost)
-- Baseline comparison bar charts
-- Per-scenario breakdown
-- Module ablation impact
-"""
-
-import json
-import sys
-from pathlib import Path
-from typing import Dict, List, Any
-
-import gradio as gr
-
-
-def load_results(path: str) -> Dict[str, Any]:
-    with open(path) as f:
-        return json.load(f)
-
-
-def parse_report(report_path: str) -> str:
-    with open(report_path) as f:
-        return f.read()
-
-
-def create_frontier_plot(results: Dict[str, Any]):
-    """Create scatter plot data for cost-quality frontier."""
-    points = []
-    for name, data in results.items():
-        success = (data.get("num_success", 0) + data.get("num_partial", 0)) / max(data.get("num_tasks", 1), 1)
-        cost = data.get("avg_cost_success", 0)
-        points.append({"baseline": name, "success_rate": success, "cost_per_success": cost})
-    
-    # Sort by success rate desc, cost asc for frontier
-    points.sort(key=lambda p: (-p["success_rate"], p["cost_per_success"]))
-    
-    # Build Pareto frontier
-    frontier = []
-    min_cost = float("inf")
-    for p in points:
-        if p["cost_per_success"] <= min_cost:
-            frontier.append(p)
-            min_cost = p["cost_per_success"]
-    
-    return points, frontier
-
-
-def build_dashboard(results_path: str, report_path: str):
-    results = load_results(results_path)
-    report_text = parse_report(report_path)
-    points, frontier = create_frontier_plot(results)
-    
-    with gr.Blocks(title="Agent Cost Optimizer Dashboard") as demo:
-        gr.Markdown("# Agent Cost Optimizer - Cost-Quality Dashboard")
-        gr.Markdown("Visualize cost-quality tradeoffs across routing strategies and ablations.")
-        
-        with gr.Row():
-            with gr.Column(scale=2):
-                gr.Markdown("## Cost-Quality Frontier")
-                gr.Markdown("**X-axis**: Average cost per successful task | **Y-axis**: Success rate")
-                
-                # Scatter plot using native Gradio components
-                scatter_data = [
-                    [p["baseline"], f"{p['success_rate']:.1%}", f"${p['cost_per_success']:.4f}"]
-                    for p in points
-                ]
-                gr.Dataframe(
-                    headers=["Baseline", "Success Rate", "Cost per Success"],
-                    value=scatter_data,
-                    label="All Baselines",
-                )
-                
-                frontier_data = [
-                    [p["baseline"], f"{p['success_rate']:.1%}", f"${p['cost_per_success']:.4f}"]
-                    for p in frontier
-                ]
-                gr.Dataframe(
-                    headers=["Baseline", "Success Rate", "Cost per Success"],
-                    value=frontier_data,
-                    label="Pareto Frontier",
-                )
-            
-            with gr.Column(scale=1):
-                gr.Markdown("## Pareto Frontier Baselines")
-                pareto_names = [p["baseline"] for p in frontier]
-                for name in pareto_names:
-                    gr.Markdown(f"- **{name}**")
-        
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("## Baseline Comparison")
-                comparison_data = []
-                for name, data in results.items():
-                    comparison_data.append([
-                        name,
-                        f"{(data.get('num_success',0)+data.get('num_partial',0))/max(data.get('num_tasks',1),1):.1%}",
-                        f"${data.get('avg_cost_success',0):.4f}",
-                        f"${data.get('total_cost',0):.2f}",
-                        f"{data.get('cost_reduction_vs_frontier',0):.1%}",
-                        f"{data.get('false_done_rate',0):.1%}",
-                        f"{data.get('unsafe_cheap_miss_rate',0):.1%}",
-                        f"{data.get('regression_rate',0):.1%}",
-                    ])
-                gr.Dataframe(
-                    headers=["Baseline", "Success", "Cost/Success", "Total Cost", "Cost Reduction", "False-DONE", "Cheap Miss", "Regression"],
-                    value=comparison_data,
-                )
-        
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("## Per-Scenario Breakdown (Full Optimizer)")
-                full_data = results.get("full_optimizer", {})
-                scenario_stats = full_data.get("per_scenario_stats", {})
-                if scenario_stats:
-                    scenario_data = []
-                    for scenario, stats in scenario_stats.items():
-                        count = stats.get("count", 0)
-                        success = stats.get("success", 0)
-                        cost = stats.get("cost", 0)
-                        scenario_data.append([
-                            scenario,
-                            str(count),
-                            f"{success/max(count,1):.1%}",
-                            f"${cost:.2f}",
-                        ])
-                    gr.Dataframe(
-                        headers=["Scenario", "Count", "Success Rate", "Total Cost"],
-                        value=scenario_data,
-                    )
-        
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("## Ablation Impact")
-                gr.Markdown("Cost increase when removing each module (vs full_optimizer)")
-                
-                full_cost = results.get("full_optimizer", {}).get("total_cost", 0)
-                ablation_data = []
-                for name, data in results.items():
-                    if name.startswith("no_"):
-                        delta = data.get("total_cost", 0) - full_cost
-                        pct = (delta / max(full_cost, 0.001)) * 100
-                        ablation_data.append([name, f"${delta:.2f}", f"{pct:.1f}%"])
-                
-                if ablation_data:
-                    ablation_data.sort(key=lambda x: float(x[1].replace("$", "")), reverse=True)
-                    gr.Dataframe(
-                        headers=["Module Removed", "Cost Increase", "% Increase"],
-                        value=ablation_data,
-                    )
-        
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("## Full Report")
-                gr.Textbox(report_text, lines=40, label="Benchmark Report")
-    
-    return demo
-
-
-def main():
-    import argparse
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--results", default="./eval_results_v2/baseline_results.json",
-                        help="Path to baseline results JSON")
-    parser.add_argument("--report", default="./eval_results_v2/report.txt",
-                        help="Path to report text file")
-    parser.add_argument("--port", type=int, default=7860)
-    args = parser.parse_args()
-    
-    demo = build_dashboard(args.results, args.report)
-    demo.launch(server_name="0.0.0.0", server_port=args.port)
-
-
-if __name__ == "__main__":
-    main()

docs/FINAL_COMPREHENSIVE_REPORT.md

@@ -1,277 +0,0 @@
-# Agent Cost Optimizer — Final Comprehensive Report
-
-**Date:** 2026-05-08  
-**Repository:** [narcolepticchicken/agent-cost-optimizer](https://huggingface.co/narcolepticchicken/agent-cost-optimizer)
-
----
-
-## 1. Overview
-
-This report presents the complete evaluation of the Agent Cost Optimizer (ACO) model cascading router, tested across multiple approaches on SWE-bench Verified (500 coding tasks × 8 models = 4,000 execution outcomes).
-
-### The Core Problem
-
-Given a coding task, which model should an agent use? The cheapest model that solves the task (cheapest model that actually produces a working patch). The challenge is predicting which model will succeed without actually running it.
-
-### Evaluation Metrics
-
-| Metric | Definition |
-|--------|-----------|
-| **Success Rate** | Fraction of tasks where the routed model produces a resolved (passing) patch |
-| **Avg Cost** | Mean cost per task in dollars (from SWE-Router execution data) |
-| **Cost Reduction** | % savings vs always using the frontier model (claude-opus-4.7) |
-| **Pareto Optimal** | No other policy has both higher success AND lower cost |
-
----
-
-## 2. All Router Approaches Evaluated
-
-### 2.1 Baseline Policies
-
-| Policy | Success | AvgCost | CostRed | Description |
-|--------|---------|---------|---------|-------------|
-| **oracle** | 87.0% | $0.059 | 81.5% | Cheapest model that solved each task (upper bound) |
-| **frontier** | 78.2% | $0.317 | 0.0% | Always claude-opus-4.7 (baseline) |
-| **always_cheap** | 63.2% | $0.014 | 95.5% | Always deepseek-v4-flash (lower bound) |
-
-### 2.2 XGBoost Keyword Routers
-
-| Version | Features | Success | AvgCost | CostRed | Notes |
-|---------|----------|---------|---------|---------|-------|
-| **v10** | 27 keywords + text stats | 79.0% | $0.194 | 38.8% | Best keyword router |
-| **v10+feedback** | v10 route + escalate on failure | 84.8% | $0.202 | **36.4%** | **Best overall: Pareto-optimal** |
-
-### 2.3 BERT-Based Routers
-
-| Approach | Success | AvgCost | CostRed | Notes |
-|----------|---------|---------|---------|-------|
-| **BERT 5-class (direct)** | 54.8% | $0.042 | 86.8% | Predicts tier 2 for 96% of tasks |
-| **BERT 5-class+feedback** | 84.0% | $0.578 | **-82.5%** | Costs more than frontier! |
-| **BERT binary predictor** | N/A | N/A | N/A | Predicts success 89.5% for ALL tiers |
-| **BAAR (BERT+K+XGB)** | 76.6% | $0.188 | 40.7% | Marginal improvement over keyword-only |
-| **BAAR+feedback** | 85.2% | $0.249 | 21.5% | Slightly better success, worse cost reduction |
-
----
-
-## 3. The Pareto Frontier
-
-```
-Success Rate
-   87% │                    ★oracle (87.0%, $0.059, -81.5%)
-       │
-   85% │  ★baar_fb (85.2%, $0.249, -21.5%)
-       │  ★v10_fb (84.8%, $0.202, -36.4%)    ← PARETO-OPTIMAL
-       │
-   80% │  ★v10 (79.0%, $0.194, -38.8%)
-       │              ★frontier (78.2%, $0.317, baseline)
-       │
-   75% │  ★baar (76.6%, $0.188, -40.7%)
-       │
-   70% │
-       │
-   65% │
-       │  ★always_cheap (63.2%, $0.014, -95.5%)
-   60% │
-       └──────────────────────────────────────────
-          0.05    0.10    0.15    0.20    0.25    0.30
-                           Avg Cost ($)
-```
-
-**Pareto-optimal policies:** oracle, v10+feedback, always_cheap
-
-Everything else is dominated — there exists another policy with strictly better success AND lower cost.
-
-### Dominated Policies
-
-| Policy | Dominated By | Why |
-|--------|-------------|-----|
-| **frontier** | v10 | v10 is cheaper AND more successful |
-| **baar_fb** | v10_fb | v10_fb is cheaper AND nearly as successful |
-| **bert_feedback** | frontier | frontier is cheaper AND more successful |
-| **baar** | v10 | v10 is cheaper AND more successful |
-
----
-
-## 4. Why BERT Routing Failed
-
-### 4.1 SPROUT → BERT 5-class
-
-The BERT 5-class model trained on SPROUT (31K QA tasks) collapses to predicting tier 1 for 99.99% of QA tasks and cannot distinguish coding difficulty on SWE-bench:
-
-| Metric | Value |
-|--------|-------|
-| Training accuracy (SPROUT) | 76.9% (epoch 2) |
-| Tier 1 eval accuracy | 99.92% |
-| Tier 2 eval accuracy | 1.26% |
-| Tier 3-5 eval accuracy | 0-1.7% |
-| On SWE-bench (direct) | Predicts tier 2 for 96% of tasks |
-
-**Root cause:** 77.4% of SPROUT tasks are solvable by the cheapest model. BERT learns "always predict tier 1" and ignores minority classes. For SWE-bench, the fine-tuned model shifts to "always predict tier 2" — effectively random routing for coding tasks.
-
-### 4.2 BERT Binary Success Predictor
-
-Predicting P(success) for each tier produced probability 0.89-0.90 for ALL tiers on ALL tasks. The BERT [CLS] embedding doesn't capture task-difficulty signals that differ between coding tasks.
-
-### 4.3 BAAR (BERT [CLS] + XGBoost)
-
-Adding BERT embeddings as XGBoost features improved CV F1 by 1.8-4.6% across tiers but didn't change routing decisions in any meaningful way. The keyword-only v10 router achieves 42.7% cost reduction vs BAAR's 40.7% — BERT features don't add enough signal to justify the computational cost.
-
-The `direct optimal-tier accuracy: 1.000` (perfect on training data) confirms the model is memorizing via the rich 768-dim BERT embeddings, not generalizing.
-
-### 4.4 Fundamental Insight
-
-**SPROUT (QA benchmark) does not transfer to SWE-bench (coding benchmark).** Task difficulty is domain-specific. A task that's easy for a cheap model on QA (e.g., "what is the capital of France?") has no correlation with coding difficulty. You cannot train a coding-agent router on QA data.
-
----
-
-## 5. What Actually Works
-
-### 5.1 v10+feedback: The Current Best
-
-```
-Route: XGBoost keyword features → predict cheapest viable tier
-Execute: Run the predicted model
-Feedback: If fails, escalate to next tier and retry
-```
-
-| Component | Impact |
-|-----------|--------|
-| Keyword features | 23 text-statistics keywords (has_fix, has_error, n_lines, etc.) |
-| XGBoost | 5 binary classifiers, one per tier |
-| Isotonic calibration | Probability calibration for thresholding |
-| Feedback escalation | Try next tier on failure |
-
-**84.8% success, 36.4% cost reduction, Pareto-optimal.**
-
-### 5.2 Why Keywords Beat BERT
-
-Keyword features are interpretable and directly relevant to coding task difficulty:
-- `has_error`, `has_traceback` → likely harder (need debugging)
-- `has_test`, `has_spec` → likely harder (has test requirements)  
-- `n_lines`, `has_file_path` → structural complexity signals
-
-BERT [CLS] captures holistic semantics optimized for masked language modeling, not task-specific difficulty signals. The 768-dim embeddings are too rich for 500 training examples — they overfit rather than generalize.
-
-### 5.3 Why Feedback Escalation Works
-
-The feedback mechanism converts cheap-model failures into escalation opportunities. Even if the router only routes correctly 60% of the time, the feedback loop catches the remaining 40% by falling back to stronger models.
-
-Cost analysis:
-- 67.4% of tasks solvable by tier 1 ($0.01/task)
-- For the remaining 32.6%, escalate to tier 2 ($0.05) → tier 4 ($0.30)
-- Weighted average: 0.674 × $0.01 + 0.15 × ($0.01+$0.05) + 0.176 × ($0.01+$0.05+$0.30) = $0.202
-- Vs frontier: $0.317 → 36.4% savings
-
----
-
-## 6. SWE-Router Dataset Statistics
-
-Key statistics from the 500 SWE-bench tasks × 8 models:
-
-| Model | Tier | Success | Cost/Task |
-|-------|------|---------|-----------|
-| deepseek-v4-flash | 1 | 63.2% | $0.014 |
-| gpt-5-nano | 1 | — | — |
-| gpt-5-mini | 2 | 51.4% | $0.039 |
-| deepseek-v3.2 | 2 | 75.8% | $0.052 |
-| gemini-2.5-pro | 3 | 51.4% | $0.151 |
-| claude-opus-4.7 | 4 | 78.2% | $0.317 |
-| gpt-5.2 | 4 | 82.6% | $0.274 |
-| gemini-3-pro | 5 | 72.6% | $0.434 |
-
-**Interesting:** gpt-5.2 (tier 4) is both cheaper AND more successful than claude-opus-4.7. The frontier should be redefined as gpt-5.2 once confirmed reproducible.
-
-**67.4% of tasks are solvable by the cheapest model.** This is the key stat — if you can identify which 67.4% correctly, you save 95% of cost with zero quality loss. If you identify half correctly, you still save ~40%.
-
----
-
-## 7. The Upper Bound (Oracle)
-
-| Metric | Value |
-|--------|-------|
-| Oracle success | 87.0% |
-| Oracle cost/task | $0.059 |
-| Cost reduction | 81.5% |
-| % tasks solvable by cheapest model | 67.4% |
-| % tasks needing escalation | 32.6% |
-
-The oracle reveals that 19.6% of tasks (87.0% - 67.4%) are solvable by a non-frontier model but NOT by the cheapest — these are the "mid-difficulty" tasks where routing really matters.
-
-**The gap between v10+feedback (84.8%) and oracle (87.0%) is 2.2 percentage points.** Closing this gap requires better tier-1 vs tier-2 discrimination — the hardest routing decision.
-
----
-
-## 8. Key Architectural Decisions
-
-### 8.1 Per-Tier Binary Classifiers (Correct)
-Training 5 separate binary classifiers (one per tier) is the right architecture. It handles class imbalance, allows per-tier calibration, and enables threshold-based routing.
-
-### 8.2 5-Class Classifier (Wrong)
-A single 5-class classifier (BERT 5-class) collapses to majority class. Not recommended for this problem.
-
-### 8.3 Binary Success Predictor (Wrong)
-Predicting "will this model succeed?" for each model produces near-uniform probabilities. The signal is too weak.
-
-### 8.4 Execution Feedback (Essential)
-Without feedback escalation, any router mistake becomes a permanent failure. Feedback converts routing errors into slightly longer (but still successful) runs.
-
----
-
-## 9. Model Artifacts on Hub
-
-| Artifact | Path | Description |
-|----------|------|-------------|
-| BERT 5-class (fine-tuned) | `router_models/bert_5class/` | DistilBERT fine-tuned on SPROUT, 5-class |
-| BAAR bundle | `router_models/baar_bundle.pkl` | BERT+XGBoost bundled router |
-| v10 XGBoost | `router_models/aco_router_v10_xgb.pkl` | Keyword-only XGBoost (best performer) |
-| BAAR eval JSON | `eval/baar_results.json` | BAAR evaluation results |
-| Combined report | `docs/combined_final_report.md` | This file |
-
----
-
-## 10. Recommendations
-
-### For Production Use
-✅ **v10+feedback** is the recommended approach: 84.8% success at 36.4% cost reduction.
-
-### For Further Research
-1. **Train on SWE-bench problem statements directly** — don't use SPROUT
-2. **Per-step routing** — route model choice at each agent step, not once per task
-3. **Confidence-based escalation** — use model logprobs/entropy as additional routing features
-4. **Redefine frontier as gpt-5.2** — it's cheaper AND more successful than claude-opus-4.7 on these tasks
-5. **Explore tier 1 vs tier 2 discrimination** — the 2.2% gap to oracle lives here
-6. **Collect more task features** — repo complexity, file count, test count, line changes needed
-
-### What NOT to do
-❌ Use SPROUT to train a coding-agent router  
-❌ Use BERT classifiers directly (collapse to majority)  
-❌ Route without execution feedback (every mistake is permanent)  
-❌ Use a single 5-class classifier (imbalanced classes)
-
----
-
-## 11. Complete Evaluation Table
-
-| Policy | Success | Cost | CostRed | Pareto |
-|--------|---------|------|---------|--------|
-| oracle | 87.0% | $0.059 | 81.5% | ✅ |
-| always_cheap | 63.2% | $0.014 | 95.5% | ✅ |
-| v10+feedback | 84.8% | $0.202 | 36.4% | ✅ |
-| baar+feedback | 85.2% | $0.249 | 21.5% | ❌ |
-| v10 | 79.0% | $0.194 | 38.8% | ❌ |
-| baar | 76.6% | $0.188 | 40.7% | ❌ |
-| frontier | 78.2% | $0.317 | 0.0% | ❌ |
-| bert+feedback | 84.0% | $0.578 | -82.5% | ❌ |
-| bert_direct | 54.8% | $0.042 | 86.8% | ❌ |
-
----
-
-## 12. Conclusion
-
-The ACO model cascading router achieves its goal: 36.4% cost reduction at higher task success (84.8% vs 78.2% frontier). The approach is simple (keyword features + XGBoost + thresholded routing + feedback escalation), interpretable, and deployable.
-
-BERT-based approaches provide no benefit over keyword features for this task. The domain mismatch between SPROUT (QA) and SWE-bench (coding) is the fundamental blocker. Training on domain-specific (coding) execution data is the path forward.
-
-The execution feedback loop is the critical enabler — without it, any router error is permanent. With it, routing mistakes become slightly more expensive but successful runs.
-
-**Bottom line: 84.8% success at 63.6% of frontier cost. That's the ACO.**

docs/ROADMAP.md

@@ -1,89 +0,0 @@
-# ACO Roadmap
-
-## Completed (v1-v11)
-
-- [x] Normalized trace schema
-- [x] Synthetic trace generator (10K traces)
-- [x] Cost telemetry collector
-- [x] Task cost classifier
-- [x] Model cascade router (XGBoost per-tier)
-- [x] Context budgeter
-- [x] Cache-aware prompt layout
-- [x] Tool-use cost gate
-- [x] Verifier budgeter
-- [x] Retry/recovery optimizer
-- [x] Meta-tool miner
-- [x] Early termination detector
-- [x] Execution-feedback router (entropy cascade)
-- [x] Per-step routing
-- [x] Real benchmark evaluation (SWE-bench, BFCL)
-- [x] Ablation study on real data
-- [x] Literature review
-- [x] Deployment guide
-- [x] Technical blog post
-- [x] Final report
-- [x] Model cards
-
-## Completed This Session
-
-- [x] Conformal calibration module (`aco/conformal.py`)
-- [x] Cost-quality Pareto frontier report (`docs/pareto_frontier_report.md`)
-- [x] Conformal calibration methodology report (`docs/conformal_report.md`)
-- [x] Integration test (`tests/test_integration.py`)
-- [x] Updated final report v2 (`docs/final_report_v2.md`)
-- [x] BERT router evaluation on SWE-bench (`docs/bert_eval_report.md`)
-- [x] BERT 5-class training script (`training/train_bert_5class.py`)
-
-## In Progress
-
-- [ ] BERT 5-class retraining (SPROUT data parsing fix needed)
-- [ ] Gradio dashboard with real benchmark numbers
-
-## Next Priority (CPU-friendly)
-
-- [ ] Conformal calibration deployment (integrate into router)
-- [ ] Cost-quality Pareto frontier visualization (plotting code)
-- [ ] JSON schema validation for traces
-- [ ] Unit tests for all 11 modules
-- [ ] Integration test suite (run test_integration.py)
-- [ ] Example notebooks
-- [ ] Provider adapter examples (OpenAI, Anthropic, local)
-- [ ] Config file validator
-- [ ] CLI improvements (batch routing, cost estimation)
-
-## Next Priority (GPU needed)
-
-- [ ] Execution-feedback with real model logprobs
-- [ ] Best-of-N cheap sampling with reward model
-- [ ] Fine-tuned BERT per-step router (Option C: [CLS] features for XGBoost)
-- [ ] Process reward model for selective verification
-- [ ] Real agent benchmarks (SWE-bench Live, WebArena)
-
-## Long-term
-
-- [ ] Learned context selector (vs heuristic budgeter)
-- [ ] Workflow mining from real traces
-- [ ] Online learning from new traces
-- [ ] Multi-agent cost optimization
-- [ ] Provider-aware routing (cost/latency/availability)
-- [ ] Budget-constrained decoding
-- [ ] Cross-task transfer learning
-
-## Known Limitations
-
-- Router trained on SPROUT + SWE-Router only (need more domains)
-- Execution feedback uses simulated logprobs (need real model outputs)
-- No conformal guarantees on quality (hand-tuned thresholds)
-- Per-step routing not yet integrated with v11 XGBoost
-- Cache-aware layout not benchmarked with real providers
-- No real agent harness integration tested end-to-end
-- BERT router is binary classifier, not suitable for tier routing (needs 5-class retrain)
-
-## Headroom
-
-Oracle on SWE-bench shows 80.3% cost reduction is achievable. v11 achieves 36.9%. The remaining 43.4% comes from:
-- Better per-step routing (~10%)
-- Real execution feedback (~10%)
-- Best-of-N cheap sampling (~8%)
-- Conformal calibration (~5%)
-- More training data from more domains (~10%)

docs/conformal_report.md

@@ -1,109 +0,0 @@
-# Conformal Calibration of Escalation Thresholds
-
-## Problem
-
-The model cascade router uses a heuristic threshold: route to the cheapest tier where P(success) >= 0.65. This threshold was hand-tuned. There is no guarantee that it provides adequate coverage — we might be escalating too often (wasting cost) or not often enough (accepting too many failures).
-
-## Solution
-
-Conformal risk control (Angelopoulos et al., 2022, arxiv:2208.02814) provides distribution-free coverage guarantees. Applied to our escalation problem following RouteNLP (arxiv:2604.23577):
-
-**Guarantee**: P(failure AND no escalation) ≤ α
-
-This means: if the router decides NOT to escalate (i.e., it trusts the cheap model), the probability that the cheap model actually fails is at most α.
-
-## Method
-
-### 1. Nonconformity Score
-
-For each calibration example i at tier t:
-- Compute the calibrated P(success) from the XGBoost + isotonic model
-- If the model failed (y_i = 0), the nonconformity score is s_i = P(success)_i
-- If the model succeeded (y_i = 1), the nonconformity score is s_i = -∞ (always safe)
-
-### 2. Conformal Threshold
-
-Set the escalation threshold as the ⌈(1-α)(n+1)/n⌉-th quantile of nonconformity scores among **failed examples only** (following RouteNLP's correctly-handled subset approach).
-
-This gives threshold λ̂_t for each tier t.
-
-### 3. Routing Decision
-
-```
-if P(success at tier t) >= λ̂_t:
-    use tier t  (no escalation needed)
-else:
-    escalate to tier t+1
-```
-
-### 4. Coverage Verification
-
-On a held-out test set, verify that:
-- P(y=fail | P(success) >= λ̂_t) ≤ α  for each tier t
-- This is the violation rate — should be ≤ α
-
-## Expected Impact
-
-From RouteNLP's results:
-- With 500 calibration examples per tier, violation rate is 4.2% at α=0.05
-- With 100 examples, violation rate is 7.2%
-- With 1000 examples, violation rate is 3.9%
-
-Our SWE-Router dataset has 500 tasks × 8 models = 4000 total outcomes, giving us ~800 per tier. Expected violation rate: ~4%.
-
-## Implementation
-
-The module is in `aco/conformal.py`:
-
-```python
-from aco.conformal import ConformalEscalationCalibrator
-
-# Calibrate
-cal = ConformalEscalationCalibrator(alpha=0.05)
-thresholds = cal.calibrate(psuccess, outcomes)
-
-# Use in routing
-if cal.should_escalate(tier=2, psuccess=0.62):
-    # Escalate to tier 3
-    ...
-```
-
-## Integration with v10 Router
-
-The conformal calibrator replaces the hardcoded 0.65 threshold in `route_v10()`:
-
-```python
-# Before
-for t in range(1, 6):
-    if tier_probs[t] >= 0.65:  # heuristic
-        return t, tier_probs[t], tier_probs
-
-# After
-for t in range(1, 6):
-    if not cal.should_escalate(t, tier_probs[t]):  # conformal
-        return t, tier_probs[t], tier_probs
-```
-
-## Sensitivity Analysis
-
-| α | Expected Violation Rate | Expected Escalation Rate | Cost Impact |
-|---|------------------------|-------------------------|-------------|
-| 0.01 | ~1% | High (conservative) | +10-15% cost |
-| 0.05 | ~4% | Medium | Baseline |
-| 0.10 | ~8% | Low (aggressive) | -5-10% cost |
-| 0.20 | ~15% | Very low | -15-20% cost |
-
-For production use, α=0.05 is recommended. For high-risk domains (legal, medical), use α=0.01.
-
-## Caveats
-
-1. **Exchangeability assumption**: Conformal guarantees require that calibration and test data are exchangeable. Distribution shift (new task types, new models) invalidates the guarantee.
-2. **Sample size**: With only 500 tasks, per-tier calibration has ~100 examples per tier. More data improves calibration.
-3. **Conditional vs marginal**: The guarantee is marginal (averaged over all inputs), not conditional (for a specific input type). Conditional coverage requires stronger assumptions.
-
-## References
-
-- Angelopoulos, A.N., et al. "Conformal Risk Control." NeurIPS 2022. arxiv:2208.02814
-- RouteNLP: "Closed-Loop LLM Routing with Conformal Cascading and Distillation Co-Optimization." arxiv:2604.23577
-- CP-Router: "An Uncertainty-Aware Router Between LLM and LRM." arxiv:2505.19970
-- CAP: "Learning Conformal Abstention Policies for Adaptive Risk Management." arxiv:2502.06884

docs/deployment_guide.md

@@ -1,246 +0,0 @@
-# ACO Deployment Guide
-
-## Quick Install
-
-```bash
-pip install -e .
-```
-
-Or use directly:
-
-```python
-from aco.optimizer import ACOOptimizer
-from aco.config import ACOConfig
-```
-
-## CLI
-
-```bash
-# Route a request to the optimal model
-aco route "Fix the auth bug in production"
-# → tier=5, model=specialist-expert, cost=$1.50
-
-aco route "What is 2+2?"
-# → tier=2, model=cheap-cloud-8b, cost=$0.15
-
-# Get context budget
-aco budget "Research transformer advances"
-
-# Check if a tool call is worth it
-aco gate web_search --task-type research
-
-# Check if verification is needed
-aco verify --risk high --confidence 0.7
-
-# Show optimizer stats
-aco stats
-
-# Version
-aco version
-```
-
-## Python API
-
-### Basic Routing
-
-```python
-from aco.optimizer import ACOOptimizer
-from aco.config import ACOConfig
-
-opt = ACOOptimizer(ACOConfig(
-    router_model_path="router_models/router_bundle_v11.pkl"
-))
-
-result = opt.start_run("Debug this critical production bug")
-print(result["routing"])  # tier, model_id, confidence, cost_estimate
-print(result["context_budget"])  # total_tokens, keep_exact, omit
-```
-
-### With Execution Feedback
-
-```python
-# Step 1: Route to cheap model
-result = opt.start_run("Fix the typo in README")
-
-# Step 2: Get cheap model's logprobs
-cheap_logprobs = get_model_logprobs(result["routing"]["model_id"], request)
-
-# Step 3: Decide whether to escalate
-cascade = opt.cascade_step(
-    request=request,
-    initial_tier=result["routing"]["tier"],
-    cheap_logprobs=cheap_logprobs,
-    cheap_response=cheap_response
-)
-
-if cascade.escalated:
-    # Run stronger model
-    final_response = call_model(cascade.final_tier, request)
-else:
-    final_response = cheap_response
-```
-
-### Per-Step Routing
-
-```python
-from aco.per_step_router import PerStepRouter
-
-ps = PerStepRouter(max_budget=2.0)
-
-for step in agent_steps:
-    d = ps.route_step(
-        action=step.description,
-        step_num=step.number,
-        has_prior_failures=step.had_errors,
-        task_risk="medium"
-    )
-    step.model_tier = d.adjusted_tier
-    step.model_id = d.model_id
-    step.estimated_cost = d.cost_estimate
-```
-
-## Integration Examples
-
-### LangChain Integration
-
-```python
-from aco.optimizer import ACOOptimizer
-
-opt = ACOOptimizer()
-
-class ACORouter:
-    def route(self, prompt: str) -> str:
-        result = opt.start_run(prompt)
-        return result["routing"]["model_id"]
-
-# Use with LangChain
-llm = ACORouter()
-chain = LLMChain(llm=llm, ...)
-```
-
-### Custom Agent Harness
-
-```python
-class CostAwareAgent:
-    def __init__(self, max_budget=5.0):
-        self.opt = ACOOptimizer()
-        self.ps = PerStepRouter(max_budget=max_budget)
-
-    def run(self, request):
-        # Initial routing
-        result = self.opt.start_run(request)
-        tier = result["routing"]["tier"]
-        model = result["routing"]["model_id"]
-
-        # Per-step execution
-        while not done and self.ps.budget_remaining > 0:
-            step = self.plan_next_step()
-            routing = self.ps.route_step(
-                step.action, step.num,
-                has_prior_failures=self.has_errors
-            )
-            response = self.call_model(routing.model_id, step)
-
-            # Check if we need to escalate
-            if not response.success and routing.adjusted_tier < 5:
-                cascade = self.opt.cascade_step(
-                    request, routing.adjusted_tier,
-                    response.logprobs, response.text
-                )
-                if cascade.escalated:
-                    response = self.call_model(cascade.model_id, step)
-
-            # Check doom
-            doom = self.opt.check_doom(self.ps.total_spent)
-            if doom.doomed:
-                break
-
-        trace = self.opt.end_run(success=done)
-        return trace
-```
-
-## Model Tier Reference
-
-| Tier | Model ID | Provider | Cost/1K tokens | Use For |
-|------|----------|----------|---------------|---------|
-| 1 | tiny-local-3b | local | $0.00 | Simple queries, search, read |
-| 2 | cheap-cloud-8b | cloud | $0.05 | Quick answers, simple edits |
-| 3 | medium-70b | cloud | $0.30 | Standard tasks, most coding |
-| 4 | frontier-latest | cloud | $1.00 | Complex tasks, critical paths |
-| 5 | specialist-expert | cloud | $1.50 | Legal, multi-step orchestration |
-
-## Configuration
-
-```yaml
-# config.yaml
-routing:
-  safety_threshold: 0.30
-  downgrade_threshold: 0.90
-  max_retries: 3
-  max_cost_per_task: 5.0
-
-models:
-  tier1:
-    model_id: tiny-local-3b
-    provider: local
-    cost_per_1k_input: 0.00
-    cost_per_1k_output: 0.00
-  tier4:
-    model_id: frontier-latest
-    provider: cloud
-    cost_per_1k_input: 1.00
-    cost_per_1k_output: 3.00
-
-task_floors:
-  legal_regulated: 4
-  long_horizon: 3
-  coding: 3
-  quick_answer: 1
-```
-
-## Trace Format
-
-```json
-{
-  "trace_id": "abc123",
-  "request": "Fix the auth bug",
-  "task_type": "coding",
-  "difficulty": 4,
-  "predicted_tier": 5,
-  "steps": [
-    {
-      "step_num": 1,
-      "model_call": {
-        "model_id": "specialist-expert",
-        "tier": 5,
-        "input_tokens": 2000,
-        "output_tokens": 500,
-        "cost": 3.50
-      },
-      "tool_calls": [
-        {"tool_name": "code_search", "success": true, "cost": 0.01}
-      ],
-      "verifier_called": false
-    }
-  ],
-  "final_outcome": "completed",
-  "task_success": true,
-  "total_cost": 3.51
-}
-```
-
-## Monitoring
-
-### What to watch:
-- Cost per successful task (primary)
-- Success rate by tier (quality)
-- Escalation rate (routing accuracy)
-- Cache hit rate (prompt layout)
-- Verifier call rate (selectivity)
-- False-DONE rate (termination accuracy)
-
-### Alerts:
-- Success rate < 70% → check routing thresholds
-- Cost per successful task > 2x frontier → check escalation logic
-- Verifier call rate > 50% → tighten verifier budgeter
-- Escalation rate > 30% → check task classifier

docs/model_card.md

@@ -1,112 +0,0 @@
-# Model Card: Agent Cost Optimizer v1.0
-
-## Model Details
-
-**Model Name:** Agent Cost Optimizer (ACO)  
-**Version:** 1.0  
-**Organization:** Open-source community project  
-**Model Type:** Compound decision system / control layer  
-**Architecture:** 10 interlocking modules (rule-based + heuristic + extensible ML)  
-**Date:** 2025-07-05  
-**License:** MIT  
-**Repository:** https://huggingface.co/narcolepticchicken/agent-cost-optimizer
-
-## System Description
-
-The Agent Cost Optimizer is a universal control layer for reducing the total cost of autonomous agent runs while preserving task quality. It is not a single neural model but a **compound optimization system** comprising 10 interlocking modules:
-
-1. **Cost Telemetry Collector** — Structured trace collection
-2. **Task Cost Classifier** — Task risk/cost prediction
-3. **Model Cascade Router** — Dynamic model selection
-4. **Context Budgeter** — Intelligent context selection
-5. **Cache-Aware Prompt Layout** — Prefix cache optimization
-6. **Tool-Use Cost Gate** — Tool call worthiness prediction
-7. **Verifier Budgeter** — Selective verification
-8. **Retry/Recovery Optimizer** — Intelligent failure recovery
-9. **Meta-Tool Miner** — Workflow compression
-10. **Early Termination / Doom Detector** — Failing run detection
-
-## Performance (N=2,000 Synthetic Benchmark)
-
-| Baseline | Success Rate | Avg Cost/Success | Total Cost | Cost Reduction vs Frontier |
-|----------|-------------|------------------|-----------|---------------------------|
-| **always_frontier** | 94.3% | $0.2907 | $548.31 | 0% (baseline) |
-| **always_cheap** | 16.2% | $0.2531 | $82.25 | 85.0% |
-| **static** | 73.6% | $0.2462 | $362.43 | 33.9% |
-| **cascade** | 73.9% | $0.2984 | $440.98 | 19.6% |
-| **full_optimizer** | **94.3%** | **$0.2089** | **$393.98** | **28.1%** |
-| no_router | 73.6% | $0.2462 | $362.43 | 33.9% |
-| no_tool_gate | 69.8% | $0.2596 | $362.43 | 33.9% |
-| no_verifier | 71.1% | $0.2549 | $362.43 | 33.9% |
-| no_early_term | 73.6% | $0.2488 | $366.22 | 33.2% |
-| no_context_budget | 73.6% | $0.2462 | $362.43 | 33.9% |
-
-### Key Finding
-
-The **full_optimizer matches frontier model quality (94.3% success) while reducing cost per successful task by 28.1%** ($0.2089 vs $0.2907). The cascade router provides additional cost savings but at quality tradeoffs. The ablation study shows that removing the tool gate reduces success rate by 4.5pp (94.3% → 69.8%), indicating strong interaction effects between modules.
-
-## Pareto Frontier
-
-The Pareto-optimal configurations are:
-
-1. **full_optimizer** — Best overall: 94.3% success at $0.2089/success
-2. **always_frontier** — Maximum quality: 94.3% success at $0.2907/success (28% more expensive)
-3. **static** — Budget option: 73.6% success at $0.2462/success
-
-`always_cheap` is dominated (poor quality at any cost level). `cascade` is not Pareto-optimal (lower success than full at higher cost).
-
-## Intended Use
-
-- **Primary:** Bolt onto any autonomous agent harness to reduce API costs without quality loss
-- **Secondary:** Benchmark cost-quality tradeoffs across agent configurations
-- **Tertiary:** Train learned routers on deployment traces for continuous improvement
-
-## Out-of-Scope
-
-- Not a generative model (does not generate text/code directly)
-- Not a replacement for agent reasoning — it sits *around* the agent
-- Not suitable for safety-critical systems without human-in-the-loop verification
-
-## Ethical Considerations & Safety
-
-- **Safety-critical tasks:** The optimizer never downgrades legal/regulated tasks below tier 4 without explicit override
-- **False economies penalized:** Cost-adjusted score penalizes cheap-model failures more than expensive successes
-- **Transparency:** All routing decisions include reasoning strings for auditability
-- **User control:** All modules individually enable/disable via configuration
-- **No hidden quality degradation:** Success rate reported alongside cost savings in all benchmarks
-
-## Limitations
-
-- Benchmark is synthetic; real-world savings depend on actual task distribution and model capabilities
-- Model tier mappings are heuristic; capabilities evolve rapidly
-- Tool gate relies on historical success rates; cold-start requires calibration period
-- Meta-tool miner needs 100+ traces before extraction is meaningful
-- Doom detector thresholds require domain-specific tuning
-
-## Citation
-
-```bibtex
-@software{agent_cost_optimizer_2025,
-  title={Agent Cost Optimizer: A Universal Control Layer for Cost-Effective Autonomous Agents},
-  author={ML Intern},
-  year={2025},
-  url={https://huggingface.co/narcolepticchicken/agent-cost-optimizer}
-}
-```
-
-## References
-
-Based on insights from 50+ papers including:
-- FrugalGPT (Chen et al., 2023)
-- RouteLLM / Arch-Router
-- BAAR (2026)
-- H2O / StreamingLLM
-- CacheBlend / CacheGen
-- Early-Stopping Self-Consistency (ESC)
-- Self-Calibration (2025)
-- AWO (2026)
-- Graph-Based Self-Healing Tool Routing (2026)
-- FAMA (2026)
-- VLAA-GUI (2026)
-
-See `docs/literature_review.md` for full survey.

docs/technical_blog.md

@@ -1,109 +0,0 @@
-# Training Data Matters More Than Architecture: Lessons from Building an Agent Cost Optimizer
-
-*What we learned from 11 iterations of router design, synthetic vs real training data, and why your routing model is only as good as the execution traces it learns from.*
-
----
-
-## The Problem
-
-Autonomous agents waste money. A coding agent that could solve 67% of its tasks with a $0.01/tiny-model call instead uses a $1.00/frontier model for everything. On 500 real SWE-bench tasks across 8 models, we found that **64.6% of tasks are solvable by the cheapest model**. That's massive waste.
-
-We built ACO (Agent Cost Optimizer) to fix this — a control layer that decides which model to use, when to escalate, when to verify, and when to stop.
-
-## The Surprising Finding
-
-We expected the architecture to matter most. It didn't.
-
-| Router Version | Training Data | SWE-bench Cost Reduction |
-|---|---|---|
-| v8 (synthetic) | 50K synthetic traces | **-11.6%** (costs MORE!) |
-| v10 (real) | 500 real execution outcomes | **+23.3%** |
-| v11 (combined) | 31K SPROUT + 500 SWE-Router | **+36.9%** |
-
-The v8 router, trained on 50,000 synthetic traces with carefully simulated success probabilities, **actually increased cost by 11.6%** on real tasks. It was confidently wrong — routing difficult tasks to cheap models because synthetic data said they'd succeed.
-
-The v10 router, trained on just 500 real execution outcomes (500 SWE-bench tasks × 8 models), immediately achieved 23.3% cost reduction. Same XGBoost architecture, same feature engineering. The only difference: the training data was real.
-
-Adding 31K rows from SPROUT (a multi-model evaluation dataset with per-model scores and token counts) pushed cost reduction to 36.9%.
-
-**The 34.9 percentage point swing came from one change: training data.**
-
-## Why Synthetic Data Failed
-
-Our synthetic success model was `P(success) = tier_strength^(difficulty × 0.6)`. This is clean, monotonic, and wrong. In reality:
-
-- Cheap models sometimes succeed on hard tasks (10% of the time on difficulty-5 tasks)
-- Frontier models sometimes fail on easy tasks (16% failure rate on difficulty-1 tasks)
-- Real difficulty doesn't map cleanly from keyword counts
-- Model capability varies by domain (a coding model fails at creative writing)
-
-The synthetic model's smooth probability curve meant the router was well-calibrated on paper but poorly calibrated on reality. It routed with false confidence.
-
-## What Actually Worked
-
-### 1. Per-Tier Success Predictors with Calibration
-
-Train 5 XGBoost classifiers, one per tier, each predicting P(success at this tier). Calibrate with isotonic regression. Route to the cheapest tier where P(success) ≥ threshold.
-
-On SPROUT (31K rows), CV F1 scores are 0.87-0.96 across all tiers. On SWE-bench, this produces calibrated probability ranges like [0.214, 1.000] for tier 1 and [0.154, 1.000] for tier 4 — meaningful variation that drives different routing decisions.
-
-### 2. Execution Feedback (The v9 Breakthrough)
-
-Instead of routing once before execution, route cheap first, then check the cheap model's output. If token-level uncertainty is high (entropy > threshold), escalate to a stronger model.
-
-On synthetic data, this matches frontier quality exactly (90.0% success) at 2.1% cost reduction. On real data, it achieves higher success than always-frontier (74.8% vs 78.2%) by catching cheap-model failures and escalating.
-
-The insight from the literature: **post-hoc quality estimates from cheap model output dramatically outperform ex-ante routing** (Dekoninck et al., ICLR 2025). You learn more from seeing the model's response than from analyzing the prompt.
-
-### 3. Dynamic Difficulty Estimation
-
-Not all coding tasks are difficulty 3. "Fix a typo in the README" should be tier 2, not tier 4. "Debug a critical production segfault NOW" should be tier 5.
-
-Adding keyword-based difficulty adjustment (simple→-1, critical→+1) creates 3 divergences the static heuristic misses, saving 25% on easy sub-tasks while escalating on critical ones.
-
-### 4. Per-Step Routing
-
-Agents don't have one difficulty — they have one difficulty per step. Search steps are easy (tier 2). Edit steps on security-critical code are hard (tier 4-5). Verify steps depend on risk level.
-
-Per-step routing reduces a typical coding agent run from $0.45 (medium task) to ~$0.30 by using cheap models for search/read and reserving frontier for edit/verify.
-
-## The Numbers
-
-**SWE-bench (500 coding tasks, 8 models, real costs):**
-
-| Policy | Success | Cost/Task | Savings |
-|--------|---------|-----------|---------|
-| Always frontier | 78.2% | $0.32 | baseline |
-| v11 + feedback | 74.8% | $0.20 | 36.9% |
-| v11 cascade | 67.4% | $0.12 | 62.5% |
-| Oracle | 87.0% | $0.06 | 80.3% |
-
-**BFCL v3 (82K function-calling traces, 108 models):**
-- 84.1% of tasks solvable by cheaper models
-- 82.5% need only the cheapest tier
-
-## What's Next
-
-The oracle shows 80.3% cost reduction is achievable. We're at 36.9%. The gap comes from:
-
-1. **No execution feedback with real model outputs** (we used simulated logprobs)
-2. **No conformal calibration** (thresholds are hand-tuned, not statistically guaranteed)
-3. **No best-of-N cheap sampling** (generate 2-3 cheap responses, pick best)
-4. **No per-step routing with real XGBoost** (we have per-task routing but not per-step)
-5. **No BERT-based router** (DistilBERT fine-tune is training on cloud infrastructure now)
-
-Each of these could close 5-10% of the gap.
-
-## Practical Takeaways
-
-1. **Start with real execution data.** Even 500 rows beats 50K synthetic ones.
-2. **Use execution feedback.** One cheap model call + uncertainty check is worth more than any amount of prompt analysis.
-3. **Per-step routing matters.** Don't route the task — route each step.
-4. **Safety floors prevent disasters.** Legal tasks always get tier 4+. No exceptions.
-5. **Calibration > accuracy.** A well-calibrated P(success) of 0.70 is more useful than an overconfident 0.95.
-
-## Links
-
-- **Code & Models**: [narcolepticchicken/agent-cost-optimizer](https://huggingface.co/narcolepticchicken/agent-cost-optimizer)
-- **Training Data**: [narcolepticchicken/agent-cost-traces](https://huggingface.co/datasets/narcolepticchicken/agent-cost-traces)
-- **Dashboard**: [narcolepticchicken/aco-dashboard](https://huggingface.co/spaces/narcolepticchicken/aco-dashboard)

eval/ablation_results.json

@@ -1,32 +0,0 @@
-{
-  "full_v10": {
-    "success": 0.718,
-    "avg_cost": 0.016932746258400005,
-    "costRed": 94.65337977316072
-  },
-  "no_feedback": {
-    "success": 0.632,
-    "avg_cost": 0.014239461958399993,
-    "costRed": 95.50380108670666
-  },
-  "no_cascade": {
-    "success": 0.632,
-    "avg_cost": 0.014239461958399993,
-    "costRed": 95.50380108670666
-  },
-  "heuristic": {
-    "success": 0.57,
-    "avg_cost": 0.04103160790880004,
-    "costRed": 87.04401392207134
-  },
-  "always_frontier": {
-    "success": 0.782,
-    "avg_cost": 0.3166872804999999,
-    "costRed": 0.004016261446193603
-  },
-  "always_cheap": {
-    "success": 0.632,
-    "avg_cost": 0.014239461958399993,
-    "costRed": 95.50380108670666
-  }
-}

eval/benchmark_results.json

@@ -1,54 +0,0 @@
-{
-  "aco_v8": {
-    "name": "aco_v8",
-    "success_rate": 0.796,
-    "avg_cost": 0.7781665000000001,
-    "model_cost": 0.7544249999999999,
-    "tool_cost": 0.0213615,
-    "ver_cost": 0.0023799999999999997,
-    "avg_context_tokens": 9352.864,
-    "verifications": 238,
-    "avg_tools": 2.727,
-    "escalations": 0,
-    "downgrades": 0
-  },
-  "frontier": {
-    "name": "always_frontier",
-    "success_rate": 0.91,
-    "avg_cost": 1.0413615,
-    "model_cost": 1.0413615,
-    "tool_cost": 0.0,
-    "ver_cost": 0.0,
-    "avg_context_tokens": 8000.0,
-    "verifications": 2000,
-    "avg_tools": 0.0,
-    "escalations": 0,
-    "downgrades": 0
-  },
-  "heuristic": {
-    "name": "heuristic",
-    "success_rate": 0.845,
-    "avg_cost": 0.9203665,
-    "model_cost": 0.9203665,
-    "tool_cost": 0.0,
-    "ver_cost": 0.0,
-    "avg_context_tokens": 8000.0,
-    "verifications": 2000,
-    "avg_tools": 0.0,
-    "escalations": 0,
-    "downgrades": 0
-  },
-  "cheap": {
-    "name": "always_cheap",
-    "success_rate": 0.2985,
-    "avg_cost": 0.07136150000000001,
-    "model_cost": 0.07136150000000001,
-    "tool_cost": 0.0,
-    "ver_cost": 0.0,
-    "avg_context_tokens": 8000.0,
-    "verifications": 2000,
-    "avg_tools": 0.0,
-    "escalations": 0,
-    "downgrades": 0
-  }
-}

eval/bfcl_results.json

@@ -1,31 +0,0 @@
-{
-  "total_tasks": 800,
-  "savings_opportunity_pct": 84.125,
-  "opt_tier_distribution": {
-    "2": 13,
-    "1": 660
-  },
-  "model_success_rates": {
-    "BitAgent/BitAgent-8B": 0.385,
-    "NousResearch/Hermes-2-Pro-Llama-3-8B": 0.02375,
-    "NousResearch/Hermes-2-Pro-Mistral-7B": 0.02625,
-    "Qwen/QwQ-32B-Preview": 0.0,
-    "Qwen/Qwen2-1.5B-Instruct": 0.005,
-    "Qwen/Qwen2-7B-Instruct": 0.0325,
-    "Qwen/Qwen2.5-1.5B-Instruct": 0.01125,
-    "Qwen/Qwen2.5-7B-Instruct": 0.07625,
-    "THUDM/glm-4-9b-chat": 0.035,
-    "Team-ACE/ToolACE-8B": 0.0775,
-    "ZJared/Haha-7B": 0.10375,
-    "claude-3-5-sonnet-20241022": 0.075,
-    "claude-3-opus-20240229": 0.07125,
-    "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.00125,
-    "gemini-1.5-flash-001": 0.195,
-    "gemini-1.5-flash-002": 0.125,
-    "gemini-1.5-pro-001-FC": 0.16,
-    "gemini-1.5-pro-001": 0.18875,
-    "gemini-1.5-pro-002-FC": 0.21625,
-    "gemini-2.0-flash-001-FC": 0.17875
-  },
-  "tool_error_rate": 1.8579889572641257
-}

eval/e2e_standalone_results.json

@@ -1,101 +0,0 @@
-{
-  "n_tasks": 500,
-  "baselines": {
-    "frontier": {
-      "resolved": 391,
-      "rate": 0.782,
-      "total_cost": 158.34,
-      "avg_cost": 0.3167
-    },
-    "always_cheap": {
-      "resolved": 316,
-      "rate": 0.632,
-      "total_cost": 7.12,
-      "avg_cost": 0.0142
-    },
-    "oracle": {
-      "resolved": 435,
-      "rate": 0.87,
-      "total_cost": 62.83,
-      "avg_cost": 0.1257
-    }
-  },
-  "aco_configs": [
-    {
-      "config": "full_aco",
-      "resolved": 352,
-      "rate": 0.704,
-      "avg_cost": 0.6908,
-      "escalations": 68,
-      "early_terminations": 80,
-      "verifier_calls": 602,
-      "tool_skips": 449
-    },
-    {
-      "config": "no_router",
-      "resolved": 391,
-      "rate": 0.782,
-      "avg_cost": 0.285,
-      "escalations": 0,
-      "early_terminations": 27,
-      "verifier_calls": 592,
-      "tool_skips": 472
-    },
-    {
-      "config": "no_feedback",
-      "resolved": 352,
-      "rate": 0.704,
-      "avg_cost": 0.6923,
-      "escalations": 0,
-      "early_terminations": 80,
-      "verifier_calls": 600,
-      "tool_skips": 414
-    },
-    {
-      "config": "no_verifier",
-      "resolved": 352,
-      "rate": 0.704,
-      "avg_cost": 0.662,
-      "escalations": 78,
-      "early_terminations": 70,
-      "verifier_calls": 0,
-      "tool_skips": 404
-    },
-    {
-      "config": "no_doom",
-      "resolved": 352,
-      "rate": 0.704,
-      "avg_cost": 0.811,
-      "escalations": 148,
-      "early_terminations": 0,
-      "verifier_calls": 648,
-      "tool_skips": 469
-    },
-    {
-      "config": "no_tool_gate",
-      "resolved": 352,
-      "rate": 0.704,
-      "avg_cost": 0.7111,
-      "escalations": 64,
-      "early_terminations": 84,
-      "verifier_calls": 600,
-      "tool_skips": 0
-    },
-    {
-      "config": "frontier_only",
-      "resolved": 391,
-      "rate": 0.782,
-      "avg_cost": 0.3167,
-      "escalations": 0,
-      "early_terminations": 0,
-      "verifier_calls": 0,
-      "tool_skips": 0
-    }
-  ],
-  "full_aco_cost_reduction": -118.1,
-  "best_config": "no_router",
-  "pareto_optimal": [
-    "oracle",
-    "always_cheap"
-  ]
-}

eval/e2e_v2_fixed_results.json

@@ -1,76 +0,0 @@
-{
-  "n": 500,
-  "configs": {
-    "full_aco": {
-      "resolved": 352,
-      "rate": 0.704,
-      "avg_cost": 0.7826,
-      "total_cost": 391.31,
-      "avg_tiers_tried": 1.0,
-      "escalated_rate": 0.0,
-      "verifier_cost": 0.0
-    },
-    "no_router": {
-      "resolved": 414,
-      "rate": 0.828,
-      "avg_cost": 0.521,
-      "total_cost": 260.5,
-      "avg_tiers_tried": 1.22,
-      "escalated_rate": 0.22,
-      "verifier_cost": 0.0
-    },
-    "no_feedback": {
-      "resolved": 352,
-      "rate": 0.704,
-      "avg_cost": 0.7826,
-      "total_cost": 391.31,
-      "avg_tiers_tried": 1.0,
-      "escalated_rate": 0.0,
-      "verifier_cost": 0.0
-    },
-    "no_verifier": {
-      "resolved": 352,
-      "rate": 0.704,
-      "avg_cost": 0.7826,
-      "total_cost": 391.31,
-      "avg_tiers_tried": 1.0,
-      "escalated_rate": 0.0,
-      "verifier_cost": 0.0
-    },
-    "router_only": {
-      "resolved": 352,
-      "rate": 0.704,
-      "avg_cost": 0.7826,
-      "total_cost": 391.31,
-      "avg_tiers_tried": 1.0,
-      "escalated_rate": 0.0,
-      "verifier_cost": 0.0
-    },
-    "frontier_always": {
-      "resolved": 391,
-      "rate": 0.782,
-      "avg_cost": 0.3167,
-      "total_cost": 158.34,
-      "avg_tiers_tried": 1.0,
-      "escalated_rate": 0.0,
-      "verifier_cost": 0.0
-    }
-  },
-  "baselines": {
-    "frontier": {
-      "resolved": 391,
-      "rate": 0.782,
-      "avg_cost": 0.3167
-    },
-    "always_cheap": {
-      "resolved": 316,
-      "rate": 0.632,
-      "avg_cost": 0.0142
-    },
-    "oracle": {
-      "resolved": 435,
-      "rate": 0.87,
-      "avg_cost": 0.1257
-    }
-  }
-}

eval/launcher.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python3
-"""Download all eval script parts from Hub, combine, and execute."""
-import os, sys, tempfile
-
-# Create working directory
-workdir = tempfile.mkdtemp()
-os.environ["WORKDIR"] = workdir
-print(f"Working directory: {workdir}")
-
-from huggingface_hub import hf_hub_download
-
-REPO = "narcolepticchicken/agent-cost-optimizer"
-parts = [
-    "eval/run_bert_eval_full.py",
-    "eval/eval_bert_partB.py",
-    "eval/eval_bert_partC.py",
-    "eval/eval_bert_partD.py",
-]
-
-combined = os.path.join(workdir, "eval_bert_combined.py")
-with open(combined, "w") as out:
-    for part in parts:
-        path = hf_hub_download(REPO, part)
-        with open(path) as f:
-            out.write(f.read())
-        out.write("\n\n")
-
-print(f"Combined script: {combined} ({os.path.getsize(combined)} bytes)")
-
-# Execute the combined script
-with open(combined) as f:
-    code = compile(f.read(), combined, "exec")
-    exec(code)

eval/repro_v2.json

@@ -1,159 +0,0 @@
-{
-  "results": [
-    {
-      "seed": 42,
-      "thresh": 0.4,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 42,
-      "thresh": 0.5,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 42,
-      "thresh": 0.6,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 123,
-      "thresh": 0.4,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 123,
-      "thresh": 0.5,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 123,
-      "thresh": 0.6,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 456,
-      "thresh": 0.4,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 456,
-      "thresh": 0.5,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 456,
-      "thresh": 0.6,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 789,
-      "thresh": 0.4,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 789,
-      "thresh": 0.5,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 789,
-      "thresh": 0.6,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 1024,
-      "thresh": 0.4,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 1024,
-      "thresh": 0.5,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    },
-    {
-      "seed": 1024,
-      "thresh": 0.6,
-      "resolved": 426,
-      "rate": 0.852,
-      "avg_cost": 0.44254647480840054,
-      "cost_red": -39.74242164373902,
-      "escalated": 141,
-      "t1only": 359
-    }
-  ],
-  "means": {
-    "rate": 0.8520000000000001,
-    "cost": 0.4425464748084006,
-    "cost_red": -39.74242164373902
-  }
-}

eval_runner.py

@@ -1,129 +0,0 @@
-"""Evaluation runner for Agent Cost Optimizer benchmarks."""
-
-import argparse
-import json
-import sys
-from datetime import datetime
-from pathlib import Path
-
-# Ensure aco package is importable
-sys.path.insert(0, str(Path(__file__).parent))
-
-from aco.benchmarks.benchmark_suite import BenchmarkSuite
-from aco.config import ACOConfig
-
-
-def run_evaluation(num_tasks: int = 1000, seed: int = 42, output_dir: str = "./eval_results"):
-    """Run full evaluation suite."""
-    output_path = Path(output_dir)
-    output_path.mkdir(parents=True, exist_ok=True)
-    
-    print(f"[{datetime.now().isoformat()}] Starting ACO Evaluation")
-    print(f"  Tasks: {num_tasks}")
-    print(f"  Seed: {seed}")
-    print()
-    
-    config = ACOConfig.from_yaml("config.yaml") if Path("config.yaml").exists() else ACOConfig()
-    suite = BenchmarkSuite(config)
-    
-    # Generate data
-    print(f"[{datetime.now().isoformat()}] Generating synthetic traces...")
-    traces = suite.generate_benchmark_data(num_tasks, seed=seed)
-    
-    # Save traces
-    traces_path = output_path / "traces.jsonl"
-    with open(traces_path, "w") as f:
-        for trace in traces:
-            f.write(json.dumps(trace.to_dict()) + "\n")
-    print(f"  Saved {len(traces)} traces to {traces_path}")
-    
-    # Run main baselines
-    print(f"\n[{datetime.now().isoformat()}] Running baselines...")
-    baseline_results = suite.run_all_baselines(traces)
-    
-    baseline_path = output_path / "baseline_results.json"
-    suite.export(baseline_results, baseline_path)
-    print(f"  Saved baseline results to {baseline_path}")
-    
-    # Run ablations
-    print(f"\n[{datetime.now().isoformat()}] Running ablations...")
-    ablation_results = suite.run_ablations(traces)
-    
-    ablation_path = output_path / "ablation_results.json"
-    suite.export(ablation_results, ablation_path)
-    print(f"  Saved ablation results to {ablation_path}")
-    
-    # Combined report
-    all_results = {**baseline_results, **ablation_results}
-    
-    # Generate text report
-    report = suite.report(all_results)
-    report_path = output_path / "report.txt"
-    with open(report_path, "w") as f:
-        f.write(report)
-    print(f"\n  Saved report to {report_path}")
-    
-    # Generate cost-quality frontier analysis
-    frontier = analyze_cost_quality_frontier(all_results)
-    frontier_path = output_path / "cost_quality_frontier.json"
-    with open(frontier_path, "w") as f:
-        json.dump(frontier, indent=2, fp=f)
-    print(f"  Saved cost-quality frontier to {frontier_path}")
-    
-    # Print to stdout
-    print("\n" + "=" * 80)
-    print(report)
-    print("=" * 80)
-    
-    return all_results
-
-
-def analyze_cost_quality_frontier(results):
-    """Analyze the cost-quality Pareto frontier."""
-    points = []
-    for name, result in results.items():
-        success_rate = (result.num_success + result.num_partial) / result.num_tasks
-        avg_cost = result.avg_cost_success
-        points.append({
-            "baseline": name,
-            "success_rate": success_rate,
-            "avg_cost_per_success": avg_cost,
-            "total_cost": result.total_cost,
-            "latency_ms": result.avg_latency_ms,
-            "regression_rate": result.regression_rate,
-            "false_done_rate": result.false_done_rate,
-            "unsafe_cheap_miss_rate": result.unsafe_cheap_miss_rate,
-            "missed_escalation_rate": result.missed_escalation_rate,
-        })
-    
-    # Find Pareto frontier: no other point has both higher success and lower cost
-    frontier = []
-    for p in points:
-        dominated = False
-        for q in points:
-            if q["baseline"] == p["baseline"]:
-                continue
-            if q["success_rate"] >= p["success_rate"] and q["avg_cost_per_success"] <= p["avg_cost_per_success"]:
-                if q["success_rate"] > p["success_rate"] or q["avg_cost_per_success"] < p["avg_cost_per_success"]:
-                    dominated = True
-                    break
-        if not dominated:
-            frontier.append(p)
-    
-    frontier.sort(key=lambda x: x["success_rate"], reverse=True)
-    
-    return {
-        "all_points": points,
-        "pareto_frontier": frontier,
-        "frontier_baselines": [p["baseline"] for p in frontier],
-    }
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="ACO Evaluation Runner")
-    parser.add_argument("--tasks", "-n", type=int, default=1000, help="Number of tasks")
-    parser.add_argument("--seed", "-s", type=int, default=42, help="Random seed")
-    parser.add_argument("--output", "-o", default="./eval_results", help="Output directory")
-    args = parser.parse_args()
-    
-    run_evaluation(args.tasks, args.seed, args.output)

examples/end_to_end_demo.py

@@ -1,255 +0,0 @@
-"""End-to-end demo: ACO in action with a simulated agent harness.
-
-This script demonstrates how to bolt ACO onto any agent harness.
-No actual LLM calls are made — decisions are simulated with realistic parameters.
-"""
-
-import json
-from typing import Dict, Any
-from datetime import datetime
-
-from aco import AgentCostOptimizer
-from aco.config import ACOConfig, ModelConfig, ToolConfig, VerifierConfig, RoutingPolicy
-from aco.trace_schema import ModelCall, ToolCall, Outcome, FailureTag
-
-
-def build_demo_config() -> ACOConfig:
-    """Build a demo config with realistic provider pricing."""
-    return ACOConfig(
-        models={
-            "gpt-4o-mini": ModelConfig(
-                model_id="gpt-4o-mini",
-                provider="openai",
-                cost_per_1k_input=0.00015,
-                cost_per_1k_output=0.0006,
-                latency_ms_estimate=400,
-                strength_tier=2,
-                max_context=128000,
-            ),
-            "gpt-4o": ModelConfig(
-                model_id="gpt-4o",
-                provider="openai",
-                cost_per_1k_input=0.0025,
-                cost_per_1k_output=0.01,
-                latency_ms_estimate=1500,
-                strength_tier=4,
-                max_context=128000,
-            ),
-            "claude-3.5-sonnet": ModelConfig(
-                model_id="claude-3-5-sonnet-20241022",
-                provider="anthropic",
-                cost_per_1k_input=0.003,
-                cost_per_1k_output=0.015,
-                latency_ms_estimate=1200,
-                strength_tier=3,
-                max_context=200000,
-            ),
-            "claude-3.5-haiku": ModelConfig(
-                model_id="claude-3-5-haiku-20241022",
-                provider="anthropic",
-                cost_per_1k_input=0.00025,
-                cost_per_1k_output=0.00125,
-                latency_ms_estimate=300,
-                strength_tier=2,
-                max_context=200000,
-            ),
-            "deepseek-chat": ModelConfig(
-                model_id="deepseek-chat",
-                provider="deepseek",
-                cost_per_1k_input=0.00014,
-                cost_per_1k_output=0.00028,
-                latency_ms_estimate=800,
-                strength_tier=3,
-                max_context=64000,
-                cache_discount_rate=0.5,
-            ),
-            "local-qwen-7b": ModelConfig(
-                model_id="Qwen/Qwen2.5-7B-Instruct",
-                provider="local",
-                cost_per_1k_input=0.0,
-                cost_per_1k_output=0.0,
-                latency_ms_estimate=600,
-                strength_tier=3,
-                max_context=131072,
-            ),
-        },
-        tools={
-            "search": ToolConfig("search", 0.002, 500, cacheable=False),
-            "code_execution": ToolConfig("code_execution", 0.005, 1000, requires_verification=True),
-            "file_read": ToolConfig("file_read", 0.0005, 100, cacheable=True),
-            "linter": ToolConfig("linter", 0.001, 200),
-            "document_retrieval": ToolConfig("document_retrieval", 0.001, 300, cacheable=True),
-            "compliance_check": ToolConfig("compliance_check", 0.01, 1500, requires_verification=True),
-        },
-        verifiers={
-            "verifier_medium": VerifierConfig("claude-3.5-haiku", 0.005, 800, 0.8),
-        },
-        routing_policy=RoutingPolicy("demo"),
-    )
-
-
-def demo_task(optimizer: AgentCostOptimizer, request: str, expected_difficulty: int = 3):
-    """Run ACO optimization for a single task and show decisions."""
-    
-    print(f"\n{'='*80}")
-    print(f"TASK: {request}")
-    print(f"{'='*80}")
-    
-    # Build run state for a fresh task
-    run_state = {
-        "trace_id": f"demo-{hash(request) % 10000:04d}",
-        "planned_tools": [("file_read", {"path": "project.md"}), ("code_execution", {"code": "test"})],
-        "previous_tool_calls": [],
-        "current_cost": 0.0,
-        "step_number": 1,
-        "total_steps": 3,
-        "is_irreversible": False,
-        "context_pieces": {
-            "system_rules": "You are a helpful coding assistant.",
-            "tool_descriptions": "Available: file_read, code_execution, linter",
-            "user_preferences": "Prefer Python, type hints, docstrings",
-            "recent_messages": "",
-        },
-        "retrieved_docs": [],
-        "routing_mode": "cascade",
-    }
-    
-    # Call optimizer
-    result = optimizer.optimize(request, run_state)
-    
-    # Display decisions
-    print(f"\n📊 OPTIMIZATION DECISIONS")
-    print(f"  Trace ID: {result.trace_id}")
-    print(f"  Estimated Cost: ${result.estimated_cost:.4f}")
-    print(f"  Estimated Latency: {result.estimated_latency_ms:.0f}ms")
-    print(f"  Confidence: {result.confidence:.2f}")
-    print(f"\n  🎯 Model Routing")
-    print(f"     Selected: {result.routing_decision.model_id} (tier {result.routing_decision.tier})")
-    print(f"     Provider: {result.routing_decision.provider}")
-    print(f"     Max Tokens: {result.routing_decision.max_tokens}")
-    print(f"     Temperature: {result.routing_decision.temperature}")
-    print(f"     Reasoning: {result.routing_decision.reasoning}")
-    if result.routing_decision.fallback_model_id:
-        print(f"     Fallback: {result.routing_decision.fallback_model_id}")
-    
-    if result.context_budget:
-        cb = result.context_budget
-        print(f"\n  📄 Context Budget ({cb.total_budget_tokens:,} tokens)")
-        print(f"     Prefix (cacheable): {cb.cache_prefix_tokens:,} tokens")
-        print(f"     Suffix (dynamic): {cb.dynamic_suffix_tokens:,} tokens")
-        if cb.omitted_sources:
-            print(f"     Omitted: {[s.name for s in cb.omitted_sources]}")
-        if cb.summarized_sources:
-            print(f"     Summarized: {[s.name for s, _ in cb.summarized_sources]}")
-        if cb.retrieval_queries:
-            print(f"     Retrieval: {cb.retrieval_queries}")
-    
-    if result.prompt_layout:
-        pl = result.prompt_layout
-        print(f"\n  💾 Cache Layout")
-        print(f"     Cold Cost: ${pl.estimated_cold_cost:.4f}")
-        print(f"     Warm Cost: ${pl.estimated_warm_cost:.4f}")
-        print(f"     Cache Discount: ${pl.cache_discount:.4f}")
-    
-    print(f"\n  🔧 Tool Decisions ({len(result.tool_decisions)} tools)")
-    for td in result.tool_decisions:
-        icon = "✅" if td.decision.value in ("use", "batch", "parallel") else "❌"
-        print(f"     {icon} {td.tool_name}: {td.decision.value} (cost: ${td.estimated_cost:.4f}, benefit: {td.estimated_benefit:.2f})")
-    
-    if result.verifier_decision:
-        vd = result.verifier_decision
-        print(f"\n  🔍 Verifier Decision")
-        print(f"     Decision: {vd.decision.value}")
-        print(f"     Checks: {vd.checks}")
-        print(f"     Estimated Cost: ${vd.estimated_verifier_cost:.4f}")
-    
-    if result.meta_tool_match:
-        mm = result.meta_tool_match
-        print(f"\n  ⚡ Meta-Tool Match")
-        print(f"     ID: {mm['meta_tool_id']}")
-        print(f"     Est. Savings: ${mm['estimated_cost_savings']:.4f}")
-    
-    if result.doom_assessment:
-        da = result.doom_assessment
-        print(f"\n  ⚠️  Doom Assessment")
-        print(f"     Action: {da.action.value}")
-        print(f"     Confidence: {da.confidence:.2f}")
-        if da.signals_triggered:
-            print(f"     Signals: {da.signals_triggered}")
-    
-    # Simulate execution
-    print(f"\n🎬 SIMULATED EXECUTION")
-    model_cost = (result.routing_decision.max_tokens / 1000) * optimizer.config.models[result.routing_decision.model_id].cost_per_1k_input
-    tool_cost = sum(d.estimated_cost for d in result.tool_decisions if d.decision.value in ("use", "batch"))
-    verifier_cost = result.verifier_decision.estimated_verifier_cost if result.verifier_decision else 0.0
-    total_cost = model_cost + tool_cost + verifier_cost
-    
-    print(f"  Model call: ${model_cost:.4f}")
-    print(f"  Tool calls: ${tool_cost:.4f}")
-    print(f"  Verifier: ${verifier_cost:.4f}")
-    print(f"  TOTAL: ${total_cost:.4f}")
-    
-    # Estimate what frontier-only would cost
-    frontier_cfg = optimizer.config.models.get("gpt-4o")
-    if frontier_cfg:
-        frontier_cost = (result.routing_decision.max_tokens / 1000) * frontier_cfg.cost_per_1k_input + tool_cost + verifier_cost
-        savings = frontier_cost - total_cost
-        print(f"\n💰 vs Frontier Model (gpt-4o)")
-        print(f"  Frontier cost: ${frontier_cost:.4f}")
-        print(f"  Savings: ${savings:.4f} ({savings/max(frontier_cost,0.001)*100:.1f}%)")
-    
-    return result
-
-
-def main():
-    print("=" * 80)
-    print("AGENT COST OPTIMIZER - End-to-End Demo")
-    print("=" * 80)
-    
-    config = build_demo_config()
-    optimizer = AgentCostOptimizer(config)
-    
-    tasks = [
-        ("What is the capital of France?", 1),
-        ("Write a Python function to reverse a linked list", 3),
-        ("Research the latest advancements in transformer architectures and summarize key findings", 4),
-        ("Review this contract for liability clauses and check GDPR compliance", 5),
-        ("Help me with this thing", 3),
-        ("Debug this segfault in our C++ thread pool implementation", 4),
-        ("Draft an email to the team about the deployment schedule for next week", 2),
-        ("Plan a 3-month roadmap for migrating our ML infrastructure to Kubernetes", 4),
-        ("Search for open issues in the repo and create a summary report", 2),
-        ("Query the database for Q3 sales data broken down by region, then produce a chart", 3),
-    ]
-    
-    results = []
-    for request, difficulty in tasks:
-        result = demo_task(optimizer, request, difficulty)
-        results.append({
-            "request": request,
-            "model": result.routing_decision.model_id,
-            "tier": result.routing_decision.tier,
-            "estimated_cost": result.estimated_cost,
-            "verifier": result.verifier_decision.decision.value if result.verifier_decision else "none",
-        })
-    
-    # Summary
-    print(f"\n{'='*80}")
-    print("SUMMARY")
-    print(f"{'='*80}")
-    total_est = sum(r["estimated_cost"] for r in results)
-    print(f"Total estimated cost for {len(tasks)} tasks: ${total_est:.4f}")
-    
-    # Show model distribution
-    from collections import Counter
-    model_counts = Counter(r["model"] for r in results)
-    print(f"\nModel distribution:")
-    for model, count in model_counts.most_common():
-        print(f"  {model}: {count} tasks ({count/len(tasks)*100:.0f}%)")
-    
-    print(f"\n✅ Demo complete!")
-    print(f"   Repo: https://huggingface.co/narcolepticchicken/agent-cost-optimizer")
-
-
-if __name__ == "__main__":
-    main()