Delete router_models/bert_5class/*, router_models/bert_router/*, router_models/baarbundle.pkl, router_models/productionbundle.pkl, router_models/routerbundle*.pkl, router_models/*_success.json, router_models/v6_eval*, training/train_router_v[1-9]*.py, training/train_router_full.py, training/train_bert_5class*.py, training/train_baar_router.py, eval/eval_bert_part*.py, eval/eval_bert_router.py, eval/run_bert_eval*.py, eval/bert_results.json, eval/*v[8-9]*.json, eval/sw*.json

eval/benchmark_v9_final.json

@@ -1,73 +0,0 @@
-{
-  "frontier": {
-    "success": 0.8996666666666666,
-    "cost": 1.0
-  },
-  "v8": {
-    "success": 0.8373333333333334,
-    "cost": 0.9151666666666726
-  },
-  "v9_e2.0_lc0.05": {
-    "success": 0.8996666666666666,
-    "cost": 0.9785500000000054,
-    "escalated": 238,
-    "ent_thr": 2.0,
-    "lc_thr": 0.05
-  },
-  "v9_e2.0_lc0.1": {
-    "success": 0.8996666666666666,
-    "cost": 0.9785500000000054,
-    "escalated": 238,
-    "ent_thr": 2.0,
-    "lc_thr": 0.1
-  },
-  "v9_e2.0_lc0.15": {
-    "success": 0.8996666666666666,
-    "cost": 0.9785500000000054,
-    "escalated": 238,
-    "ent_thr": 2.0,
-    "lc_thr": 0.15
-  },
-  "v9_e2.5_lc0.05": {
-    "success": 0.8576666666666667,
-    "cost": 0.9261833333333388,
-    "escalated": 94,
-    "ent_thr": 2.5,
-    "lc_thr": 0.05
-  },
-  "v9_e2.5_lc0.1": {
-    "success": 0.8576666666666667,
-    "cost": 0.9261833333333388,
-    "escalated": 94,
-    "ent_thr": 2.5,
-    "lc_thr": 0.1
-  },
-  "v9_e2.5_lc0.15": {
-    "success": 0.8576666666666667,
-    "cost": 0.9261833333333388,
-    "escalated": 94,
-    "ent_thr": 2.5,
-    "lc_thr": 0.15
-  },
-  "v9_e3.0_lc0.05": {
-    "success": 0.836,
-    "cost": 0.9132500000000056,
-    "escalated": 51,
-    "ent_thr": 3.0,
-    "lc_thr": 0.05
-  },
-  "v9_e3.0_lc0.1": {
-    "success": 0.834,
-    "cost": 0.9130833333333389,
-    "escalated": 49,
-    "ent_thr": 3.0,
-    "lc_thr": 0.1
-  },
-  "v9_e3.0_lc0.15": {
-    "success": 0.834,
-    "cost": 0.9130833333333389,
-    "escalated": 49,
-    "ent_thr": 3.0,
-    "lc_thr": 0.15
-  }
-}

eval/benchmark_v9_results.json

@@ -1,32 +0,0 @@
-{
-  "always_frontier": {
-    "success": 0.901,
-    "avg_cost": 1.0,
-    "unsafe": 0.0
-  },
-  "always_cheap": {
-    "success": 0.29133333333333333,
-    "avg_cost": 0.049999999999999954,
-    "unsafe": 0.7203333333333334
-  },
-  "heuristic": {
-    "success": 0.834,
-    "avg_cost": 0.8830500000000058,
-    "unsafe": 0.11466666666666667
-  },
-  "v8_router": {
-    "success": 0.8406666666666667,
-    "avg_cost": 0.8864666666666725,
-    "unsafe": 0.0
-  },
-  "v9_feedback": {
-    "success": 0.86,
-    "avg_cost": 0.9651333333333396,
-    "unsafe": 0.0
-  },
-  "oracle": {
-    "success": 0.9986666666666667,
-    "avg_cost": 0.4130333333333279,
-    "unsafe": 0.0
-  }
-}

eval/bert_results.json

@@ -1,4 +0,0 @@
-{
-  "eval_accuracy": 0.9446835987946621,
-  "eval_f1": 0.971529854879805
-}

eval/eval_bert_part2.py

@@ -1,40 +0,0 @@
-
-# ── Part 2: Load data and BERT model ──
-
-from datasets import load_dataset
-from transformers import AutoTokenizer, AutoModelForSequenceClassification
-import torch
-
-print("\n[1] Loading SWE-Router data...")
-traces = defaultdict(dict)
-for model in MODELS:
-    try:
-        ds = load_dataset(f'SWE-Router/swebench-verified-{model}', split='test')
-        for row in ds:
-            iid = row['instance_id']
-            traces[iid][model] = {
-                'resolved': row['resolved'],
-                'cost': float(row['instance_cost']),
-                'problem': row['problem_statement'],
-            }
-        print(f"  {model}: loaded")
-    except Exception as e:
-        print(f"  {model}: FAILED - {e}")
-
-print(f"  Total tasks: {len(traces)}")
-
-print("\n[2] Loading BERT router...")
-REPO = "narcolepticchicken/agent-cost-optimizer"
-tokenizer = AutoTokenizer.from_pretrained(f"{REPO}", subfolder="router_models/bert_router")
-bert_model = AutoModelForSequenceClassification.from_pretrained(f"{REPO}", subfolder="router_models/bert_router")
-bert_model.eval()
-print(f"  BERT model loaded, num_labels={bert_model.config.num_labels}")
-
-print("\n[3] Loading v11 XGBoost router...")
-from huggingface_hub import hf_hub_download
-v11_path = hf_hub_download(REPO, "router_models/router_bundle_v11.pkl")
-v11_bundle = pickle.load(open(v11_path, "rb"))
-v11_tier_clfs = {int(k):v for k,v in v11_bundle["tier_clfs"].items()}
-v11_tier_calibs = {int(k):v for k,v in v11_bundle["tier_calibrators"].items()}
-v11_feat_keys = v11_bundle["feat_keys"]
-print(f"  v11 loaded, features={len(v11_feat_keys)}")

eval/eval_bert_part3.py

@@ -1,140 +0,0 @@
-
-# ── Part 3: Run evaluations ──
-
-def route_bert(problem_text):
-    """Route using BERT classifier."""
-    inputs = tokenizer(problem_text, truncation=True, max_length=512, return_tensors="pt")
-    with torch.no_grad():
-        logits = bert_model(**inputs).logits
-    pred_class = torch.argmax(logits, dim=-1).item()
-    tier = pred_class + 1  # classes are 0-4 → tiers 1-5
-    probs = torch.softmax(logits, dim=-1)[0]
-    confidence = float(probs[pred_class])
-    return tier, confidence
-
-def route_v11(problem_text):
-    """Route using v11 XGBoost + isotonic calibration."""
-    feats = extract_features(problem_text)
-    feat_vec = np.array([float(feats.get(k, 0.0)) for k in v11_feat_keys], dtype=np.float32).reshape(1,-1)
-    tier_probs = {}
-    for t in range(1, 6):
-        p_raw = v11_tier_clfs[t].predict_proba(feat_vec)[0, 1]
-        p_cal = float(v11_tier_calibs[t].transform([p_raw])[0])
-        tier_probs[t] = p_cal
-    # Find cheapest tier with P(success) >= 0.65
-    for t in range(1, 6):
-        if tier_probs[t] >= 0.65:
-            return t, tier_probs[t], tier_probs
-    return 5, tier_probs[5], tier_probs
-
-TASK_FLOOR = {"quick_answer":1,"coding":3,"research":3,"document_drafting":2,
-              "legal_regulated":4,"tool_heavy":2,"retrieval_heavy":2,"long_horizon":3,"unknown_ambiguous":3}
-
-def classify_task(text):
-    r = text.lower()
-    if any(k in r for k in ["contract","legal","compliance","gdpr","privacy"]): return "legal_regulated"
-    if any(k in r for k in ["debug","fix","bug","implement","refactor","code","function"]): return "coding"
-    if any(k in r for k in ["research","find sources","literature","investigate"]): return "research"
-    if any(k in r for k in ["search","fetch","api","query","database"]): return "tool_heavy"
-    if any(k in r for k in ["plan","roadmap","orchestrate","migrate","deploy"]): return "long_horizon"
-    if any(k in r for k in ["draft","write","compose","document"]): return "document_drafting"
-    if any(k in r for k in ["what is","explain","define","briefly"]): return "quick_answer"
-    return "unknown_ambiguous"
-
-policies = defaultdict(lambda: {"success":0,"cost":0.0,"n":0})
-
-print("\n[4] Evaluating all policies on SWE-Router...")
-for iid, model_results in traces.items():
-    problem = next(iter(model_results.values()))['problem']
-    task_type = classify_task(problem)
-    floor = TASK_FLOOR.get(task_type, 2)
-
-    # Oracle
-    resolved = [(m, r) for m, r in model_results.items() if r['resolved']]
-    if resolved:
-        cheapest = min(resolved, key=lambda x: TIER_COST.get(MODEL_TIER[x[0]], 1.0))
-        policies['oracle']['success'] += 1
-        policies['oracle']['cost'] += cheapest[1]['cost']
-    else:
-        policies['oracle']['cost'] += min(r['cost'] for r in model_results.values())
-    policies['oracle']['n'] += 1
-
-    # Always frontier
-    f_model = 'claude-opus-4.7'
-    if f_model in model_results:
-        policies['frontier']['success'] += int(model_results[f_model]['resolved'])
-        policies['frontier']['cost'] += model_results[f_model]['cost']
-    policies['frontier']['n'] += 1
-
-    # BERT router
-    bert_tier, bert_conf = route_bert(problem)
-    bert_tier = max(bert_tier, floor)  # enforce safety floor
-    m_bert = TIER_TO_SWE.get(bert_tier, 'claude-opus-4.7')
-    if m_bert in model_results:
-        policies['bert']['success'] += int(model_results[m_bert]['resolved'])
-        policies['bert']['cost'] += model_results[m_bert]['cost']
-    else:
-        policies['bert']['success'] += int(model_results.get('claude-opus-4.7',{}).get('resolved',0))
-        policies['bert']['cost'] += model_results.get('claude-opus-4.7',{}).get('cost',0.3)
-    policies['bert']['n'] += 1
-
-    # v11 XGBoost
-    v11_tier, v11_conf, v11_probs = route_v11(problem)
-    v11_tier = max(v11_tier, floor)  # enforce safety floor
-    m_v11 = TIER_TO_SWE.get(v11_tier, 'claude-opus-4.7')
-    if m_v11 in model_results:
-        policies['v11_xgboost']['success'] += int(model_results[m_v11]['resolved'])
-        policies['v11_xgboost']['cost'] += model_results[m_v11]['cost']
-    else:
-        policies['v11_xgboost']['success'] += int(model_results.get('claude-opus-4.7',{}).get('resolved',0))
-        policies['v11_xgboost']['cost'] += model_results.get('claude-opus-4.7',{}).get('cost',0.3)
-    policies['v11_xgboost']['n'] += 1
-
-    # BERT + feedback (escalate on failure)
-    if m_bert in model_results and model_results[m_bert]['resolved']:
-        policies['bert_feedback']['success'] += 1
-        policies['bert_feedback']['cost'] += model_results[m_bert]['cost']
-    else:
-        # Escalate one tier
-        up_tier = min(bert_tier + 1, 5)
-        m_up = TIER_TO_SWE.get(up_tier, 'claude-opus-4.7')
-        if m_up in model_results and model_results[m_up]['resolved']:
-            policies['bert_feedback']['success'] += 1
-            policies['bert_feedback']['cost'] += model_results.get(m_bert,{}).get('cost',0.01)
-            policies['bert_feedback']['cost'] += model_results[m_up]['cost']
-        else:
-            # Last resort: frontier
-            if f_model in model_results and model_results[f_model]['resolved']:
-                policies['bert_feedback']['success'] += 1
-                policies['bert_feedback']['cost'] += model_results.get(m_bert,{}).get('cost',0.01)
-                policies['bert_feedback']['cost'] += model_results[f_model]['cost']
-            else:
-                policies['bert_feedback']['cost'] += model_results.get(m_bert,{}).get('cost',0.01)
-    policies['bert_feedback']['n'] += 1
-
-    # v11 + feedback (escalate on failure)
-    if m_v11 in model_results and model_results[m_v11]['resolved']:
-        policies['v11_feedback']['success'] += 1
-        policies['v11_feedback']['cost'] += model_results[m_v11]['cost']
-    else:
-        up_tier = min(v11_tier + 1, 5)
-        m_up = TIER_TO_SWE.get(up_tier, 'claude-opus-4.7')
-        if m_up in model_results and model_results[m_up]['resolved']:
-            policies['v11_feedback']['success'] += 1
-            policies['v11_feedback']['cost'] += model_results.get(m_v11,{}).get('cost',0.01)
-            policies['v11_feedback']['cost'] += model_results[m_up]['cost']
-        else:
-            if f_model in model_results and model_results[f_model]['resolved']:
-                policies['v11_feedback']['success'] += 1
-                policies['v11_feedback']['cost'] += model_results.get(m_v11,{}).get('cost',0.01)
-                policies['v11_feedback']['cost'] += model_results[f_model]['cost']
-            else:
-                policies['v11_feedback']['cost'] += model_results.get(m_v11,{}).get('cost',0.01)
-    policies['v11_feedback']['n'] += 1
-
-    # Always cheap
-    c_model = 'deepseek-v4-flash'
-    if c_model in model_results:
-        policies['always_cheap']['success'] += int(model_results[c_model]['resolved'])
-        policies['always_cheap']['cost'] += model_results[c_model]['cost']
-    policies['always_cheap']['n'] += 1

eval/eval_bert_part4.py

@@ -1,61 +0,0 @@
-
-# ── Part 4: Print results ──
-
-print(f"\n\n{'='*70}")
-print("BERT vs XGBoost ROUTER COMPARISON ON SWE-BENCH")
-print(f"{'='*70}")
-
-fr = policies['frontier']
-fr_cost = fr['cost'] / fr['n']
-fr_succ = fr['success'] / fr['n']
-
-print(f"\n{'Policy':<20} {'Success':>10} {'AvgCost':>10} {'CostRed':>10}")
-print("-"*52)
-order = ['oracle','bert_feedback','v11_feedback','bert','v11_xgboost','frontier','always_cheap']
-for name in order:
-    if name not in policies:
-        continue
-    r = policies[name]
-    sr = r['success']/r['n'] if r['n'] > 0 else 0
-    ac = r['cost']/r['n'] if r['n'] > 0 else 0
-    cr = (1 - ac/fr_cost)*100 if fr_cost > 0 else 0
-    print(f"{name:<20} {sr:>10.3f} {ac:>10.4f} {cr:>9.1f}%")
-
-# BERT tier distribution
-print(f"\n\nBERT tier distribution:")
-bert_tiers = defaultdict(int)
-for iid, model_results in traces.items():
-    problem = next(iter(model_results.values()))['problem']
-    t, c = route_bert(problem)
-    bert_tiers[t] += 1
-for t in sorted(bert_tiers):
-    print(f"  Tier {t}: {bert_tiers[t]}")
-
-# Quality gap analysis
-print(f"\n\nQuality gap vs frontier:")
-for name in ['bert','bert_feedback','v11_xgboost','v11_feedback']:
-    r = policies[name]
-    sr = r['success']/r['n'] if r['n'] > 0 else 0
-    gap = (sr - fr_succ) * 100
-    print(f"  {name}: {gap:+.1f}pp vs frontier")
-
-# Save results
-results = {}
-for name, r in policies.items():
-    sr = r['success']/r['n'] if r['n'] > 0 else 0
-    ac = r['cost']/r['n'] if r['n'] > 0 else 0
-    cr = (1 - ac/fr_cost)*100 if fr_cost > 0 else 0
-    results[name] = {"success": round(sr, 4), "avg_cost": round(ac, 4), "costRed": round(cr, 1)}
-
-# Upload results to Hub
-from huggingface_hub import HfApi
-api = HfApi()
-import tempfile, json
-with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
-    json.dump(results, f, indent=2)
-    api.upload_file(path_or_fileobj=f.name, path_in_repo="eval/bert_vs_xgboost_results.json",
-                    repo_id="narcolepticchicken/agent-cost-optimizer", repo_type="model")
-    os.unlink(f.name)
-
-print(f"\nResults saved to eval/bert_vs_xgboost_results.json")
-print("DONE!")

eval/eval_bert_partB.py

@@ -1,68 +0,0 @@
-
-# ── Load models ──
-print("\n[2] Loading BERT router from Hub...")
-from transformers import AutoTokenizer, AutoModelForSequenceClassification
-import torch
-
-REPO = "narcolepticchicken/agent-cost-optimizer"
-tokenizer = AutoTokenizer.from_pretrained(REPO, subfolder="router_models/bert_router")
-bert_model = AutoModelForSequenceClassification.from_pretrained(REPO, subfolder="router_models/bert_router")
-bert_model.eval()
-print(f"  BERT loaded, num_labels={bert_model.config.num_labels}")
-
-# BERT is binary (success/fail) — we'll use it as a per-tier success predictor
-# by prepending "Tier X:" to the input text
-
-print("\n[3] Loading v10 XGBoost router...")
-from huggingface_hub import hf_hub_download
-import pickle
-v10_path = hf_hub_download(REPO, "router_models/router_bundle_v10_fixed.pkl")
-v10_bundle = pickle.load(open(v10_path, "rb"))
-print(f"  v10 bundle keys: {list(v10_bundle.keys())}")
-
-# v10 may have different structure — inspect
-if "tier_clfs" in v10_bundle:
-    v10_tier_clfs = {int(k):v for k,v in v10_bundle["tier_clfs"].items()}
-    v10_tier_calibs = {int(k):v for k,v in v10_bundle["tier_calibrators"].items()}
-    v10_feat_keys = v10_bundle["feat_keys"]
-    print(f"  v10 loaded, features={len(v10_feat_keys)}")
-    HAS_V10 = True
-else:
-    HAS_V10 = False
-    print(f"  v10 bundle structure: {type(v10_bundle)}")
-
-# ── Routing functions ──
-def route_bert(problem_text):
-    """BERT binary classifier: predict success probability at each tier.
-    Route to cheapest tier where P(success) > 0.5."""
-    tier_probs = {}
-    for tier in range(1, 6):
-        prompt = f"[Tier {tier}] {problem_text}"
-        inputs = tokenizer(prompt, truncation=True, max_length=512, return_tensors="pt")
-        with torch.no_grad():
-            logits = bert_model(**inputs).logits
-        probs = torch.softmax(logits, dim=-1)[0]
-        # Binary: class 1 = success
-        tier_probs[tier] = float(probs[1]) if bert_model.config.num_labels == 2 else float(probs[tier-1])
-    
-    # Route to cheapest tier with P(success) >= 0.5
-    for t in range(1, 6):
-        if tier_probs[t] >= 0.5:
-            return t, tier_probs[t], tier_probs
-    return 5, tier_probs[5], tier_probs
-
-def route_v10(problem_text):
-    """v10 XGBoost cascade router."""
-    if not HAS_V10:
-        return 4, 0.5, {t:0.5 for t in range(1,6)}
-    feats = extract_features(problem_text)
-    feat_vec = np.array([float(feats.get(k, 0.0)) for k in v10_feat_keys], dtype=np.float32).reshape(1,-1)
-    tier_probs = {}
-    for t in range(1, 6):
-        p_raw = v10_tier_clfs[t].predict_proba(feat_vec)[0, 1]
-        p_cal = float(v10_tier_calibs[t].transform([p_raw])[0])
-        tier_probs[t] = p_cal
-    for t in range(1, 6):
-        if tier_probs[t] >= 0.65:
-            return t, tier_probs[t], tier_probs
-    return 5, tier_probs[5], tier_probs

eval/eval_bert_partC.py

@@ -1,95 +0,0 @@
-
-# ── Evaluate ──
-policies = defaultdict(lambda: {"success":0,"cost":0.0,"n":0})
-
-print("\n[4] Evaluating all policies...")
-for idx, (iid, model_results) in enumerate(traces.items()):
-    if idx % 100 == 0:
-        print(f"  Progress: {idx}/{len(traces)}")
-    problem = next(iter(model_results.values()))['problem']
-    task_type = classify_task(problem)
-    floor = TASK_FLOOR.get(task_type, 2)
-    f_model = 'claude-opus-4.7'
-
-    # Oracle
-    resolved = [(m, r) for m, r in model_results.items() if r['resolved']]
-    if resolved:
-        cheapest = min(resolved, key=lambda x: TIER_COST.get(MODEL_TIER[x[0]], 1.0))
-        policies['oracle']['success'] += 1
-        policies['oracle']['cost'] += cheapest[1]['cost']
-    else:
-        policies['oracle']['cost'] += min(r['cost'] for r in model_results.values())
-    policies['oracle']['n'] += 1
-
-    # Always frontier
-    if f_model in model_results:
-        policies['frontier']['success'] += int(model_results[f_model]['resolved'])
-        policies['frontier']['cost'] += model_results[f_model]['cost']
-    policies['frontier']['n'] += 1
-
-    # BERT (per-tier success prediction with cascade)
-    bert_tier, bert_conf, bert_probs = route_bert(problem)
-    bert_tier = max(bert_tier, floor)
-    m_bert = TIER_TO_SWE.get(bert_tier, f_model)
-    if m_bert in model_results:
-        policies['bert']['success'] += int(model_results[m_bert]['resolved'])
-        policies['bert']['cost'] += model_results[m_bert]['cost']
-    else:
-        policies['bert']['success'] += int(model_results.get(f_model,{}).get('resolved',0))
-        policies['bert']['cost'] += model_results.get(f_model,{}).get('cost',0.3)
-    policies['bert']['n'] += 1
-
-    # v10 XGBoost
-    v10_tier, v10_conf, v10_probs = route_v10(problem)
-    v10_tier = max(v10_tier, floor)
-    m_v10 = TIER_TO_SWE.get(v10_tier, f_model)
-    if m_v10 in model_results:
-        policies['v10_xgboost']['success'] += int(model_results[m_v10]['resolved'])
-        policies['v10_xgboost']['cost'] += model_results[m_v10]['cost']
-    else:
-        policies['v10_xgboost']['success'] += int(model_results.get(f_model,{}).get('resolved',0))
-        policies['v10_xgboost']['cost'] += model_results.get(f_model,{}).get('cost',0.3)
-    policies['v10_xgboost']['n'] += 1
-
-    # BERT + feedback (escalate on failure)
-    if m_bert in model_results and model_results[m_bert]['resolved']:
-        policies['bert_feedback']['success'] += 1
-        policies['bert_feedback']['cost'] += model_results[m_bert]['cost']
-    else:
-        up_tier = min(bert_tier + 1, 5)
-        m_up = TIER_TO_SWE.get(up_tier, f_model)
-        if m_up in model_results and model_results[m_up]['resolved']:
-            policies['bert_feedback']['success'] += 1
-            policies['bert_feedback']['cost'] += model_results.get(m_bert,{}).get('cost',0.01) + model_results[m_up]['cost']
-        elif f_model in model_results and model_results[f_model]['resolved']:
-            policies['bert_feedback']['success'] += 1
-            policies['bert_feedback']['cost'] += model_results.get(m_bert,{}).get('cost',0.01) + model_results[f_model]['cost']
-        else:
-            policies['bert_feedback']['cost'] += model_results.get(m_bert,{}).get('cost',0.01)
-    policies['bert_feedback']['n'] += 1
-
-    # v10 + feedback
-    if m_v10 in model_results and model_results[m_v10]['resolved']:
-        policies['v10_feedback']['success'] += 1
-        policies['v10_feedback']['cost'] += model_results[m_v10]['cost']
-    else:
-        up_tier = min(v10_tier + 1, 5)
-        m_up = TIER_TO_SWE.get(up_tier, f_model)
-        if m_up in model_results and model_results[m_up]['resolved']:
-            policies['v10_feedback']['success'] += 1
-            policies['v10_feedback']['cost'] += model_results.get(m_v10,{}).get('cost',0.01) + model_results[m_up]['cost']
-        elif f_model in model_results and model_results[f_model]['resolved']:
-            policies['v10_feedback']['success'] += 1
-            policies['v10_feedback']['cost'] += model_results.get(m_v10,{}).get('cost',0.01) + model_results[f_model]['cost']
-        else:
-            policies['v10_feedback']['cost'] += model_results.get(m_v10,{}).get('cost',0.01)
-    policies['v10_feedback']['n'] += 1
-
-    # Always cheap
-    c_model = 'deepseek-v4-flash'
-    if c_model in model_results:
-        policies['always_cheap']['success'] += int(model_results[c_model]['resolved'])
-        policies['always_cheap']['cost'] += model_results[c_model]['cost']
-    policies['always_cheap']['n'] += 1
-
-print(f"  Progress: {len(traces)}/{len(traces)} - DONE")

eval/eval_bert_partD.py

@@ -1,81 +0,0 @@
-
-# ── Results ──
-fr = policies['frontier']
-fr_cost = fr['cost'] / fr['n']
-fr_succ = fr['success'] / fr['n']
-
-print(f"\n\n{'='*70}")
-print("BERT vs XGBoost ROUTER COMPARISON ON SWE-BENCH")
-print(f"{'='*70}")
-print(f"\n{'Policy':<20} {'Success':>10} {'AvgCost':>10} {'CostRed':>10}")
-print("-"*52)
-for name in ['oracle','bert_feedback','v11_feedback','bert','v11_xgboost','frontier','always_cheap']:
-    # Map v11 names to v10 if v11 not available
-    actual_name = name
-    if name == 'v11_xgboost' and 'v11_xgboost' not in policies and 'v10_xgboost' in policies:
-        actual_name = 'v10_xgboost'
-    if name == 'v11_feedback' and 'v11_feedback' not in policies and 'v10_feedback' in policies:
-        actual_name = 'v10_feedback'
-    if actual_name not in policies:
-        continue
-    r = policies[actual_name]
-    sr = r['success']/r['n'] if r['n'] > 0 else 0
-    ac = r['cost']/r['n'] if r['n'] > 0 else 0
-    cr = (1-ac/fr_cost)*100 if fr_cost > 0 else 0
-    print(f"{name:<20} {sr:>10.3f} {ac:>10.4f} {cr:>9.1f}%")
-
-print(f"\nQuality gap vs frontier:")
-for name in ['bert','bert_feedback','v11_xgboost','v11_feedback']:
-    actual_name = name
-    if name == 'v11_xgboost' and 'v11_xgboost' not in policies and 'v10_xgboost' in policies:
-        actual_name = 'v10_xgboost'
-    if name == 'v11_feedback' and 'v11_feedback' not in policies and 'v10_feedback' in policies:
-        actual_name = 'v10_feedback'
-    if actual_name not in policies:
-        continue
-    r = policies[actual_name]
-    sr = r['success']/r['n'] if r['n'] > 0 else 0
-    gap = (sr - fr_succ) * 100
-    print(f"  {name}: {gap:+.1f}pp vs frontier")
-
-# BERT tier distribution
-print(f"\nBERT tier distribution (first 100 tasks):")
-bert_tiers = defaultdict(int)
-bert_probs_dist = defaultdict(list)
-for iid, model_results in list(traces.items())[:100]:
-    problem = next(iter(model_results.values()))['problem']
-    t, conf, probs = route_bert(problem)
-    bert_tiers[t] += 1
-    for tier, p in probs.items():
-        bert_probs_dist[tier].append(p)
-print(f"  Tier routing counts:")
-for t in sorted(bert_tiers):
-    print(f"    Tier {t}: {bert_tiers[t]}")
-print(f"  Per-tier P(success) stats:")
-for t in sorted(bert_probs_dist):
-    ps = bert_probs_dist[t]
-    print(f"    Tier {t}: mean={np.mean(ps):.3f}, std={np.std(ps):.3f}, min={np.min(ps):.3f}, max={np.max(ps):.3f}")
-
-# Save results to local file and try to upload to Hub
-results = {}
-for name, r in policies.items():
-    sr = r['success']/r['n'] if r['n'] > 0 else 0
-    ac = r['cost']/r['n'] if r['n'] > 0 else 0
-    cr = (1-ac/fr_cost)*100 if fr_cost > 0 else 0
-    results[name] = {"success": round(sr, 4), "avg_cost": round(ac, 4), "costRed": round(cr, 1)}
-
-import tempfile, json
-try:
-    from huggingface_hub import HfApi
-    api = HfApi()
-    with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
-        json.dump(results, f, indent=2)
-        api.upload_file(path_or_fileobj=f.name, path_in_repo="eval/bert_vs_xgboost_results.json",
-                        repo_id="narcolepticchicken/agent-cost-optimizer", repo_type="model")
-        os.unlink(f.name)
-    print(f"\nResults uploaded to eval/bert_vs_xgboost_results.json on Hub")
-except Exception as e:
-    print(f"\nCould not upload to Hub: {e}")
-    print(f"Results JSON:\n{json.dumps(results, indent=2)}")
-
-print("\nDONE!")

eval/eval_bert_router.py

@@ -1,61 +0,0 @@
-"""Evaluate BERT router on SWE-Router and compare with v11 XGBoost."""
-import json, os, sys, random, pickle, math
-import numpy as np
-from collections import defaultdict
-
-# ── Constants ──
-MODELS = ['claude-opus-4.7','gpt-5-mini','gpt-5-nano','gpt-5.2',
-          'gemini-2.5-pro','gemini-3-pro','deepseek-v3.2','deepseek-v4-flash']
-
-MODEL_TIER = {
-    'deepseek-v4-flash': 1, 'gpt-5-nano': 1,
-    'gpt-5-mini': 2, 'deepseek-v3.2': 2,
-    'gemini-2.5-pro': 3,
-    'claude-opus-4.7': 4, 'gpt-5.2': 4,
-    'gemini-3-pro': 5,
-}
-TIER_COST = {1:0.01, 2:0.05, 3:0.15, 4:0.30, 5:0.50}
-TIER_TO_SWE = {
-    1: 'deepseek-v4-flash', 2: 'gpt-5-mini',
-    3: 'gemini-2.5-pro', 4: 'claude-opus-4.7', 5: 'gemini-3-pro',
-}
-
-# ── Feature extraction (same as training) ──
-CODE_KW = ["python","javascript","code","function","bug","debug","refactor","implement","test",
-           "compile","runtime","segfault","thread","async","class","module","import","error","traceback"]
-LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"]
-RESEARCH_KW = ["research","investigate","compare","analyze","survey","paper"]
-TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
-CRITICAL_KW = ["critical","production","urgent","emergency","live","deployed","safety","security"]
-SIMPLE_KW = ["typo","simple","quick","brief","minor","small","easy","trivial","just"]
-LONG_KW = ["plan","project","roadmap","orchestrate","migrate","pipeline","deploy","architecture"]
-MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"]
-
-def extract_features(problem_text):
-    r = problem_text.lower()
-    feats = {
-        'req_len': len(problem_text), 'num_words': len(problem_text.split()),
-        'has_code': int(any(k in r for k in CODE_KW)),
-        'n_code': sum(1 for k in CODE_KW if k in r),
-        'has_legal': int(any(k in r for k in LEGAL_KW)),
-        'has_research': int(any(k in r for k in RESEARCH_KW)),
-        'has_tool': int(any(k in r for k in TOOL_KW)),
-        'has_critical': int(any(k in r for k in CRITICAL_KW)),
-        'has_simple': int(any(k in r for k in SIMPLE_KW)),
-        'has_long': int(any(k in r for k in LONG_KW)),
-        'has_math': int(any(k in r for k in MATH_KW)),
-        'has_error_msg': int('error' in r or 'traceback' in r or 'exception' in r),
-        'has_file_path': int('/' in r),
-        'n_lines': problem_text.count('\n') + 1,
-        'has_version': int('version' in r or 'update' in r or 'upgrade' in r),
-        'has_add': int('add' in r or 'new' in r or 'create' in r),
-        'has_fix': int('fix' in r or 'bug' in r or 'issue' in r or 'broken' in r),
-        'has_change': int('change' in r or 'modify' in r or 'update' in r),
-        'has_remove': int('remove' in r or 'delete' in r or 'drop' in r),
-        'has_test': int('test' in r or 'spec' in r or 'assert' in r),
-        'has_doc': int('doc' in r or 'readme' in r or 'comment' in r),
-    }
-    return feats
-
-print("BERT Router Evaluation on SWE-Router")
-print("="*60)

eval/run_bert_eval.py

@@ -1,47 +0,0 @@
-#!/usr/bin/env python3
-"""Evaluate BERT router on SWE-Router and compare with v11 XGBoost.
-Combines eval_bert_router.py + eval_bert_part2.py + eval_bert_part3.py + eval_bert_part4.py
-"""
-import json, os, sys, random, pickle, math
-import numpy as np
-from collections import defaultdict
-
-MODELS = ['claude-opus-4.7','gpt-5-mini','gpt-5-nano','gpt-5.2',
-          'gemini-2.5-pro','gemini-3-pro','deepseek-v3.2','deepseek-v4-flash']
-MODEL_TIER = {
-    'deepseek-v4-flash': 1, 'gpt-5-nano': 1,
-    'gpt-5-mini': 2, 'deepseek-v3.2': 2,
-    'gemini-2.5-pro': 3,
-    'claude-opus-4.7': 4, 'gpt-5.2': 4,
-    'gemini-3-pro': 5,
-}
-TIER_COST = {1:0.01, 2:0.05, 3:0.15, 4:0.30, 5:0.50}
-TIER_TO_SWE = {1:'deepseek-v4-flash',2:'gpt-5-mini',3:'gemini-2.5-pro',4:'claude-opus-4.7',5:'gemini-3-pro'}
-CODE_KW = ["python","javascript","code","function","bug","debug","refactor","implement","test","compile","runtime","segfault","thread","async","class","module","import","error","traceback"]
-LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"]
-RESEARCH_KW = ["research","investigate","compare","analyze","survey","paper"]
-TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
-CRITICAL_KW = ["critical","production","urgent","emergency","live","deployed","safety","security"]
-SIMPLE_KW = ["typo","simple","quick","brief","minor","small","easy","trivial","just"]
-LONG_KW = ["plan","project","roadmap","orchestrate","migrate","pipeline","deploy","architecture"]
-MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"]
-
-def extract_features(pt):
-    r = pt.lower()
-    return {'req_len':len(pt),'num_words':len(pt.split()),'has_code':int(any(k in r for k in CODE_KW)),'n_code':sum(1 for k in CODE_KW if k in r),'has_legal':int(any(k in r for k in LEGAL_KW)),'has_research':int(any(k in r for k in RESEARCH_KW)),'has_tool':int(any(k in r for k in TOOL_KW)),'has_critical':int(any(k in r for k in CRITICAL_KW)),'has_simple':int(any(k in r for k in SIMPLE_KW)),'has_long':int(any(k in r for k in LONG_KW)),'has_math':int(any(k in r for k in MATH_KW)),'has_error_msg':int('error' in r or 'traceback' in r),'has_file_path':int('/' in r),'n_lines':pt.count('\n')+1,'has_version':int('version' in r or 'update' in r),'has_add':int('add' in r or 'new' in r),'has_fix':int('fix' in r or 'bug' in r),'has_change':int('change' in r or 'modify' in r),'has_remove':int('remove' in r),'has_test':int('test' in r or 'spec' in r),'has_doc':int('doc' in r)}
-
-TASK_FLOOR = {"quick_answer":1,"coding":3,"research":3,"document_drafting":2,"legal_regulated":4,"tool_heavy":2,"retrieval_heavy":2,"long_horizon":3,"unknown_ambiguous":3}
-
-def classify_task(text):
-    r = text.lower()
-    if any(k in r for k in ["contract","legal","compliance","gdpr","privacy"]): return "legal_regulated"
-    if any(k in r for k in ["debug","fix","bug","implement","refactor","code","function"]): return "coding"
-    if any(k in r for k in ["research","find sources","literature","investigate"]): return "research"
-    if any(k in r for k in ["search","fetch","api","query","database"]): return "tool_heavy"
-    if any(k in r for k in ["plan","roadmap","orchestrate","migrate","deploy"]): return "long_horizon"
-    if any(k in r for k in ["draft","write","compose","document"]): return "document_drafting"
-    if any(k in r for k in ["what is","explain","define","briefly"]): return "quick_answer"
-    return "unknown_ambiguous"
-
-print("BERT Router Evaluation on SWE-Router")
-print("="*60)

eval/run_bert_eval_full.py

@@ -1,66 +0,0 @@
-"""Evaluate BERT router on SWE-Router and compare with v11 XGBoost."""
-import json, os, sys, random, pickle, math
-import numpy as np
-from collections import defaultdict
-
-MODELS = ['claude-opus-4.7','gpt-5-mini','gpt-5-nano','gpt-5.2',
-          'gemini-2.5-pro','gemini-3-pro','deepseek-v3.2','deepseek-v4-flash']
-MODEL_TIER = {'deepseek-v4-flash':1,'gpt-5-nano':1,'gpt-5-mini':2,'deepseek-v3.2':2,
-              'gemini-2.5-pro':3,'claude-opus-4.7':4,'gpt-5.2':4,'gemini-3-pro':5}
-TIER_COST = {1:0.01,2:0.05,3:0.15,4:0.30,5:0.50}
-TIER_TO_SWE = {1:'deepseek-v4-flash',2:'gpt-5-mini',3:'gemini-2.5-pro',4:'claude-opus-4.7',5:'gemini-3-pro'}
-TASK_FLOOR = {"quick_answer":1,"coding":3,"research":3,"document_drafting":2,
-              "legal_regulated":4,"tool_heavy":2,"retrieval_heavy":2,"long_horizon":3,"unknown_ambiguous":3}
-CODE_KW = ["python","javascript","code","function","bug","debug","refactor","implement","test",
-           "compile","runtime","segfault","thread","async","class","module","import","error","traceback"]
-LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"]
-RESEARCH_KW = ["research","investigate","compare","analyze","survey","paper"]
-TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
-CRITICAL_KW = ["critical","production","urgent","emergency","live","deployed","safety","security"]
-SIMPLE_KW = ["typo","simple","quick","brief","minor","small","easy","trivial","just"]
-LONG_KW = ["plan","project","roadmap","orchestrate","migrate","pipeline","deploy","architecture"]
-MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"]
-
-def extract_features(pt):
-    r = pt.lower()
-    return {'req_len':len(pt),'num_words':len(pt.split()),
-        'has_code':int(any(k in r for k in CODE_KW)),'n_code':sum(1 for k in CODE_KW if k in r),
-        'has_legal':int(any(k in r for k in LEGAL_KW)),'has_research':int(any(k in r for k in RESEARCH_KW)),
-        'has_tool':int(any(k in r for k in TOOL_KW)),'has_critical':int(any(k in r for k in CRITICAL_KW)),
-        'has_simple':int(any(k in r for k in SIMPLE_KW)),'has_long':int(any(k in r for k in LONG_KW)),
-        'has_math':int(any(k in r for k in MATH_KW)),'has_error_msg':int('error' in r or 'traceback' in r),
-        'has_file_path':int('/' in r),'n_lines':pt.count('\n')+1,
-        'has_version':int('version' in r or 'update' in r),'has_add':int('add' in r or 'new' in r),
-        'has_fix':int('fix' in r or 'bug' in r),'has_change':int('change' in r or 'modify' in r),
-        'has_remove':int('remove' in r),'has_test':int('test' in r or 'spec' in r),
-        'has_doc':int('doc' in r or 'readme' in r)}
-
-def classify_task(text):
-    r = text.lower()
-    if any(k in r for k in ["contract","legal","compliance","gdpr","privacy"]): return "legal_regulated"
-    if any(k in r for k in ["debug","fix","bug","implement","refactor","code","function"]): return "coding"
-    if any(k in r for k in ["research","find sources","literature","investigate"]): return "research"
-    if any(k in r for k in ["search","fetch","api","query","database"]): return "tool_heavy"
-    if any(k in r for k in ["plan","roadmap","orchestrate","migrate","deploy"]): return "long_horizon"
-    if any(k in r for k in ["draft","write","compose","document"]): return "document_drafting"
-    if any(k in r for k in ["what is","explain","define","briefly"]): return "quick_answer"
-    return "unknown_ambiguous"
-
-print("BERT Router Evaluation on SWE-Router")
-print("="*60)
-
-# ── Load SWE-Router data ──
-print("\n[1] Loading SWE-Router data...")
-from datasets import load_dataset
-traces = defaultdict(dict)
-for model in MODELS:
-    try:
-        ds = load_dataset(f'SWE-Router/swebench-verified-{model}', split='test')
-        for row in ds:
-            iid = row['instance_id']
-            traces[iid][model] = {'resolved': row['resolved'], 'cost': float(row['instance_cost']),
-                                  'problem': row['problem_statement']}
-        print(f"  {model}: loaded")
-    except Exception as e:
-        print(f"  {model}: FAILED - {e}")
-print(f"  Total tasks: {len(traces)}")

eval/run_bert_eval_launcher.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-"""Download eval script parts from Hub, combine, and run."""
-import os, subprocess
-from huggingface_hub import hf_hub_download
-
-REPO = "narcolepticchicken/agent-cost-optimizer"
-parts = ["eval/run_bert_eval_full.py", "eval/eval_bert_partB.py", "eval/eval_bert_partC.py", "eval/eval_bert_partD.py"]
-
-combined = "/app/eval_bert_combined.py"
-with open(combined, "w") as out:
-    for part in parts:
-        path = hf_hub_download(REPO, part)
-        with open(path) as f:
-            out.write(f.read())
-        out.write("\n\n")
-
-print(f"Combined script: {combined}")
-print(f"Size: {os.path.getsize(combined)} bytes")
-
-# Run it
-import subprocess, sys
-result = subprocess.run([sys.executable, combined], capture_output=False)
-sys.exit(result.returncode)

eval/swe_bench_results.json

@@ -1,27 +0,0 @@
-{
-  "always_frontier": {
-    "success": 0.782,
-    "avg_cost": 0.3166872804999999,
-    "n": 500
-  },
-  "always_cheap": {
-    "success": 0.632,
-    "avg_cost": 0.014239461958399993,
-    "n": 500
-  },
-  "aco_v8": {
-    "success": 0.756,
-    "avg_cost": 0.29145885426799994,
-    "n": 500
-  },
-  "oracle": {
-    "success": 0.87,
-    "avg_cost": 0.05456037742200003,
-    "n": 500
-  },
-  "aco_v9_feedback": {
-    "success": 0.826,
-    "avg_cost": 0.4847524783679998,
-    "n": 500
-  }
-}

eval/swe_v10_results.json

@@ -1,38 +0,0 @@
-{
-  "v10_direct": {
-    "success": 0.766,
-    "avg_cost": 0.1878,
-    "costRed": 40.7
-  },
-  "v10_cascade": {
-    "success": 0.756,
-    "avg_cost": 0.1767,
-    "costRed": 44.2
-  },
-  "v10_feedback": {
-    "success": 0.848,
-    "avg_cost": 0.2014,
-    "costRed": 36.4
-  },
-  "v8_synthetic": {
-    "success": 0.658,
-    "avg_cost": 0.3534,
-    "costRed": -11.6
-  },
-  "frontier": {
-    "success": 0.782,
-    "avg_cost": 0.3167,
-    "costRed": 0.0
-  },
-  "oracle": {
-    "success": 0.87,
-    "avg_cost": 0.0624,
-    "costRed": 80.3
-  },
-  "always_cheap": {
-    "success": 0.632,
-    "avg_cost": 0.0142,
-    "costRed": 95.5
-  },
-  "key_finding": "v10 trained on REAL SWE-Router data achieves 36-44% cost reduction vs 8% for synthetic-trained v8. v10_feedback achieves HIGHER success than always-frontier (84.8% vs 78.2%) at 36.4% cost reduction."
-}

eval/v8_results.json

@@ -1,69 +0,0 @@
-{
-  "always_frontier": {
-    "success": 0.8935,
-    "avg_cost": 1.0,
-    "unsafe_rate": 0.026,
-    "false_done": 0.0805,
-    "tier_dist": {
-      "4": 2000
-    }
-  },
-  "always_cheap": {
-    "success": 0.238,
-    "avg_cost": 0.04999999999999823,
-    "unsafe_rate": 0.762,
-    "false_done": 0.0,
-    "tier_dist": {
-      "1": 2000
-    }
-  },
-  "heuristic_static": {
-    "success": 0.825,
-    "avg_cost": 0.929275000000004,
-    "unsafe_rate": 0.058,
-    "false_done": 0.117,
-    "tier_dist": {
-      "4": 674,
-      "3": 646,
-      "5": 443,
-      "2": 237
-    }
-  },
-  "oracle": {
-    "success": 1.0,
-    "avg_cost": 0.46504999999999164,
-    "unsafe_rate": 0.0,
-    "false_done": 0.0,
-    "tier_dist": {
-      "3": 647,
-      "1": 476,
-      "2": 567,
-      "4": 258,
-      "5": 52
-    }
-  },
-  "v8_dynamic+ML": {
-    "success": 0.82,
-    "avg_cost": 0.922775000000004,
-    "unsafe_rate": 0.0615,
-    "false_done": 0.1185,
-    "tier_dist": {
-      "3": 754,
-      "4": 538,
-      "5": 471,
-      "2": 237
-    }
-  },
-  "v8_dynamic_only": {
-    "success": 0.82,
-    "avg_cost": 0.922775000000004,
-    "unsafe_rate": 0.0615,
-    "false_done": 0.1185,
-    "tier_dist": {
-      "3": 754,
-      "4": 538,
-      "5": 471,
-      "2": 237
-    }
-  }
-}

router_models/bert_5class/config.json

@@ -1,42 +0,0 @@
-{
-  "activation": "gelu",
-  "architectures": [
-    "DistilBertForSequenceClassification"
-  ],
-  "attention_dropout": 0.1,
-  "bos_token_id": null,
-  "dim": 768,
-  "dropout": 0.1,
-  "dtype": "float32",
-  "eos_token_id": null,
-  "hidden_dim": 3072,
-  "id2label": {
-    "0": "LABEL_0",
-    "1": "LABEL_1",
-    "2": "LABEL_2",
-    "3": "LABEL_3",
-    "4": "LABEL_4"
-  },
-  "initializer_range": 0.02,
-  "label2id": {
-    "LABEL_0": 0,
-    "LABEL_1": 1,
-    "LABEL_2": 2,
-    "LABEL_3": 3,
-    "LABEL_4": 4
-  },
-  "max_position_embeddings": 512,
-  "model_type": "distilbert",
-  "n_heads": 12,
-  "n_layers": 6,
-  "pad_token_id": 0,
-  "problem_type": "single_label_classification",
-  "qa_dropout": 0.1,
-  "seq_classif_dropout": 0.2,
-  "sinusoidal_pos_embds": false,
-  "tie_weights_": true,
-  "tie_word_embeddings": true,
-  "transformers_version": "5.8.0",
-  "use_cache": false,
-  "vocab_size": 30522
-}

router_models/bert_5class/metadata.json

@@ -1,47 +0,0 @@
-{
-  "model": "distilbert-base-uncased",
-  "num_labels": 5,
-  "task": "5-class optimal tier prediction",
-  "training_dataset": "CARROT-LLM-Routing/SPROUT",
-  "n_training_samples": 27871,
-  "correctness_threshold": 0.7,
-  "tier_mapping": {
-    "wxai-granite-3-2b-instruct-8k-max-tokens": 1,
-    "wxai-llama-3-2-1b-instruct": 1,
-    "wxai-llama-3-2-3b-instruct": 1,
-    "wxai-llama-3-1-8b-instruct": 2,
-    "wxai-granite-3-8b-instruct-8k-max-tokens": 2,
-    "openai-gpt-4o-mini": 2,
-    "wxai-mixtral-8x7b-instruct-v01": 3,
-    "aws-titan-text-premier-v1": 3,
-    "openai-gpt-4o": 4,
-    "wxai-llama-3-1-70b-instruct": 4,
-    "wxai-llama-3-3-70b-instruct": 4,
-    "aws-claude-3-5-sonnet-v1": 4,
-    "wxai-llama-3-405b-instruct": 5
-  },
-  "max_seq_len": 512,
-  "epochs": 5,
-  "learning_rate": 2e-05,
-  "final_metrics": {
-    "eval_loss": 0.6286362409591675,
-    "eval_accuracy": 0.7694543106231837,
-    "eval_off_by_one": 0.9228285437520181,
-    "eval_acc_tier1": 0.9991586032814472,
-    "eval_acc_tier2": 0.012605042016806723,
-    "eval_acc_tier3": 0.0,
-    "eval_acc_tier4": 0.017391304347826087,
-    "eval_acc_tier5": 0.0,
-    "eval_runtime": 12.4974,
-    "eval_samples_per_second": 247.811,
-    "eval_steps_per_second": 3.921,
-    "epoch": 5.0
-  },
-  "label_distribution": {
-    "1": 23972,
-    "2": 4727,
-    "4": 1119,
-    "5": 636,
-    "3": 514
-  }
-}

router_models/bert_5class/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3865791a1f5061c928e8252f6badb06956b94f91e4f11ba3130e15e2590ff5cf
-size 267841796

router_models/bert_5class/tokenizer_config.json

@@ -1,15 +0,0 @@
-{
-  "backend": "tokenizers",
-  "cls_token": "[CLS]",
-  "do_lower_case": true,
-  "is_local": false,
-  "local_files_only": false,
-  "mask_token": "[MASK]",
-  "model_max_length": 512,
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "strip_accents": null,
-  "tokenize_chinese_chars": true,
-  "tokenizer_class": "BertTokenizer",
-  "unk_token": "[UNK]"
-}

router_models/bert_router/config.json

@@ -1,28 +0,0 @@
-{
-  "activation": "gelu",
-  "architectures": [
-    "DistilBertForSequenceClassification"
-  ],
-  "attention_dropout": 0.1,
-  "bos_token_id": null,
-  "dim": 768,
-  "dropout": 0.1,
-  "dtype": "float32",
-  "eos_token_id": null,
-  "hidden_dim": 3072,
-  "initializer_range": 0.02,
-  "max_position_embeddings": 512,
-  "model_type": "distilbert",
-  "n_heads": 12,
-  "n_layers": 6,
-  "pad_token_id": 0,
-  "problem_type": "single_label_classification",
-  "qa_dropout": 0.1,
-  "seq_classif_dropout": 0.2,
-  "sinusoidal_pos_embds": false,
-  "tie_weights_": true,
-  "tie_word_embeddings": true,
-  "transformers_version": "5.8.0",
-  "use_cache": false,
-  "vocab_size": 30522
-}

router_models/bert_router/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:21baff75e1ebe25d72bd3c13383eb7d563cf1830c03a37dc81d173706fe473d9
-size 267832560

router_models/bert_router/tokenizer_config.json

@@ -1,15 +0,0 @@
-{
-  "backend": "tokenizers",
-  "cls_token": "[CLS]",
-  "do_lower_case": true,
-  "is_local": false,
-  "local_files_only": false,
-  "mask_token": "[MASK]",
-  "model_max_length": 512,
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "strip_accents": null,
-  "tokenize_chinese_chars": true,
-  "tokenizer_class": "BertTokenizer",
-  "unk_token": "[UNK]"
-}

router_models/v6_eval_results.json

@@ -1,304 +0,0 @@
-{
-  "always_frontier": {
-    "success": 0.8935,
-    "avg_cost": 1.0,
-    "unsafe_rate": 0.023,
-    "false_done": 0.0835,
-    "tier_dist": {
-      "4": 2000
-    },
-    "escalations": 878,
-    "downgrades": 420,
-    "heuristic_only": 702
-  },
-  "always_cheap": {
-    "success": 0.2225,
-    "avg_cost": 0.04999999999999823,
-    "unsafe_rate": 0.7775,
-    "false_done": 0.0,
-    "tier_dist": {
-      "1": 2000
-    },
-    "escalations": 0,
-    "downgrades": 2000,
-    "heuristic_only": 0
-  },
-  "heuristic_diff+1": {
-    "success": 0.8405,
-    "avg_cost": 0.9271500000000037,
-    "unsafe_rate": 0.047,
-    "false_done": 0.1125,
-    "tier_dist": {
-      "4": 702,
-      "3": 651,
-      "5": 420,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 0,
-    "heuristic_only": 2000
-  },
-  "heuristic_floor": {
-    "success": 0.5875,
-    "avg_cost": 0.5013749999999922,
-    "unsafe_rate": 0.28,
-    "false_done": 0.1325,
-    "tier_dist": {
-      "3": 913,
-      "2": 651,
-      "4": 209,
-      "1": 227
-    },
-    "escalations": 0,
-    "downgrades": 2000,
-    "heuristic_only": 0
-  },
-  "oracle": {
-    "success": 0.998,
-    "avg_cost": 0.47694999999999094,
-    "unsafe_rate": 0.0,
-    "false_done": 0.002,
-    "tier_dist": {
-      "3": 644,
-      "5": 46,
-      "2": 571,
-      "4": 294,
-      "1": 445
-    },
-    "escalations": 94,
-    "downgrades": 1486,
-    "heuristic_only": 420
-  },
-  "hybrid_s0.25_d0.70": {
-    "success": 0.8175,
-    "avg_cost": 0.8221500000000023,
-    "unsafe_rate": 0.0595,
-    "false_done": 0.123,
-    "tier_dist": {
-      "4": 1122,
-      "3": 651,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 420,
-    "heuristic_only": 1580
-  },
-  "hybrid_s0.25_d0.75": {
-    "success": 0.8175,
-    "avg_cost": 0.8221500000000023,
-    "unsafe_rate": 0.0595,
-    "false_done": 0.123,
-    "tier_dist": {
-      "4": 1122,
-      "3": 651,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 420,
-    "heuristic_only": 1580
-  },
-  "hybrid_s0.25_d0.80": {
-    "success": 0.8225,
-    "avg_cost": 0.8416500000000027,
-    "unsafe_rate": 0.058,
-    "false_done": 0.1195,
-    "tier_dist": {
-      "4": 1044,
-      "3": 651,
-      "5": 78,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 342,
-    "heuristic_only": 1658
-  },
-  "hybrid_s0.25_d0.85": {
-    "success": 0.838,
-    "avg_cost": 0.9084000000000035,
-    "unsafe_rate": 0.048,
-    "false_done": 0.114,
-    "tier_dist": {
-      "4": 777,
-      "3": 651,
-      "5": 345,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 75,
-    "heuristic_only": 1925
-  },
-  "hybrid_s0.30_d0.70": {
-    "success": 0.8175,
-    "avg_cost": 0.8221500000000023,
-    "unsafe_rate": 0.0595,
-    "false_done": 0.123,
-    "tier_dist": {
-      "4": 1122,
-      "3": 651,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 420,
-    "heuristic_only": 1580
-  },
-  "hybrid_s0.30_d0.75": {
-    "success": 0.8175,
-    "avg_cost": 0.8221500000000023,
-    "unsafe_rate": 0.0595,
-    "false_done": 0.123,
-    "tier_dist": {
-      "4": 1122,
-      "3": 651,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 420,
-    "heuristic_only": 1580
-  },
-  "hybrid_s0.30_d0.80": {
-    "success": 0.8225,
-    "avg_cost": 0.8416500000000027,
-    "unsafe_rate": 0.058,
-    "false_done": 0.1195,
-    "tier_dist": {
-      "4": 1044,
-      "3": 651,
-      "5": 78,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 342,
-    "heuristic_only": 1658
-  },
-  "hybrid_s0.30_d0.85": {
-    "success": 0.838,
-    "avg_cost": 0.9084000000000035,
-    "unsafe_rate": 0.048,
-    "false_done": 0.114,
-    "tier_dist": {
-      "4": 777,
-      "3": 651,
-      "5": 345,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 75,
-    "heuristic_only": 1925
-  },
-  "hybrid_s0.35_d0.70": {
-    "success": 0.8175,
-    "avg_cost": 0.8221500000000023,
-    "unsafe_rate": 0.0595,
-    "false_done": 0.123,
-    "tier_dist": {
-      "4": 1122,
-      "3": 651,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 420,
-    "heuristic_only": 1580
-  },
-  "hybrid_s0.35_d0.75": {
-    "success": 0.8175,
-    "avg_cost": 0.8221500000000023,
-    "unsafe_rate": 0.0595,
-    "false_done": 0.123,
-    "tier_dist": {
-      "4": 1122,
-      "3": 651,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 420,
-    "heuristic_only": 1580
-  },
-  "hybrid_s0.35_d0.80": {
-    "success": 0.8225,
-    "avg_cost": 0.8416500000000027,
-    "unsafe_rate": 0.058,
-    "false_done": 0.1195,
-    "tier_dist": {
-      "4": 1044,
-      "3": 651,
-      "5": 78,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 342,
-    "heuristic_only": 1658
-  },
-  "hybrid_s0.35_d0.85": {
-    "success": 0.838,
-    "avg_cost": 0.9084000000000035,
-    "unsafe_rate": 0.048,
-    "false_done": 0.114,
-    "tier_dist": {
-      "4": 777,
-      "3": 651,
-      "5": 345,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 75,
-    "heuristic_only": 1925
-  },
-  "hybrid_s0.40_d0.70": {
-    "success": 0.8175,
-    "avg_cost": 0.8221500000000023,
-    "unsafe_rate": 0.0595,
-    "false_done": 0.123,
-    "tier_dist": {
-      "4": 1122,
-      "3": 651,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 420,
-    "heuristic_only": 1580
-  },
-  "hybrid_s0.40_d0.75": {
-    "success": 0.8175,
-    "avg_cost": 0.8221500000000023,
-    "unsafe_rate": 0.0595,
-    "false_done": 0.123,
-    "tier_dist": {
-      "4": 1122,
-      "3": 651,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 420,
-    "heuristic_only": 1580
-  },
-  "hybrid_s0.40_d0.80": {
-    "success": 0.8225,
-    "avg_cost": 0.8416500000000027,
-    "unsafe_rate": 0.058,
-    "false_done": 0.1195,
-    "tier_dist": {
-      "4": 1044,
-      "3": 651,
-      "5": 78,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 342,
-    "heuristic_only": 1658
-  },
-  "hybrid_s0.40_d0.85": {
-    "success": 0.838,
-    "avg_cost": 0.9084000000000035,
-    "unsafe_rate": 0.048,
-    "false_done": 0.114,
-    "tier_dist": {
-      "4": 777,
-      "3": 651,
-      "5": 345,
-      "2": 227
-    },
-    "escalations": 0,
-    "downgrades": 75,
-    "heuristic_only": 1925
-  }
-}

training/train_baar_router.py

@@ -1,470 +0,0 @@
-#!/usr/bin/env python3
-"""BAAR-Style Profiling Router for ACO.
-
-Inspired by BAAR (Shnitzer et al., 2025): "Budget-Aware Adaptive Routing."
-The core idea: profile with a cheap model first, use its output as a feature
-for deciding whether to escalate to a stronger model.
-
-Our approach:
-1. BERT [CLS] embedding of the problem statement → semantic features
-2. Combine with classic keyword features
-3. XGBoost predicts P(success) for each tier
-4. Route to cheapest tier with P(success) >= threshold
-5. Compare: BERT+XGBoost vs keyword-only XGBoost vs frontier vs oracle
-
-The key innovation over v10: BERT features capture semantic difficulty
-that keyword features miss (e.g., "fix the segfault in the async event loop"
-vs "add a comment to the README" — keywords see "segfault" and "async" and
-think it's hard, but the BERT embedding captures the holistic semantics).
-"""
-import json, os, sys, pickle, random
-import numpy as np
-from collections import defaultdict
-from datasets import load_dataset
-from transformers import AutoTokenizer, AutoModel
-import torch
-
-# ── Config ──────────────────────────────────────────────────────
-REPO = "narcolepticchicken/agent-cost-optimizer"
-TIER_COST = {1: 0.01, 2: 0.05, 3: 0.15, 4: 0.30, 5: 0.50}
-TIER_TO_SWE = {
-    1: "deepseek-v4-flash", 2: "gpt-5-mini",
-    3: "gemini-2.5-pro", 4: "claude-opus-4.7", 5: "gemini-3-pro",
-}
-MODEL_TIER = {
-    "deepseek-v4-flash": 1, "gpt-5-nano": 1,
-    "gpt-5-mini": 2, "deepseek-v3.2": 2,
-    "gemini-2.5-pro": 3,
-    "claude-opus-4.7": 4, "gpt-5.2": 4,
-    "gemini-3-pro": 5,
-}
-MODELS = list(MODEL_TIER.keys())
-
-print("=" * 70)
-print("BAAR-STYLE PROFILING ROUTER: BERT + XGBoost")
-print("=" * 70)
-
-# ── 1. Extract BERT [CLS] features ──────────────────────────────
-print("\n[1] Loading BERT for feature extraction...")
-
-# First try the trained 5-class model (has domain knowledge from SPROUT)
-try:
-    tokenizer = AutoTokenizer.from_pretrained(
-        REPO, subfolder="router_models/bert_5class"
-    )
-    bert_model = AutoModel.from_pretrained(
-        REPO, subfolder="router_models/bert_5class"
-    )
-    print("  ✓ Loaded trained BERT 5-class from Hub (SPROUT-fine-tuned)")
-except Exception as e:
-    print(f"  Trained BERT failed: {e}")
-    tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
-    bert_model = AutoModel.from_pretrained("distilbert-base-uncased")
-    print("  Using base distilbert (not fine-tuned)")
-
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-bert_model = bert_model.to(device)
-bert_model.eval()
-
-# ── 2. Load SWE-Router ──────────────────────────────────────────
-print("\n[2] Loading SWE-Router execution data...")
-traces = defaultdict(dict)
-for model_name in MODELS:
-    ds = load_dataset(f"SWE-Router/swebench-verified-{model_name}", split="test")
-    for row in ds:
-        iid = row["instance_id"]
-        traces[iid][model_name] = {
-            "resolved": row["resolved"],
-            "cost": float(row["instance_cost"]),
-            "api_calls": int(row["api_calls"]),
-            "problem": row["problem_statement"],
-        }
-    print(f"  {model_name}: loaded")
-print(f"  Total tasks: {len(traces)}")
-
-# ── 3. Build combined features ──────────────────────────────────
-print("\n[3] Extracting BERT [CLS] embeddings + keyword features...")
-
-# Keyword features (same as v10)
-CODE_KW = ["python","javascript","code","function","bug","debug","refactor","implement","test",
-           "compile","runtime","segfault","thread","async","class","module","import","error","traceback"]
-LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"]
-RESEARCH_KW = ["research","investigate","compare","analyze","survey","paper"]
-TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
-CRITICAL_KW = ["critical","production","urgent","emergency","live","deployed","safety","security"]
-SIMPLE_KW = ["typo","simple","quick","brief","minor","small","easy","trivial","just"]
-LONG_KW = ["plan","project","roadmap","orchestrate","migrate","pipeline","deploy","architecture"]
-MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"]
-
-BERT_DIM = 768  # distilbert-base-uncased
-
-def extract_features(problem_text):
-    """Extract combined BERT + keyword features."""
-    r = problem_text.lower()
-
-    # Classic keyword features
-    kw_feats = [
-        len(problem_text),           # req_len
-        len(problem_text.split()),   # num_words
-        int(any(k in r for k in CODE_KW)),  # has_code
-        sum(1 for k in CODE_KW if k in r),  # n_code
-        int(any(k in r for k in LEGAL_KW)),
-        int(any(k in r for k in RESEARCH_KW)),
-        int(any(k in r for k in TOOL_KW)),
-        int(any(k in r for k in CRITICAL_KW)),
-        int(any(k in r for k in SIMPLE_KW)),
-        int(any(k in r for k in LONG_KW)),
-        int(any(k in r for k in MATH_KW)),
-        int("error" in r or "traceback" in r or "exception" in r),
-        int("/" in r and "." in problem_text),  # has_file_path
-        problem_text.count("\n") + 1,  # n_lines
-        int("version" in r or "update" in r or "upgrade" in r),
-        int("add" in r or "new" in r or "create" in r),
-        int("fix" in r or "bug" in r or "issue" in r or "broken" in r),
-        int("change" in r or "modify" in r or "update" in r),
-        int("remove" in r or "delete" in r or "drop" in r),
-        int("test" in r or "spec" in r or "assert" in r),
-        int("doc" in r or "readme" in r or "comment" in r),
-        int("see also" in r or "related" in r),
-        int("steps to reproduce" in r or "reproduce" in r),
-    ]
-    
-    # BERT [CLS] embedding
-    enc = tokenizer(
-        problem_text[:2000], truncation=True, max_length=512,
-        return_tensors="pt",
-    )
-    enc = {k: v.to(device) for k, v in enc.items()}
-    with torch.no_grad():
-        outputs = bert_model(**enc)
-        cls_embedding = outputs.last_hidden_state[:, 0, :]  # [1, 768]
-    
-    # Combine
-    full = kw_feats + cls_embedding[0].cpu().numpy().tolist()
-    return full
-
-# Build training data
-X_feats = []
-y_tiers = {t: [] for t in range(1, 6)}
-y_optimal = []
-feat_names = (
-    ["req_len","num_words","has_code","n_code","has_legal","has_research","has_tool",
-     "has_critical","has_simple","has_long","has_math","has_error_msg","has_file_path",
-     "n_lines","has_version","has_add","has_fix","has_change","has_remove","has_test",
-     "has_doc","has_see_also","has_steps"] +
-    [f"bert_{i}" for i in range(BERT_DIM)]
-)
-
-for iid, model_results in traces.items():
-    problem = next(iter(model_results.values()))["problem"]
-    feats = extract_features(problem)
-    X_feats.append(feats)
-    
-    # Per-tier success ground truth
-    tier_success = {}
-    for model_name, result in model_results.items():
-        tier = MODEL_TIER[model_name]
-        if tier not in tier_success:
-            tier_success[tier] = False
-        if result["resolved"]:
-            tier_success[tier] = True
-    
-    for t in range(1, 6):
-        y_tiers[t].append(int(tier_success.get(t, False)))
-    
-    # Optimal tier = cheapest that succeeded
-    optimal = 5
-    for t in range(1, 6):
-        if tier_success.get(t, False):
-            optimal = t
-            break
-    y_optimal.append(optimal)
-
-X = np.array(X_feats, dtype=np.float32)
-
-print(f"  Feature matrix: {X.shape} ({len(feat_names)} features)")
-print(f"  BERT features: {BERT_DIM} dims + {len(feat_names)-BERT_DIM} keyword feats")
-print(f"  Per-tier success rates:")
-for t in range(1, 6):
-    s = sum(y_tiers[t])
-    print(f"    Tier {t}: {s}/{len(y_tiers[t])} = {s/len(y_tiers[t])*100:.1f}%")
-print(f"  Optimal tier distribution:")
-opt_dist = defaultdict(int)
-for t in y_optimal:
-    opt_dist[t] += 1
-for t in sorted(opt_dist):
-    print(f"    Tier {t}: {opt_dist[t]} ({opt_dist[t]/len(y_optimal)*100:.1f}%)")
-
-# ── 4. Train XGBoost with BERT features ─────────────────────────
-print("\n[4] Training BAAR-style XGBoost + BERT router...")
-
-from xgboost import XGBClassifier
-from sklearn.calibration import IsotonicRegression
-from sklearn.model_selection import cross_val_score
-import warnings
-warnings.filterwarnings("ignore")
-
-tier_clfs = {}
-tier_calibs = {}
-tier_cv = {}
-
-for t in range(1, 6):
-    y = np.array(y_tiers[t])
-    n_pos = y.sum()
-    n_neg = len(y) - n_pos
-    spw = max(1, n_neg / max(n_pos, 1))
-    
-    clf = XGBClassifier(
-        n_estimators=200, max_depth=6, learning_rate=0.05,
-        subsample=0.8, colsample_bytree=0.8,
-        scale_pos_weight=spw,
-        eval_metric="logloss",
-        random_state=42,
-        verbosity=0,
-    )
-    
-    try:
-        scores = cross_val_score(clf, X, y, cv=5, scoring="f1")
-        tier_cv[t] = float(scores.mean())
-    except:
-        tier_cv[t] = 0.0
-    
-    clf.fit(X, y)
-    
-    # Calibrate probabilities
-    p_raw = clf.predict_proba(X)[:, 1]
-    cal = IsotonicRegression(out_of_bounds="clip")
-    cal.fit(p_raw, y)
-    
-    tier_clfs[t] = clf
-    tier_calibs[t] = cal
-    
-    p_cal = cal.transform(p_raw)
-    brier = np.mean((p_cal - y) ** 2)
-    
-    print(f"  Tier {t}: n_pos={n_pos}, CV_f1={tier_cv[t]:.3f}, Brier={brier:.4f}")
-
-# Direct optimal-tier predictor
-opt_clf = XGBClassifier(
-    n_estimators=300, max_depth=6, learning_rate=0.05,
-    subsample=0.8, colsample_bytree=0.8,
-    eval_metric="mlogloss",
-    random_state=42, num_class=5, verbosity=0,
-)
-opt_clf.fit(X, np.array(y_optimal) - 1)
-opt_pred = opt_clf.predict(X) + 1
-opt_acc = np.mean(opt_pred == np.array(y_optimal))
-print(f"  Direct optimal-tier accuracy: {opt_acc:.3f}")
-
-# Also train a keyword-only baseline for ablation
-print("\n[4b] Training keyword-only baseline (no BERT)...")
-kw_X = X[:, :23]  # First 23 are keyword features
-
-kw_tier_clfs = {}
-kw_tier_calibs = {}
-kw_tier_cv = {}
-
-for t in range(1, 6):
-    y = np.array(y_tiers[t])
-    n_pos = y.sum()
-    n_neg = len(y) - n_pos
-    spw = max(1, n_neg / max(n_pos, 1))
-    
-    clf = XGBClassifier(
-        n_estimators=200, max_depth=6, learning_rate=0.05,
-        subsample=0.8, colsample_bytree=0.8,
-        scale_pos_weight=spw,
-        eval_metric="logloss",
-        random_state=42,
-        verbosity=0,
-    )
-    
-    try:
-        scores = cross_val_score(clf, kw_X, y, cv=5, scoring="f1")
-        kw_tier_cv[t] = float(scores.mean())
-    except:
-        kw_tier_cv[t] = 0.0
-    
-    clf.fit(kw_X, y)
-    p_raw = clf.predict_proba(kw_X)[:, 1]
-    cal = IsotonicRegression(out_of_bounds="clip")
-    cal.fit(p_raw, y)
-    
-    kw_tier_clfs[t] = clf
-    kw_tier_calibs[t] = cal
-
-# ── 5. Evaluate all policies ─────────────────────────────────────
-print("\n[5] Evaluating routing policies...")
-
-policies = defaultdict(lambda: {"success": 0, "cost": 0.0, "n": 0})
-
-for iid, model_results in traces.items():
-    problem = next(iter(model_results.values()))["problem"]
-    feats = extract_features(problem)
-    feat_vec = np.array(feats, dtype=np.float32).reshape(1, -1)
-    kw_vec = np.array(feats[:23], dtype=np.float32).reshape(1, -1)
-
-    # ─── BAAR (BERT + XGBoost) ───
-    tier_probs_baar = {}
-    for t in range(1, 6):
-        p_raw = tier_clfs[t].predict_proba(feat_vec)[0, 1]
-        p_cal = float(tier_calibs[t].transform([p_raw])[0])
-        tier_probs_baar[t] = p_cal
-
-    baar_tier = 5
-    for t in range(1, 6):
-        if tier_probs_baar[t] >= 0.5:
-            baar_tier = t
-            break
-    baar_model = TIER_TO_SWE.get(baar_tier, "claude-opus-4.7")
-    if baar_model in model_results:
-        policies["baar"]["success"] += int(model_results[baar_model]["resolved"])
-        policies["baar"]["cost"] += model_results[baar_model]["cost"]
-    policies["baar"]["n"] += 1
-
-    # ─── BAAR + feedback (escalate on failure) ───
-    try_tier = baar_tier
-    total_cost = 0.0
-    success = False
-    while try_tier <= 5:
-        tm = TIER_TO_SWE.get(try_tier, "claude-opus-4.7")
-        if tm in model_results:
-            total_cost += model_results[tm]["cost"]
-            if model_results[tm]["resolved"]:
-                success = True
-                break
-        try_tier += 1
-    policies["baar_feedback"]["success"] += int(success)
-    policies["baar_feedback"]["cost"] += total_cost
-    policies["baar_feedback"]["n"] += 1
-
-    # ─── Keyword-only ───
-    kw_tier = 5
-    for t in range(1, 6):
-        p_raw = kw_tier_clfs[t].predict_proba(kw_vec)[0, 1]
-        p_cal = float(kw_tier_calibs[t].transform([p_raw])[0])
-        if p_cal >= 0.5:
-            kw_tier = t
-            break
-    kw_model = TIER_TO_SWE.get(kw_tier, "claude-opus-4.7")
-    if kw_model in model_results:
-        policies["keyword_only"]["success"] += int(model_results[kw_model]["resolved"])
-        policies["keyword_only"]["cost"] += model_results[kw_model]["cost"]
-    policies["keyword_only"]["n"] += 1
-
-    # ─── Frontiers ───
-    f_model = "claude-opus-4.7"
-    policies["frontier"]["success"] += int(model_results[f_model]["resolved"])
-    policies["frontier"]["cost"] += model_results[f_model]["cost"]
-    policies["frontier"]["n"] += 1
-
-    c_model = "deepseek-v4-flash"
-    if c_model in model_results:
-        policies["always_cheap"]["success"] += int(model_results[c_model]["resolved"])
-        policies["always_cheap"]["cost"] += model_results[c_model]["cost"]
-    policies["always_cheap"]["n"] += 1
-
-    # ─── Oracle ───
-    resolved = [(m, r) for m, r in model_results.items() if r["resolved"]]
-    if resolved:
-        cheapest = min(resolved, key=lambda x: TIER_COST.get(MODEL_TIER[x[0]], 1.0))
-        policies["oracle"]["success"] += 1
-        policies["oracle"]["cost"] += cheapest[1]["cost"]
-    policies["oracle"]["n"] += 1
-
-# ── 6. Print results ────────────────────────────────────────────
-fr_cost = policies["frontier"]["cost"] / max(policies["frontier"]["n"], 1)
-
-print(f"\n{'='*70}")
-print("BAAR-STYLE ROUTER RESULTS ON SWE-BENCH (500 tasks)")
-print(f"{'='*70}")
-print(f"\n{'Policy':<25} {'Success':>10} {'AvgCost':>10} {'CostRed':>10}")
-print("-" * 55)
-
-for name in ["oracle", "baar_feedback", "baar", "keyword_only", "frontier", "always_cheap"]:
-    r = policies[name]
-    sr = r["success"] / max(r["n"], 1)
-    ac = r["cost"] / max(r["n"], 1)
-    cr = (1 - ac / max(fr_cost, 0.0001)) * 100
-    print(f"  {name:<23} {sr:>10.3f} {ac:>10.4f} {cr:>9.1f}%")
-
-# Compare CV scores
-print(f"\n{'='*70}")
-print("CROSS-VALIDATION F1 COMPARISON")
-print(f"{'='*70}")
-print(f"\n{'Tier':<8} {'Keyword-only':>15} {'BAAR (BERT+KW)':>18} {'Improvement':>14}")
-print("-" * 55)
-for t in range(1, 6):
-    kw_cv = kw_tier_cv.get(t, 0)
-    baar_cv = tier_cv.get(t, 0)
-    imp = baar_cv - kw_cv
-    print(f"  Tier {t:<4} {kw_cv:>15.3f} {baar_cv:>18.3f} {imp:>13.3f}")
-
-# Save results
-results = {
-    "router": "BAAR-style (BERT [CLS] + XGBoost)",
-    "n_tasks": len(traces),
-    "n_features": X.shape[1],
-    "n_bert_dims": BERT_DIM,
-    "n_keyword_feats": 23,
-    "policies": {
-        name: {
-            "success": r["success"],
-            "success_rate": round(r["success"] / max(r["n"], 1), 4),
-            "avg_cost": round(r["cost"] / max(r["n"], 1), 4),
-            "cost_reduction_pct": round(
-                (1 - (r["cost"] / max(r["n"], 1)) / max(fr_cost, 0.0001)) * 100, 1
-            ),
-        }
-        for name, r in policies.items()
-    },
-    "cv_f1_comparison": {
-        "keyword_only": {str(k): v for k, v in kw_tier_cv.items()},
-        "baar_bert_kw": {str(k): v for k, v in tier_cv.items()},
-    },
-    "opt_tier_accuracy": float(opt_acc),
-}
-
-with open("/tmp/baar_results.json", "w") as f:
-    json.dump(results, f, indent=2)
-
-# Upload to Hub
-from huggingface_hub import HfApi
-api = HfApi()
-api.upload_file(
-    path_or_fileobj="/tmp/baar_results.json",
-    path_in_repo="eval/baar_results.json",
-    repo_id=REPO,
-    repo_type="model",
-)
-print(f"\n  ✓ Results uploaded to Hub: eval/baar_results.json")
-
-# Save model bundle
-bundle = {
-    "tier_clfs": {str(k): v for k, v in tier_clfs.items()},
-    "tier_calibrators": {str(k): v for k, v in tier_calibs.items()},
-    "opt_clf": opt_clf,
-    "kw_tier_clfs": {str(k): v for k, v in kw_tier_clfs.items()},
-    "kw_tier_calibrators": {str(k): v for k, v in kw_tier_calibs.items()},
-    "feat_names": feat_names,
-    "bert_dim": BERT_DIM,
-    "n_keyword_feats": 23,
-    "version": "baar_v1",
-    "description": "BAAR-style router: BERT [CLS] embeddings + keyword features → XGBoost tier predictor",
-    "tier_cost": TIER_COST,
-}
-
-with open("/tmp/baar_bundle.pkl", "wb") as f:
-    pickle.dump(bundle, f)
-
-api.upload_file(
-    path_or_fileobj="/tmp/baar_bundle.pkl",
-    path_in_repo="router_models/baar_bundle.pkl",
-    repo_id=REPO,
-    repo_type="model",
-)
-print(f"  ✓ BAAR bundle uploaded: router_models/baar_bundle.pkl")
-
-print(f"\n{'='*70}")
-print("DONE! BAAR-style router built and evaluated.")
-print("=" * 70)

training/train_bert_5class.py

@@ -1,187 +0,0 @@
-"""Retrain BERT as a 5-class tier router.
-
-Uses SPROUT data to predict optimal tier (1-5) directly.
-"""
-import os, json, random
-import numpy as np
-from datasets import load_dataset
-from transformers import (
-    AutoTokenizer, AutoModelForSequenceClassification,
-    TrainingArguments, Trainer, DataCollatorWithPadding,
-)
-import torch
-
-REPO = "narcolepticchicken/agent-cost-optimizer"
-print("BERT 5-Class Tier Router Training")
-print("="*60)
-
-# ── Load SPROUT ──
-print("\n[1] Loading SPROUT dataset...")
-ds = load_dataset("CARROT-LLM-Routing/SPROUT", split="train", trust_remote_code=True)
-print(f"  Total rows: {len(ds)}")
-print(f"  Columns: {ds.column_names}")
-
-# ── Model tier mapping ──
-# SPROUT models → tiers (same as v11 training)
-MODEL_TIER_MAP = {}
-TIER_MODELS = {
-    1: ["gemma-2-2b-it","phi-3-mini-128k-instruct","qwen2.5-3b-instruct",
-        "llama-3.2-3b-instruct","deepseek-v3.2"],
-    2: ["gemma-2-9b-it","mistral-7b-instruct-v0.3","qwen2.5-7b-instruct",
-        "llama-3.1-8b-instruct","gpt-5-nano","gpt-5-mini"],
-    3: ["qwen2.5-32b-instruct","mixtral-8x7b-instruct-v0.1",
-        "gemma-2-27b-it","gemini-2.5-pro"],
-    4: ["claude-opus-4.7","gpt-5.2","llama-3.1-70b-instruct",
-        "qwen2.5-72b-instruct"],
-    5: ["gemini-3-pro","deepseek-v4-flash"],
-}
-for tier, models in TIER_MODELS.items():
-    for m in models:
-        MODEL_TIER_MAP[m.lower()] = tier
-
-# ── Build training data ──
-print("\n[2] Building training data...")
-texts = []
-labels = []
-skipped = 0
-
-for row in ds:
-    # Find the cheapest tier that succeeded
-    best_tier = 5  # default
-    found = False
-    
-    # Try to find model results in the row
-    for tier in range(1, 6):
-        for m in TIER_MODELS.get(tier, []):
-            m_lower = m.lower()
-            # Check various possible column names
-            for col in ds.column_names:
-                if m_lower in col.lower():
-                    val = row.get(col)
-                    if isinstance(val, (int, float)) and val > 0:
-                        best_tier = tier
-                        found = True
-                        break
-            if found:
-                break
-        if found:
-            break
-    
-    # Get the prompt/question text
-    prompt = ""
-    for col in ["prompt", "question", "input", "query", "problem_statement", "instruction"]:
-        if col in ds.column_names:
-            prompt = str(row[col])
-            break
-    
-    if not prompt:
-        # Try first string column
-        for col in ds.column_names:
-            if isinstance(row[col], str) and len(row[col]) > 20:
-                prompt = row[col]
-                break
-    
-    if prompt and len(prompt) > 10:
-        texts.append(prompt[:2000])  # truncate long texts
-        labels.append(best_tier - 1)  # 0-indexed for classification
-    else:
-        skipped += 1
-
-print(f"  Training samples: {len(texts)}")
-print(f"  Skipped: {skipped}")
-print(f"  Label distribution:")
-from collections import Counter
-label_dist = Counter(labels)
-for label in sorted(label_dist):
-    print(f"    Tier {label+1}: {label_dist[label]} ({label_dist[label]/len(labels)*100:.1f}%)")
-
-# ── Tokenize ──
-print("\n[3] Tokenizing...")
-tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
-
-encodings = tokenizer(texts, truncation=True, max_length=512, padding=False)
-
-class TierDataset(torch.utils.data.Dataset):
-    def __init__(self, encodings, labels):
-        self.encodings = encodings
-        self.labels = labels
-    def __getitem__(self, idx):
-        item = {k: torch.tensor(v[idx]) for k, v in self.encodings.items()}
-        item["labels"] = torch.tensor(self.labels[idx])
-        return item
-    def __len__(self):
-        return len(self.labels)
-
-# Split
-split = int(0.9 * len(texts))
-train_ds = TierDataset(
-    {k: v[:split] for k, v in encodings.items()},
-    labels[:split],
-)
-eval_ds = TierDataset(
-    {k: v[split:] for k, v in encodings.items()},
-    labels[split:],
-)
-print(f"  Train: {len(train_ds)}, Eval: {len(eval_ds)}")
-
-# ── Train ──
-print("\n[4] Training 5-class BERT router...")
-model = AutoModelForSequenceClassification.from_pretrained(
-    "distilbert-base-uncased", num_labels=5
-)
-
-training_args = TrainingArguments(
-    output_dir="/tmp/bert_5class",
-    num_train_epochs=3,
-    per_device_train_batch_size=32,
-    per_device_eval_batch_size=64,
-    learning_rate=2e-5,
-    weight_decay=0.01,
-    eval_strategy="epoch",
-    save_strategy="epoch",
-    load_best_model_at_end=True,
-    metric_for_best_model="eval_accuracy",
-    logging_steps=50,
-    disable_tqdm=True,
-)
-
-def compute_metrics(eval_pred):
-    logits, labels = eval_pred
-    preds = np.argmax(logits, axis=-1)
-    acc = np.mean(preds == labels)
-    return {"accuracy": acc}
-
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=train_ds,
-    eval_dataset=eval_ds,
-    tokenizer=tokenizer,
-    data_collator=DataCollatorWithPadding(tokenizer),
-    compute_metrics=compute_metrics,
-)
-
-trainer.train()
-
-# ── Save and upload ──
-print("\n[5] Saving model...")
-save_dir = "/tmp/bert_5class_final"
-model.save_pretrained(save_dir)
-tokenizer.save_pretrained(save_dir)
-print(f"  Saved to {save_dir}")
-
-# Upload to Hub
-from huggingface_hub import HfApi
-api = HfApi()
-for fname in os.listdir(save_dir):
-    fpath = os.path.join(save_dir, fname)
-    if os.path.isfile(fpath):
-        api.upload_file(
-            path_or_fileobj=fpath,
-            path_in_repo=f"router_models/bert_5class/{fname}",
-            repo_id=REPO,
-            repo_type="model",
-        )
-        print(f"  Uploaded {fname}")
-
-print("\nDONE! 5-class BERT router saved to router_models/bert_5class/")

training/train_bert_5class_v2.py

@@ -1,291 +0,0 @@
-#!/usr/bin/env python3
-"""BERT 5-Class Tier Router — V2 with PROPER SPROUT parsing.
-
-Fixes the V1 bug: SPROUT model columns are dicts like:
-    {'judge_response': '{"correctness_score": 0.8, "justification": "..."}'}
-which need JSON parsing, not naive int/float checks.
-
-Also writes a robust label extraction that maps SPROUT's 13 models → ACO's 5 cost tiers
-and labels each task with the cheapest tier that achieves correctness_score >= THRESHOLD.
-"""
-import json, os, sys
-import numpy as np
-from collections import Counter
-from datasets import load_dataset
-import torch
-from transformers import (
-    AutoTokenizer,
-    AutoModelForSequenceClassification,
-    TrainingArguments,
-    Trainer,
-    DataCollatorWithPadding,
-)
-from sklearn.model_selection import train_test_split
-
-REPO = "narcolepticchicken/agent-cost-optimizer"
-CORRECTNESS_THRESHOLD = 0.7  # correctness_score >= this = "succeeded"
-MAX_SEQ_LEN = 512
-BATCH_SIZE = 32
-EPOCHS = 5
-LR = 2e-5
-
-print("="*70)
-print("BERT 5-CLASS TIER ROUTER — V2 (Fixed SPROUT Parsing)")
-print("="*70)
-
-# ── 1. Map SPROUT models → ACO cost tiers ──────────────────────────
-SPROUT_MODEL_TO_TIER = {
-    # Tier 1: smallest/cheapest
-    "wxai-granite-3-2b-instruct-8k-max-tokens": 1,
-    "wxai-llama-3-2-1b-instruct": 1,
-    "wxai-llama-3-2-3b-instruct": 1,
-    # Tier 2: small/medium
-    "wxai-llama-3-1-8b-instruct": 2,
-    "wxai-granite-3-8b-instruct-8k-max-tokens": 2,
-    "openai-gpt-4o-mini": 2,
-    # Tier 3: medium
-    "wxai-mixtral-8x7b-instruct-v01": 3,
-    "aws-titan-text-premier-v1": 3,
-    # Tier 4: strong
-    "openai-gpt-4o": 4,
-    "wxai-llama-3-1-70b-instruct": 4,
-    "wxai-llama-3-3-70b-instruct": 4,
-    "aws-claude-3-5-sonnet-v1": 4,
-    # Tier 5: strongest
-    "wxai-llama-3-405b-instruct": 5,
-}
-
-SPROUT_MODEL_COLS = list(SPROUT_MODEL_TO_TIER.keys())
-print(f"\n[1] SPROUT model mapping: {len(SPROUT_MODEL_COLS)} models → 5 ACO tiers")
-
-# ── 2. Load SPROUT & extract correctness scores ────────────────────
-print("\n[2] Loading SPROUT dataset...")
-ds = load_dataset("CARROT-LLM-Routing/SPROUT", split="train", trust_remote_code=True)
-print(f"  Total rows: {len(ds)}")
-
-print("\n[3] Parsing judge_response JSON and extracting correctness_scores...")
-
-texts = []
-labels = []
-parse_errors = 0
-no_score = 0
-all_failed = 0
-label_counter = Counter()
-
-# Track per-model success rates for sanity check
-model_success_counts = Counter()
-model_total_counts = Counter()
-
-for idx, row in enumerate(ds):
-    prompt = str(row.get("prompt") or "")
-    if not prompt or len(prompt) < 10:
-        all_failed += 1
-        continue
-
-    # Find the cheapest tier where any model succeeded
-    best_tier = 5  # default to frontier
-    found_success = False
-
-    for model_col in SPROUT_MODEL_COLS:
-        tier = SPROUT_MODEL_TO_TIER[model_col]
-        
-        if model_col not in row:
-            continue
-
-        val = row[model_col]
-        
-        # Handle the nested dict structure
-        score = None
-        try:
-            if isinstance(val, dict):
-                # {'judge_response': '{"correctness_score": 0.8, ...}'}
-                jr = val.get("judge_response", "")
-                if isinstance(jr, str) and jr.strip():
-                    jr_parsed = json.loads(jr)
-                    score = jr_parsed.get("correctness_score", None)
-                elif isinstance(jr, dict):
-                    # Already parsed JSON
-                    score = jr.get("correctness_score", None)
-            elif isinstance(val, str):
-                # Direct JSON string
-                val_parsed = json.loads(val)
-                jr = val_parsed.get("judge_response", "")
-                if isinstance(jr, str) and jr.strip():
-                    jr_parsed = json.loads(jr)
-                    score = jr_parsed.get("correctness_score", None)
-                elif isinstance(jr, dict):
-                    score = jr.get("correctness_score", None)
-                else:
-                    score = val_parsed.get("correctness_score", None)
-        except (json.JSONDecodeError, AttributeError, TypeError) as e:
-            parse_errors += 1
-            continue
-
-        if score is None:
-            no_score += 1
-            continue
-
-        model_total_counts[model_col] += 1
-        succeeded = float(score) >= CORRECTNESS_THRESHOLD
-        if succeeded:
-            model_success_counts[model_col] += 1
-            if not found_success or tier < best_tier:
-                best_tier = tier
-                found_success = True
-
-    if not found_success:
-        all_failed += 1
-        best_tier = 5
-
-    texts.append(prompt[:2000])
-    labels.append(best_tier - 1)  # 0-indexed
-    label_counter[best_tier] += 1
-
-print(f"\n[3a] Label extraction complete:")
-print(f"    Valid samples:   {len(texts)}")
-print(f"    All-failed:      {all_failed}")
-print(f"    Parse errors:    {parse_errors}")
-print(f"    Missing scores:  {no_score}")
-print(f"    Label distribution (tier → 0-idx):")
-for tier in range(1, 6):
-    c = label_counter.get(tier, 0)
-    print(f"      Tier {tier}: {c} ({c/max(len(texts),1)*100:.1f}%)")
-
-print(f"\n[3b] Per-model success rates (@ threshold {CORRECTNESS_THRESHOLD}):")
-for model_col in SPROUT_MODEL_COLS:
-    total = model_total_counts[model_col]
-    if total > 0:
-        succ = model_success_counts[model_col]
-        print(f"    {model_col:<50}: {succ}/{total} = {succ/total*100:.1f}%")
-
-# ── 4. Tokenize ───────────────────────────────────────────────────
-print(f"\n[4] Tokenizing {len(texts)} samples with distilbert-base-uncased...")
-tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
-
-encodings = tokenizer(
-    texts, truncation=True, max_length=MAX_SEQ_LEN, padding=False
-)
-
-class TierDataset(torch.utils.data.Dataset):
-    def __init__(self, encodings, labels):
-        self.encodings = encodings
-        self.labels = labels
-    def __getitem__(self, idx):
-        item = {k: torch.tensor(v[idx]) for k, v in self.encodings.items()}
-        item["labels"] = torch.tensor(self.labels[idx])
-        return item
-    def __len__(self):
-        return len(self.labels)
-
-# Split
-indices = list(range(len(texts)))
-train_idx, eval_idx = train_test_split(indices, test_size=0.1, random_state=42)
-
-train_enc = {k: [v[i] for i in train_idx] for k, v in encodings.items()}
-eval_enc = {k: [v[i] for i in eval_idx] for k, v in encodings.items()}
-train_labels = [labels[i] for i in train_idx]
-eval_labels = [labels[i] for i in eval_idx]
-
-train_ds = TierDataset(train_enc, train_labels)
-eval_ds = TierDataset(eval_enc, eval_labels)
-print(f"  Train: {len(train_ds)}, Eval: {len(eval_ds)}")
-
-# ── 5. Train ──────────────────────────────────────────────────────
-print("\n[5] Training 5-class DistilBERT router...")
-
-model = AutoModelForSequenceClassification.from_pretrained(
-    "distilbert-base-uncased", num_labels=5
-)
-
-training_args = TrainingArguments(
-    output_dir="/tmp/bert_5class_v2",
-    num_train_epochs=EPOCHS,
-    per_device_train_batch_size=BATCH_SIZE,
-    per_device_eval_batch_size=BATCH_SIZE * 2,
-    learning_rate=LR,
-    weight_decay=0.01,
-    warmup_ratio=0.1,
-    eval_strategy="epoch",
-    save_strategy="epoch",
-    load_best_model_at_end=True,
-    metric_for_best_model="eval_accuracy",
-    logging_steps=100,
-    logging_first_step=True,
-    disable_tqdm=True,
-    report_to="none",
-)
-
-def compute_metrics(eval_pred):
-    logits, labels = eval_pred
-    preds = np.argmax(logits, axis=-1)
-    acc = np.mean(preds == labels).item()
-    
-    # Per-class accuracy
-    per_class = {}
-    for c in range(5):
-        mask = labels == c
-        if mask.sum() > 0:
-            per_class[f"acc_tier{c+1}"] = np.mean(preds[mask] == labels[mask]).item()
-    
-    off_by_one = np.mean(np.abs(preds - labels) <= 1).item()
-    
-    return {"accuracy": acc, "off_by_one": off_by_one, **per_class}
-
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=train_ds,
-    eval_dataset=eval_ds,
-    tokenizer=tokenizer,
-    data_collator=DataCollatorWithPadding(tokenizer),
-    compute_metrics=compute_metrics,
-)
-
-result = trainer.train()
-final_metrics = trainer.evaluate()
-print(f"\n  Final metrics: {final_metrics}")
-
-# ── 6. Save and upload ────────────────────────────────────────────
-print("\n[6] Saving model...")
-save_dir = "/tmp/bert_5class_v2_final"
-model.save_pretrained(save_dir)
-tokenizer.save_pretrained(save_dir)
-
-meta = {
-    "model": "distilbert-base-uncased",
-    "num_labels": 5,
-    "task": "5-class optimal tier prediction",
-    "training_dataset": "CARROT-LLM-Routing/SPROUT",
-    "n_training_samples": len(train_ds),
-    "correctness_threshold": CORRECTNESS_THRESHOLD,
-    "tier_mapping": SPROUT_MODEL_TO_TIER,
-    "max_seq_len": MAX_SEQ_LEN,
-    "epochs": EPOCHS,
-    "learning_rate": LR,
-    "final_metrics": {k: float(v) if isinstance(v, (np.floating, np.integer)) else v 
-                      for k, v in final_metrics.items()},
-    "label_distribution": dict(label_counter),
-}
-with open(os.path.join(save_dir, "metadata.json"), "w") as f:
-    json.dump(meta, f, indent=2)
-print(f"  Saved to {save_dir}")
-
-# Upload
-from huggingface_hub import HfApi
-
-api = HfApi()
-for fname in os.listdir(save_dir):
-    fpath = os.path.join(save_dir, fname)
-    if os.path.isfile(fpath):
-        api.upload_file(
-            path_or_fileobj=fpath,
-            path_in_repo=f"router_models/bert_5class/{fname}",
-            repo_id=REPO,
-            repo_type="model",
-        )
-        print(f"  Uploaded {fname}")
-
-print("\n" + "="*70)
-print("DONE! BERT 5-class router saved to router_models/bert_5class/")
-print(f"Repo: https://huggingface.co/{REPO}")
-print("="*70)

training/train_bert_5class_v3.py

@@ -1,292 +0,0 @@
-#!/usr/bin/env python3
-"""BERT 5-Class Tier Router — V3 with ROBUST SPROUT parsing.
-
-Fixes V2 issues:
-1. Handle escaped single-quotes in judge_response JSON (Model\'s → Model's)  
-2. Add accelerate dependency
-3. More robust fallback parsing for edge cases
-"""
-import json, os, sys, re
-import numpy as np
-from collections import Counter
-from datasets import load_dataset
-import torch
-from transformers import (
-    AutoTokenizer,
-    AutoModelForSequenceClassification,
-    TrainingArguments,
-    Trainer,
-    DataCollatorWithPadding,
-)
-from sklearn.model_selection import train_test_split
-
-REPO = "narcolepticchicken/agent-cost-optimizer"
-CORRECTNESS_THRESHOLD = 0.7
-MAX_SEQ_LEN = 512
-BATCH_SIZE = 32
-EPOCHS = 5
-LR = 2e-5
-
-print("="*70)
-print("BERT 5-CLASS TIER ROUTER — V3 (Robust SPROUT Parsing)")
-print("="*70)
-
-# ── 1. Map SPROUT models → ACO cost tiers ──────────────────────────
-SPROUT_MODEL_TO_TIER = {
-    "wxai-granite-3-2b-instruct-8k-max-tokens": 1,
-    "wxai-llama-3-2-1b-instruct": 1,
-    "wxai-llama-3-2-3b-instruct": 1,
-    "wxai-llama-3-1-8b-instruct": 2,
-    "wxai-granite-3-8b-instruct-8k-max-tokens": 2,
-    "openai-gpt-4o-mini": 2,
-    "wxai-mixtral-8x7b-instruct-v01": 3,
-    "aws-titan-text-premier-v1": 3,
-    "openai-gpt-4o": 4,
-    "wxai-llama-3-1-70b-instruct": 4,
-    "wxai-llama-3-3-70b-instruct": 4,
-    "aws-claude-3-5-sonnet-v1": 4,
-    "wxai-llama-3-405b-instruct": 5,
-}
-SPROUT_MODEL_COLS = list(SPROUT_MODEL_TO_TIER.keys())
-
-print(f"\n[1] SPROUT model mapping: {len(SPROUT_MODEL_COLS)} models → 5 ACO tiers")
-
-# ── 2. Robust JSON parsing helper ────────────────────────────────
-def extract_correctness_score(val):
-    """Robustly extract correctness_score from SPROUT's nested dict/JSON structure."""
-    if val is None:
-        return None
-    
-    judge_raw = None
-    if isinstance(val, dict):
-        judge_raw = val.get("judge_response")
-    elif isinstance(val, str):
-        try:
-            parsed = json.loads(val)
-            if isinstance(parsed, dict):
-                judge_raw = parsed.get("judge_response")
-            else:
-                return None
-        except json.JSONDecodeError:
-            return None
-    else:
-        return None
-    
-    if judge_raw is None:
-        return None
-    if isinstance(judge_raw, (int, float)):
-        return float(judge_raw)
-    if isinstance(judge_raw, dict):
-        score = judge_raw.get("correctness_score")
-        return float(score) if score is not None else None
-    if not isinstance(judge_raw, str) or not judge_raw.strip():
-        return None
-    
-    # Normal JSON parse
-    try:
-        parsed = json.loads(judge_raw)
-        score = parsed.get("correctness_score")
-        return float(score) if score is not None else None
-    except json.JSONDecodeError:
-        pass
-    
-    # Fix common escape issues
-    cleaned = judge_raw.replace("\\'", "'")
-    try:
-        parsed = json.loads(cleaned)
-        score = parsed.get("correctness_score")
-        return float(score) if score is not None else None
-    except json.JSONDecodeError:
-        pass
-    
-    # Regex fallback
-    m = re.search(r'"correctness_score"\s*:\s*([\d.]+)', judge_raw)
-    if m:
-        try:
-            return float(m.group(1))
-        except ValueError:
-            pass
-    return None
-
-# ── 3. Load SPROUT & extract labels ──────────────────────────────
-print("\n[2] Loading SPROUT dataset...")
-ds = load_dataset("CARROT-LLM-Routing/SPROUT", split="train")
-print(f"  Total rows: {len(ds)}")
-
-print("\n[3] Extracting labels with robust parsing...")
-
-texts = []
-labels = []
-parse_errors = 0
-all_failed = 0
-label_counter = Counter()
-model_success_counts = Counter()
-model_total_counts = Counter()
-
-for idx, row in enumerate(ds):
-    prompt = str(row.get("prompt") or "")
-    if not prompt or len(prompt) < 10:
-        all_failed += 1
-        continue
-
-    best_tier = 5
-    found_success = False
-
-    for model_col in SPROUT_MODEL_COLS:
-        tier = SPROUT_MODEL_TO_TIER[model_col]
-        score = extract_correctness_score(row.get(model_col))
-        
-        if score is None:
-            parse_errors += 1
-            continue
-        
-        model_total_counts[model_col] += 1
-        if score >= CORRECTNESS_THRESHOLD:
-            model_success_counts[model_col] += 1
-            if not found_success or tier < best_tier:
-                best_tier = tier
-                found_success = True
-
-    if not found_success:
-        all_failed += 1
-        best_tier = 5
-
-    texts.append(prompt[:2000])
-    labels.append(best_tier - 1)
-    label_counter[best_tier] += 1
-
-print(f"\n[3a] Label extraction complete:")
-print(f"    Valid samples:   {len(texts)}")
-print(f"    All-failed:      {all_failed}")
-print(f"    Parse errors:    {parse_errors}")
-print(f"    Label distribution:")
-for tier in range(1, 6):
-    c = label_counter.get(tier, 0)
-    pct = c / max(len(texts), 1) * 100
-    print(f"      Tier {tier}: {c} ({pct:.1f}%)")
-
-print(f"\n[3b] Per-model success rates (@ threshold {CORRECTNESS_THRESHOLD}):")
-for model_col in SPROUT_MODEL_COLS:
-    total = model_total_counts[model_col]
-    if total > 0:
-        succ = model_success_counts[model_col]
-        print(f"    {model_col:<50}: {succ}/{total} = {succ/total*100:.1f}%")
-
-# ── 4. Tokenize ───────────────────────────────────────────────────
-print(f"\n[4] Tokenizing {len(texts)} samples...")
-tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
-encodings = tokenizer(texts, truncation=True, max_length=MAX_SEQ_LEN, padding=False)
-
-class TierDataset(torch.utils.data.Dataset):
-    def __init__(self, encodings, labels):
-        self.encodings = encodings
-        self.labels = labels
-    def __getitem__(self, idx):
-        item = {k: torch.tensor(v[idx]) for k, v in self.encodings.items()}
-        item["labels"] = torch.tensor(self.labels[idx])
-        return item
-    def __len__(self):
-        return len(self.labels)
-
-indices = list(range(len(texts)))
-train_idx, eval_idx = train_test_split(indices, test_size=0.1, random_state=42)
-train_enc = {k: [v[i] for i in train_idx] for k, v in encodings.items()}
-eval_enc = {k: [v[i] for i in eval_idx] for k, v in encodings.items()}
-
-train_ds = TierDataset(train_enc, [labels[i] for i in train_idx])
-eval_ds = TierDataset(eval_enc, [labels[i] for i in eval_idx])
-print(f"  Train: {len(train_ds)}, Eval: {len(eval_ds)}")
-
-# ── 5. Train ──────────────────────────────────────────────────────
-print("\n[5] Training 5-class DistilBERT router...")
-
-model = AutoModelForSequenceClassification.from_pretrained(
-    "distilbert-base-uncased", num_labels=5
-)
-
-training_args = TrainingArguments(
-    output_dir="/tmp/bert_5class_v3",
-    num_train_epochs=EPOCHS,
-    per_device_train_batch_size=BATCH_SIZE,
-    per_device_eval_batch_size=BATCH_SIZE * 2,
-    learning_rate=LR,
-    weight_decay=0.01,
-    warmup_steps=500,
-    eval_strategy="epoch",
-    save_strategy="epoch",
-    load_best_model_at_end=True,
-    metric_for_best_model="eval_accuracy",
-    logging_steps=100,
-    logging_first_step=True,
-    disable_tqdm=True,
-    report_to="none",
-)
-
-def compute_metrics(eval_pred):
-    logits, labels = eval_pred
-    preds = np.argmax(logits, axis=-1)
-    acc = np.mean(preds == labels).item()
-    per_class = {}
-    for c in range(5):
-        mask = labels == c
-        if mask.sum() > 0:
-            per_class[f"acc_tier{c+1}"] = np.mean(preds[mask] == labels[mask]).item()
-    off_by_one = np.mean(np.abs(preds - labels) <= 1).item()
-    return {"accuracy": acc, "off_by_one": off_by_one, **per_class}
-
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=train_ds,
-    eval_dataset=eval_ds,
-    processing_class=tokenizer,
-    data_collator=DataCollatorWithPadding(tokenizer),
-    compute_metrics=compute_metrics,
-)
-
-result = trainer.train()
-final_metrics = trainer.evaluate()
-print(f"\n  Final metrics: {final_metrics}")
-
-# ── 6. Save and upload ────────────────────────────────────────────
-print("\n[6] Saving model...")
-save_dir = "/tmp/bert_5class_v3_final"
-model.save_pretrained(save_dir)
-tokenizer.save_pretrained(save_dir)
-
-meta = {
-    "model": "distilbert-base-uncased",
-    "num_labels": 5,
-    "task": "5-class optimal tier prediction",
-    "training_dataset": "CARROT-LLM-Routing/SPROUT",
-    "n_training_samples": len(train_ds),
-    "correctness_threshold": CORRECTNESS_THRESHOLD,
-    "tier_mapping": SPROUT_MODEL_TO_TIER,
-    "max_seq_len": MAX_SEQ_LEN,
-    "epochs": EPOCHS,
-    "learning_rate": LR,
-    "final_metrics": {k: float(v) if isinstance(v, (np.floating, np.integer)) else v
-                      for k, v in final_metrics.items()},
-    "label_distribution": dict(label_counter),
-}
-with open(os.path.join(save_dir, "metadata.json"), "w") as f:
-    json.dump(meta, f, indent=2)
-print(f"  Saved to {save_dir}")
-
-from huggingface_hub import HfApi
-api = HfApi()
-for fname in os.listdir(save_dir):
-    fpath = os.path.join(save_dir, fname)
-    if os.path.isfile(fpath):
-        api.upload_file(
-            path_or_fileobj=fpath,
-            path_in_repo=f"router_models/bert_5class/{fname}",
-            repo_id=REPO,
-            repo_type="model",
-        )
-        print(f"  Uploaded {fname}")
-
-print("\n" + "="*70)
-print("DONE! BERT 5-class router saved to router_models/bert_5class/")
-print(f"Repo: https://huggingface.co/{REPO}")
-print("="*70)

training/train_router_full.py

@@ -1,433 +0,0 @@
-#!/usr/bin/env python3
-"""Train a learned model router for Agent Cost Optimizer."""
-import json, os, sys, random, pickle, uuid
-import numpy as np
-from datetime import datetime
-from collections import defaultdict
-from typing import Dict, List, Tuple, Any, Optional
-
-TASK_TYPES = ["quick_answer","coding","research","document_drafting",
-              "legal_regulated","tool_heavy","retrieval_heavy",
-              "long_horizon","unknown_ambiguous"]
-TT2IDX = {t:i for i,t in enumerate(TASK_TYPES)}
-
-CODE_KW = ["python","javascript","code","function","bug","debug","refactor",
-           "implement","test","compile","runtime","class","module","async","thread"]
-LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"]
-RESEARCH_KW = ["research","find sources","literature","investigate","compare","analyze","survey"]
-TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
-LONG_KW = ["plan","project","roadmap","orchestrate","multi-step","migrate","pipeline","deploy"]
-MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"]
-
-TIER_STR = {1:0.35,2:0.55,3:0.80,4:0.93,5:0.97}
-TIER_COST = {1:0.05,2:0.15,3:0.75,4:1.0,5:1.5}
-
-TASK_TEMPLATES = {
-    "quick_answer":["What is the capital of France?","Explain quantum computing briefly.",
-        "What is 237*452?","Define photosynthesis.","Who wrote Hamlet?",
-        "What is the speed of light?","List the primary colors.","What is GDP?"],
-    "coding":["Write a Python function to reverse a linked list.",
-        "Fix the bug in this React component.","Refactor auth module to JWT.",
-        "Implement LRU cache in Go.","Debug segfault in C++ thread pool.",
-        "Add unit tests for the payment module.","Optimize this SQL query.",
-        "Create a REST API for user management.","Implement binary search in Rust."],
-    "research":["Research latest transformer advances.",
-        "Find sources comparing LoRA and full FT.",
-        "Investigate data center climate impact.",
-        "Survey privacy-preserving ML techniques.",
-        "Compare reinforcement learning algorithms for robotics."],
-    "document_drafting":["Draft project proposal for ML pipeline.",
-        "Write email to team about deployment.","Create technical report on performance."],
-    "legal_regulated":["Review this contract for liability clauses.",
-        "Check GDPR compliance for data pipeline.","Draft privacy policy section.",
-        "Verify regulatory compliance for medical device software."],
-    "tool_heavy":["Search open issues and create summary.",
-        "Fetch API docs and generate client code.","Query Q3 sales and produce chart."],
-    "retrieval_heavy":["Answer based on 50-page document.",
-        "Find all payment processing mentions.","Retrieve relevant cases for legal query."],
-    "long_horizon":["Plan 3-month roadmap.","Orchestrate multi-region deployment.",
-        "Redesign data architecture end-to-end.","Migrate monolith to microservices."],
-    "unknown_ambiguous":["Help me with this thing.",
-        "I need something about the server.","Can you look into that issue?"],
-}
-
-def tsp(tier, diff):
-    return TIER_STR[tier] ** (diff * 0.6)
-
-def extract_features(request, task_type, difficulty=3):
-    r = request.lower()
-    f = {
-        "req_len": len(request),
-        "num_words": len(request.split()),
-        "has_code": int(any(k in r for k in CODE_KW)),
-        "n_code": sum(1 for k in CODE_KW if k in r),
-        "has_legal": int(any(k in r for k in LEGAL_KW)),
-        "n_legal": sum(1 for k in LEGAL_KW if k in r),
-        "has_research": int(any(k in r for k in RESEARCH_KW)),
-        "n_research": sum(1 for k in RESEARCH_KW if k in r),
-        "has_tool": int(any(k in r for k in TOOL_KW)),
-        "n_tool": sum(1 for k in TOOL_KW if k in r),
-        "has_long": int(any(k in r for k in LONG_KW)),
-        "has_math": int(any(k in r for k in MATH_KW)),
-        "tt_idx": TT2IDX.get(task_type, 8),
-        "difficulty": difficulty,
-    }
-    for tt in TASK_TYPES:
-        f[f"tt_{tt}"] = int(task_type == tt)
-    return f
-
-def gen_trace(idx, rng):
-    tt = rng.choice(list(TASK_TEMPLATES.keys()))
-    diff = {"quick_answer":1,"document_drafting":2,"tool_heavy":2,"retrieval_heavy":2,
-            "research":3,"coding":3,"unknown_ambiguous":3,"long_horizon":4,"legal_regulated":5}[tt]
-    tier_out = {}
-    for t in range(1,6):
-        tier_out[t] = rng.random() < tsp(t, diff)
-    opt = 5
-    for t in range(1,6):
-        if tier_out[t]:
-            opt = t
-            break
-    if diff <= 2:
-        actual = rng.choices([1,2,3,4,5],weights=[3,4,2,1,0.5])[0]
-    elif diff == 3:
-        actual = rng.choices([1,2,3,4,5],weights=[1,2,4,2,1])[0]
-    elif diff == 4:
-        actual = rng.choices([1,2,3,4,5],weights=[0.5,1,2,4,2])[0]
-    else:
-        actual = rng.choices([1,2,3,4,5],weights=[0.2,0.5,1,3,4])[0]
-    outcome = "success" if tier_out[actual] else "failure"
-    req = rng.choice(TASK_TEMPLATES[tt])
-    feats = extract_features(req, tt, diff)
-    return {"feats":feats,"opt":opt,"actual":actual,"outcome":outcome,
-            "tier_out":tier_out,"tt":tt,"diff":diff,"req":req}
-
-print("="*80)
-print("AGENT COST OPTIMIZER - TRAINED ROUTER TRAINING")
-print("="*80)
-
-# ─── Generate Training Data ────────────────────────────────────────
-print("\n[1] Generating 50K training traces...")
-rng = random.Random(42)
-traces = [gen_trace(i, rng) for i in range(50000)]
-print(f"  Generated {len(traces)} traces")
-
-opt_dist = defaultdict(int)
-for t in traces:
-    opt_dist[t["opt"]] += 1
-for k in sorted(opt_dist):
-    print(f"  opt_tier={k}: {opt_dist[k]} ({opt_dist[k]/len(traces)*100:.1f}%)")
-
-# ─── Build Feature Matrix ──────────────────────────────────────────
-print("\n[2] Building feature matrix...")
-
-def feats_to_vec(feats):
-    """Convert feature dict to fixed-order numpy array."""
-    keys = sorted(feats.keys())
-    return np.array([float(feats[k]) for k in keys], dtype=np.float32)
-
-# Get feature key order from first trace
-FEAT_KEYS = sorted(traces[0]["feats"].keys())
-NUM_FEATURES = len(FEAT_KEYS)
-print(f"  Feature count: {NUM_FEATURES}")
-print(f"  Features: {FEAT_KEYS}")
-
-def feats_to_vec_safe(feats):
-    return np.array([float(feats.get(k, 0.0)) for k in FEAT_KEYS], dtype=np.float32)
-
-X_all = np.array([feats_to_vec_safe(t["feats"]) for t in traces])
-y_opt = np.array([t["opt"] for t in traces])
-y_actual = np.array([t["actual"] for t in traces])
-y_outcome = np.array([1 if t["outcome"]=="success" else 0 for t in traces])
-
-# ─── Per-Tier Success Classifiers ───────────────────────────────────
-print("\n[3] Training per-tier P(success|query) classifiers...")
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import classification_report, accuracy_score, f1_score
-
-# For each tier, create binary label: did this tier succeed?
-per_tier_labels = {}
-for tier in range(1, 6):
-    labels = []
-    for t in traces:
-        labels.append(1 if t["tier_out"].get(tier, False) else 0)
-    per_tier_labels[tier] = np.array(labels)
-    succ_rate = per_tier_labels[tier].mean()
-    print(f"  Tier {tier}: success rate = {succ_rate:.3f}")
-
-# Split train/test
-X_train, X_test, idx_train, idx_test = train_test_split(
-    X_all, range(len(traces)), test_size=0.2, random_state=42, stratify=y_opt
-)
-
-print(f"\n  Train: {len(X_train)}, Test: {len(X_test)}")
-
-# ─── XGBoost Per-Tier Classifiers ───────────────────────────────────
-print("\n[4] Training XGBoost per-tier classifiers...")
-import xgboost as xgb
-
-tier_classifiers = {}
-for tier in range(1, 6):
-    y_train_tier = per_tier_labels[tier][idx_train]
-    y_test_tier = per_tier_labels[tier][idx_test]
-
-    clf = xgb.XGBClassifier(
-        n_estimators=100,
-        max_depth=4,
-        learning_rate=0.1,
-        subsample=0.8,
-        colsample_bytree=0.8,
-        objective="binary:logistic",
-        eval_metric="logloss",
-        random_state=42,
-        verbosity=0,
-    )
-    clf.fit(X_train, y_train_tier)
-
-    y_pred = clf.predict(X_test)
-    y_prob = clf.predict_proba(X_test)[:, 1]
-
-    acc = accuracy_score(y_test_tier, y_pred)
-    f1 = f1_score(y_test_tier, y_pred, zero_division=0)
-
-    tier_classifiers[tier] = clf
-    print(f"  Tier {tier}: accuracy={acc:.3f}, f1={f1:.3f}")
-
-# ─── CARROT-Style Router Decision ────────────────────────────────────
-print("\n[5] Building CARROT-style router...")
-
-def route_carrot(features_vec, tier_clfs, mu=0.7):
-    """Route to argmin_tier [mu*(1-P_success_tier) + (1-mu)*cost_tier].
-    
-    mu controls quality-vs-cost tradeoff:
-      mu=1.0: maximize quality only (always frontier)
-      mu=0.0: minimize cost only (always cheapest)
-      mu=0.7: 70% quality, 30% cost (our default)
-    """
-    if features_vec.ndim == 1:
-        features_vec = features_vec.reshape(1, -1)
-
-    best_tier = 3
-    best_score = float("inf")
-
-    for tier in range(1, 6):
-        p_success = tier_clfs[tier].predict_proba(features_vec)[0, 1]
-        cost_norm = TIER_COST[tier] / TIER_COST[5]  # normalize to [0,1]
-        score = mu * (1.0 - p_success) + (1.0 - mu) * cost_norm
-        if score < best_score:
-            best_score = score
-            best_tier = tier
-
-    return best_tier
-
-# Evaluate on test set
-print("\n[6] Evaluating CARROT router on test set...")
-
-mu_values = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
-
-for mu in mu_values:
-    correct = 0
-    total_cost = 0.0
-    unsafe_misses = 0
-    for i in idx_test:
-        t = traces[i]
-        x = feats_to_vec_safe(t["feats"]).reshape(1, -1)
-        pred_tier = route_carrot(x, tier_classifiers, mu=mu)
-        opt_tier = t["opt"]
-
-        # Check if predicted tier would succeed
-        would_succeed = t["tier_out"].get(pred_tier, False)
-        if would_succeed:
-            correct += 1
-
-        # Cost of predicted tier
-        total_cost += TIER_COST[pred_tier]
-
-        # Unsafe miss: predicted cheap tier for hard task
-        if pred_tier < opt_tier and not would_succeed:
-            unsafe_misses += 1
-
-    n_test = len(idx_test)
-    success_rate = correct / n_test
-    avg_cost = total_cost / n_test
-    unsafe_rate = unsafe_misses / n_test
-
-    # Compare to heuristic (task-type based)
-    heuristic_correct = 0
-    heuristic_cost = 0.0
-    for i in idx_test:
-        t = traces[i]
-        # Heuristic: route by task type (from classifier.py)
-        tt = t["tt"]
-        diff = t["diff"]
-        h_tier = min(diff + 1, 5)  # simple: difficulty + 1
-        if t["tier_out"].get(h_tier, False):
-            heuristic_correct += 1
-        heuristic_cost += TIER_COST[h_tier]
-
-    h_success = heuristic_correct / n_test
-    h_avg_cost = heuristic_cost / n_test
-
-    # Frontier baseline
-    frontier_correct = sum(1 for i in idx_test if traces[i]["tier_out"].get(4, False) or traces[i]["tier_out"].get(5, False))
-    frontier_rate = frontier_correct / n_test
-    frontier_avg_cost = TIER_COST[4]  # always tier 4
-
-    print(f"\n  mu={mu:.1f}:")
-    print(f"    CARROT:    success={success_rate:.3f}, avg_cost={avg_cost:.4f}, unsafe_miss={unsafe_rate:.3f}")
-    print(f"    Heuristic: success={h_success:.3f}, avg_cost={h_avg_cost:.4f}")
-    print(f"    Frontier:  success={frontier_rate:.3f}, avg_cost={frontier_avg_cost:.4f}")
-    print(f"    Cost reduction vs frontier: {(1-avg_cost/frontier_avg_cost)*100:.1f}%")
-    print(f"    Cost reduction vs heuristic: {(1-avg_cost/h_avg_cost)*100:.1f}%")
-
-# ─── XGBoost Direct Optimal-Tier Classifier ─────────────────────────
-print("\n\n[7] Training XGBoost direct optimal-tier classifier...")
-
-y_train_opt = y_opt[idx_train] - 1  # XGB needs 0-indexed
-y_test_opt = y_opt[idx_test] - 1
-
-direct_clf = xgb.XGBClassifier(
-    n_estimators=200,
-    max_depth=5,
-    learning_rate=0.1,
-    subsample=0.8,
-    colsample_bytree=0.8,
-    objective="multi:softmax",
-    num_class=6,
-    eval_metric="mlogloss",
-    random_state=42,
-    verbosity=0,
-)
-direct_clf.fit(X_train, y_train_opt)
-
-y_pred_opt = direct_clf.predict(X_test)
-acc_direct = accuracy_score(y_test_opt, y_pred_opt)
-print(f"  Direct classifier accuracy: {acc_direct:.3f}")
-
-# Detailed classification report
-from sklearn.metrics import confusion_matrix
-cm = confusion_matrix(y_test_opt, y_pred_opt, labels=[1,2,3,4,5])
-print(f"\n  Confusion Matrix (rows=true, cols=predicted):")
-print(f"  {'':>10} {'T1':>6} {'T2':>6} {'T3':>6} {'T4':>6} {'T5':>6}")
-for i, tier in enumerate([1,2,3,4,5]):
-    row = f"  True T{tier:>1}:"
-    for j in range(5):
-        row += f" {cm[i][j]:>6}"
-    print(row)
-
-# Evaluate direct classifier on test set
-print("\n[8] Evaluating direct optimal-tier classifier...")
-direct_correct = 0
-direct_cost = 0.0
-direct_unsafe = 0
-direct_underkill = 0
-direct_overkill = 0
-
-for i, test_idx in enumerate(idx_test):
-    t = traces[test_idx]
-    x = feats_to_vec_safe(t["feats"]).reshape(1, -1)
-    pred_tier = int(direct_clf.predict(x)[0]) + 1  # back to 1-indexed
-    opt_tier = t["opt"]
-
-    would_succeed = t["tier_out"].get(pred_tier, False)
-    if would_succeed:
-        direct_correct += 1
-    direct_cost += TIER_COST[pred_tier]
-
-    if pred_tier < opt_tier:
-        direct_underkill += 1
-        if not would_succeed:
-            direct_unsafe += 1
-    elif pred_tier > opt_tier:
-        direct_overkill += 1
-
-n_test = len(idx_test)
-print(f"  Success rate: {direct_correct/n_test:.3f}")
-print(f"  Avg cost: {direct_cost/n_test:.4f}")
-print(f"  Underkill (pred < optimal): {direct_underkill/n_test:.3f}")
-print(f"  Overkill (pred > optimal): {direct_overkill/n_test:.3f}")
-print(f"  Unsafe misses: {direct_unsafe/n_test:.3f}")
-
-# ─── Comparison: All Routers ────────────────────────────────────────
-print("\n\n" + "="*80)
-print("FINAL COMPARISON: ALL ROUTERS ON TEST SET")
-print("="*80)
-
-# 1. Always frontier
-f_succ = sum(1 for i in idx_test if traces[i]["tier_out"].get(4,False) or traces[i]["tier_out"].get(5,False))
-f_cost = TIER_COST[4] * n_test
-
-# 2. Always cheapest
-c_succ = sum(1 for i in idx_test if traces[i]["tier_out"].get(1,False) or traces[i]["tier_out"].get(2,False))
-c_cost = TIER_COST[1] * n_test
-
-# 3. Heuristic (difficulty + 1)
-h_succ = 0; h_cost = 0.0
-for i in idx_test:
-    t = traces[i]
-    h_tier = min(t["diff"] + 1, 5)
-    if t["tier_out"].get(h_tier, False): h_succ += 1
-    h_cost += TIER_COST[h_tier]
-
-# 4. CARROT (best mu)
-best_mu = 0.7
-carrot_succ = 0; carrot_cost = 0.0; carrot_unsafe = 0
-for i in idx_test:
-    t = traces[i]
-    x = feats_to_vec_safe(t["feats"]).reshape(1, -1)
-    pred = route_carrot(x, tier_classifiers, mu=best_mu)
-    if t["tier_out"].get(pred, False): carrot_succ += 1
-    carrot_cost += TIER_COST[pred]
-    if pred < t["opt"] and not t["tier_out"].get(pred, False):
-        carrot_unsafe += 1
-
-# 5. Direct XGB
-d_succ = direct_correct
-d_cost = direct_cost
-d_unsafe = direct_unsafe
-
-# 6. Oracle (always picks optimal)
-o_succ = sum(1 for i in idx_test if traces[i]["tier_out"].get(traces[i]["opt"], False))
-o_cost = sum(TIER_COST[traces[i]["opt"]] for i in idx_test)
-
-print(f"\n{'Router':<20} {'Success':>10} {'AvgCost':>10} {'CostRed':>10} {'Unsafe':>10}")
-print("-"*60)
-for name, succ, cost, unsafe in [
-    ("always_frontier", f_succ, f_cost, 0),
-    ("always_cheap", c_succ, c_cost, 0),
-    ("heuristic", h_succ, h_cost, 0),
-    (f"CARROT(mu={best_mu})", carrot_succ, carrot_cost, carrot_unsafe),
-    ("direct_xgb", d_succ, d_cost, d_unsafe),
-    ("oracle", o_succ, o_cost, 0),
-]:
-    sr = succ/n_test
-    ac = cost/n_test
-    cr = (1 - cost/f_cost)*100
-    um = unsafe/n_test
-    print(f"{name:<20} {sr:>10.3f} {ac:>10.4f} {cr:>9.1f}% {um:>10.3f}")
-
-# ─── Save Models ────────────────────────────────────────────────────
-print("\n\n[9] Saving models...")
-os.makedirs("/app/router_models", exist_ok=True)
-
-# Save per-tier classifiers
-for tier, clf in tier_classifiers.items():
-    clf.save_model(f"/app/router_models/tier_{tier}_success.json")
-    print(f"  Saved tier_{tier}_success.json")
-
-# Save direct classifier
-direct_clf.save_model("/app/router_models/direct_optimal_tier.json")
-print(f"  Saved direct_optimal_tier.json")
-
-# Save feature keys
-with open("/app/router_models/feat_keys.json", "w") as f:
-    json.dump(FEAT_KEYS, f)
-print(f"  Saved feat_keys.json ({len(FEAT_KEYS)} features)")
-
-# Save tier config
-with open("/app/router_models/tier_config.json", "w") as f:
-    json.dump({"tier_cost": TIER_COST, "tier_str": TIER_STR}, f)
-print(f"  Saved tier_config.json")
-
-print("\n\nDONE! Models saved to /app/router_models/")
-print("Next step: integrate trained router into ACO ModelCascadeRouter._route_learned()")

training/train_router_v1.py

@@ -1,433 +0,0 @@
-#!/usr/bin/env python3
-"""Train a learned model router for Agent Cost Optimizer."""
-import json, os, sys, random, pickle, uuid
-import numpy as np
-from datetime import datetime
-from collections import defaultdict
-from typing import Dict, List, Tuple, Any, Optional
-
-TASK_TYPES = ["quick_answer","coding","research","document_drafting",
-              "legal_regulated","tool_heavy","retrieval_heavy",
-              "long_horizon","unknown_ambiguous"]
-TT2IDX = {t:i for i,t in enumerate(TASK_TYPES)}
-
-CODE_KW = ["python","javascript","code","function","bug","debug","refactor",
-           "implement","test","compile","runtime","class","module","async","thread"]
-LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"]
-RESEARCH_KW = ["research","find sources","literature","investigate","compare","analyze","survey"]
-TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
-LONG_KW = ["plan","project","roadmap","orchestrate","multi-step","migrate","pipeline","deploy"]
-MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"]
-
-TIER_STR = {1:0.35,2:0.55,3:0.80,4:0.93,5:0.97}
-TIER_COST = {1:0.05,2:0.15,3:0.75,4:1.0,5:1.5}
-
-TASK_TEMPLATES = {
-    "quick_answer":["What is the capital of France?","Explain quantum computing briefly.",
-        "What is 237*452?","Define photosynthesis.","Who wrote Hamlet?",
-        "What is the speed of light?","List the primary colors.","What is GDP?"],
-    "coding":["Write a Python function to reverse a linked list.",
-        "Fix the bug in this React component.","Refactor auth module to JWT.",
-        "Implement LRU cache in Go.","Debug segfault in C++ thread pool.",
-        "Add unit tests for the payment module.","Optimize this SQL query.",
-        "Create a REST API for user management.","Implement binary search in Rust."],
-    "research":["Research latest transformer advances.",
-        "Find sources comparing LoRA and full FT.",
-        "Investigate data center climate impact.",
-        "Survey privacy-preserving ML techniques.",
-        "Compare reinforcement learning algorithms for robotics."],
-    "document_drafting":["Draft project proposal for ML pipeline.",
-        "Write email to team about deployment.","Create technical report on performance."],
-    "legal_regulated":["Review this contract for liability clauses.",
-        "Check GDPR compliance for data pipeline.","Draft privacy policy section.",
-        "Verify regulatory compliance for medical device software."],
-    "tool_heavy":["Search open issues and create summary.",
-        "Fetch API docs and generate client code.","Query Q3 sales and produce chart."],
-    "retrieval_heavy":["Answer based on 50-page document.",
-        "Find all payment processing mentions.","Retrieve relevant cases for legal query."],
-    "long_horizon":["Plan 3-month roadmap.","Orchestrate multi-region deployment.",
-        "Redesign data architecture end-to-end.","Migrate monolith to microservices."],
-    "unknown_ambiguous":["Help me with this thing.",
-        "I need something about the server.","Can you look into that issue?"],
-}
-
-def tsp(tier, diff):
-    return TIER_STR[tier] ** (diff * 0.6)
-
-def extract_features(request, task_type, difficulty=3):
-    r = request.lower()
-    f = {
-        "req_len": len(request),
-        "num_words": len(request.split()),
-        "has_code": int(any(k in r for k in CODE_KW)),
-        "n_code": sum(1 for k in CODE_KW if k in r),
-        "has_legal": int(any(k in r for k in LEGAL_KW)),
-        "n_legal": sum(1 for k in LEGAL_KW if k in r),
-        "has_research": int(any(k in r for k in RESEARCH_KW)),
-        "n_research": sum(1 for k in RESEARCH_KW if k in r),
-        "has_tool": int(any(k in r for k in TOOL_KW)),
-        "n_tool": sum(1 for k in TOOL_KW if k in r),
-        "has_long": int(any(k in r for k in LONG_KW)),
-        "has_math": int(any(k in r for k in MATH_KW)),
-        "tt_idx": TT2IDX.get(task_type, 8),
-        "difficulty": difficulty,
-    }
-    for tt in TASK_TYPES:
-        f[f"tt_{tt}"] = int(task_type == tt)
-    return f
-
-def gen_trace(idx, rng):
-    tt = rng.choice(list(TASK_TEMPLATES.keys()))
-    diff = {"quick_answer":1,"document_drafting":2,"tool_heavy":2,"retrieval_heavy":2,
-            "research":3,"coding":3,"unknown_ambiguous":3,"long_horizon":4,"legal_regulated":5}[tt]
-    tier_out = {}
-    for t in range(1,6):
-        tier_out[t] = rng.random() < tsp(t, diff)
-    opt = 5
-    for t in range(1,6):
-        if tier_out[t]:
-            opt = t
-            break
-    if diff <= 2:
-        actual = rng.choices([1,2,3,4,5],weights=[3,4,2,1,0.5])[0]
-    elif diff == 3:
-        actual = rng.choices([1,2,3,4,5],weights=[1,2,4,2,1])[0]
-    elif diff == 4:
-        actual = rng.choices([1,2,3,4,5],weights=[0.5,1,2,4,2])[0]
-    else:
-        actual = rng.choices([1,2,3,4,5],weights=[0.2,0.5,1,3,4])[0]
-    outcome = "success" if tier_out[actual] else "failure"
-    req = rng.choice(TASK_TEMPLATES[tt])
-    feats = extract_features(req, tt, diff)
-    return {"feats":feats,"opt":opt,"actual":actual,"outcome":outcome,
-            "tier_out":tier_out,"tt":tt,"diff":diff,"req":req}
-
-print("="*80)
-print("AGENT COST OPTIMIZER - TRAINED ROUTER TRAINING")
-print("="*80)
-
-# ─── Generate Training Data ────────────────────────────────────────
-print("\n[1] Generating 50K training traces...")
-rng = random.Random(42)
-traces = [gen_trace(i, rng) for i in range(50000)]
-print(f"  Generated {len(traces)} traces")
-
-opt_dist = defaultdict(int)
-for t in traces:
-    opt_dist[t["opt"]] += 1
-for k in sorted(opt_dist):
-    print(f"  opt_tier={k}: {opt_dist[k]} ({opt_dist[k]/len(traces)*100:.1f}%)")
-
-# ─── Build Feature Matrix ──────────────────────────────────────────
-print("\n[2] Building feature matrix...")
-
-def feats_to_vec(feats):
-    """Convert feature dict to fixed-order numpy array."""
-    keys = sorted(feats.keys())
-    return np.array([float(feats[k]) for k in keys], dtype=np.float32)
-
-# Get feature key order from first trace
-FEAT_KEYS = sorted(traces[0]["feats"].keys())
-NUM_FEATURES = len(FEAT_KEYS)
-print(f"  Feature count: {NUM_FEATURES}")
-print(f"  Features: {FEAT_KEYS}")
-
-def feats_to_vec_safe(feats):
-    return np.array([float(feats.get(k, 0.0)) for k in FEAT_KEYS], dtype=np.float32)
-
-X_all = np.array([feats_to_vec_safe(t["feats"]) for t in traces])
-y_opt = np.array([t["opt"] for t in traces])
-y_actual = np.array([t["actual"] for t in traces])
-y_outcome = np.array([1 if t["outcome"]=="success" else 0 for t in traces])
-
-# ─── Per-Tier Success Classifiers ───────────────────────────────────
-print("\n[3] Training per-tier P(success|query) classifiers...")
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import classification_report, accuracy_score, f1_score
-
-# For each tier, create binary label: did this tier succeed?
-per_tier_labels = {}
-for tier in range(1, 6):
-    labels = []
-    for t in traces:
-        labels.append(1 if t["tier_out"].get(tier, False) else 0)
-    per_tier_labels[tier] = np.array(labels)
-    succ_rate = per_tier_labels[tier].mean()
-    print(f"  Tier {tier}: success rate = {succ_rate:.3f}")
-
-# Split train/test
-X_train, X_test, idx_train, idx_test = train_test_split(
-    X_all, range(len(traces)), test_size=0.2, random_state=42, stratify=y_opt
-)
-
-print(f"\n  Train: {len(X_train)}, Test: {len(X_test)}")
-
-# ─── XGBoost Per-Tier Classifiers ───────────────────────────────────
-print("\n[4] Training XGBoost per-tier classifiers...")
-import xgboost as xgb
-
-tier_classifiers = {}
-for tier in range(1, 6):
-    y_train_tier = per_tier_labels[tier][idx_train]
-    y_test_tier = per_tier_labels[tier][idx_test]
-
-    clf = xgb.XGBClassifier(
-        n_estimators=100,
-        max_depth=4,
-        learning_rate=0.1,
-        subsample=0.8,
-        colsample_bytree=0.8,
-        objective="binary:logistic",
-        eval_metric="logloss",
-        random_state=42,
-        verbosity=0,
-    )
-    clf.fit(X_train, y_train_tier)
-
-    y_pred = clf.predict(X_test)
-    y_prob = clf.predict_proba(X_test)[:, 1]
-
-    acc = accuracy_score(y_test_tier, y_pred)
-    f1 = f1_score(y_test_tier, y_pred, zero_division=0)
-
-    tier_classifiers[tier] = clf
-    print(f"  Tier {tier}: accuracy={acc:.3f}, f1={f1:.3f}")
-
-# ─── CARROT-Style Router Decision ────────────────────────────────────
-print("\n[5] Building CARROT-style router...")
-
-def route_carrot(features_vec, tier_clfs, mu=0.7):
-    """Route to argmin_tier [mu*(1-P_success_tier) + (1-mu)*cost_tier].
-    
-    mu controls quality-vs-cost tradeoff:
-      mu=1.0: maximize quality only (always frontier)
-      mu=0.0: minimize cost only (always cheapest)
-      mu=0.7: 70% quality, 30% cost (our default)
-    """
-    if features_vec.ndim == 1:
-        features_vec = features_vec.reshape(1, -1)
-
-    best_tier = 3
-    best_score = float("inf")
-
-    for tier in range(1, 6):
-        p_success = tier_clfs[tier].predict_proba(features_vec)[0, 1]
-        cost_norm = TIER_COST[tier] / TIER_COST[5]  # normalize to [0,1]
-        score = mu * (1.0 - p_success) + (1.0 - mu) * cost_norm
-        if score < best_score:
-            best_score = score
-            best_tier = tier
-
-    return best_tier
-
-# Evaluate on test set
-print("\n[6] Evaluating CARROT router on test set...")
-
-mu_values = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
-
-for mu in mu_values:
-    correct = 0
-    total_cost = 0.0
-    unsafe_misses = 0
-    for i in idx_test:
-        t = traces[i]
-        x = feats_to_vec_safe(t["feats"]).reshape(1, -1)
-        pred_tier = route_carrot(x, tier_classifiers, mu=mu)
-        opt_tier = t["opt"]
-
-        # Check if predicted tier would succeed
-        would_succeed = t["tier_out"].get(pred_tier, False)
-        if would_succeed:
-            correct += 1
-
-        # Cost of predicted tier
-        total_cost += TIER_COST[pred_tier]
-
-        # Unsafe miss: predicted cheap tier for hard task
-        if pred_tier < opt_tier and not would_succeed:
-            unsafe_misses += 1
-
-    n_test = len(idx_test)
-    success_rate = correct / n_test
-    avg_cost = total_cost / n_test
-    unsafe_rate = unsafe_misses / n_test
-
-    # Compare to heuristic (task-type based)
-    heuristic_correct = 0
-    heuristic_cost = 0.0
-    for i in idx_test:
-        t = traces[i]
-        # Heuristic: route by task type (from classifier.py)
-        tt = t["tt"]
-        diff = t["diff"]
-        h_tier = min(diff + 1, 5)  # simple: difficulty + 1
-        if t["tier_out"].get(h_tier, False):
-            heuristic_correct += 1
-        heuristic_cost += TIER_COST[h_tier]
-
-    h_success = heuristic_correct / n_test
-    h_avg_cost = heuristic_cost / n_test
-
-    # Frontier baseline
-    frontier_correct = sum(1 for i in idx_test if traces[i]["tier_out"].get(4, False) or traces[i]["tier_out"].get(5, False))
-    frontier_rate = frontier_correct / n_test
-    frontier_avg_cost = TIER_COST[4]  # always tier 4
-
-    print(f"\n  mu={mu:.1f}:")
-    print(f"    CARROT:    success={success_rate:.3f}, avg_cost={avg_cost:.4f}, unsafe_miss={unsafe_rate:.3f}")
-    print(f"    Heuristic: success={h_success:.3f}, avg_cost={h_avg_cost:.4f}")
-    print(f"    Frontier:  success={frontier_rate:.3f}, avg_cost={frontier_avg_cost:.4f}")
-    print(f"    Cost reduction vs frontier: {(1-avg_cost/frontier_avg_cost)*100:.1f}%")
-    print(f"    Cost reduction vs heuristic: {(1-avg_cost/h_avg_cost)*100:.1f}%")
-
-# ─── XGBoost Direct Optimal-Tier Classifier ─────────────────────────
-print("\n\n[7] Training XGBoost direct optimal-tier classifier...")
-
-y_train_opt = y_opt[idx_train] - 1  # XGB needs 0-indexed
-y_test_opt = y_opt[idx_test] - 1
-
-direct_clf = xgb.XGBClassifier(
-    n_estimators=200,
-    max_depth=5,
-    learning_rate=0.1,
-    subsample=0.8,
-    colsample_bytree=0.8,
-    objective="multi:softmax",
-    num_class=6,
-    eval_metric="mlogloss",
-    random_state=42,
-    verbosity=0,
-)
-direct_clf.fit(X_train, y_train_opt)
-
-y_pred_opt = direct_clf.predict(X_test)
-acc_direct = accuracy_score(y_test_opt, y_pred_opt)
-print(f"  Direct classifier accuracy: {acc_direct:.3f}")
-
-# Detailed classification report
-from sklearn.metrics import confusion_matrix
-cm = confusion_matrix(y_test_opt, y_pred_opt, labels=[1,2,3,4,5])
-print(f"\n  Confusion Matrix (rows=true, cols=predicted):")
-print(f"  {'':>10} {'T1':>6} {'T2':>6} {'T3':>6} {'T4':>6} {'T5':>6}")
-for i, tier in enumerate([1,2,3,4,5]):
-    row = f"  True T{tier:>1}:"
-    for j in range(5):
-        row += f" {cm[i][j]:>6}"
-    print(row)
-
-# Evaluate direct classifier on test set
-print("\n[8] Evaluating direct optimal-tier classifier...")
-direct_correct = 0
-direct_cost = 0.0
-direct_unsafe = 0
-direct_underkill = 0
-direct_overkill = 0
-
-for i, test_idx in enumerate(idx_test):
-    t = traces[test_idx]
-    x = feats_to_vec_safe(t["feats"]).reshape(1, -1)
-    pred_tier = int(direct_clf.predict(x)[0]) + 1  # back to 1-indexed
-    opt_tier = t["opt"]
-
-    would_succeed = t["tier_out"].get(pred_tier, False)
-    if would_succeed:
-        direct_correct += 1
-    direct_cost += TIER_COST[pred_tier]
-
-    if pred_tier < opt_tier:
-        direct_underkill += 1
-        if not would_succeed:
-            direct_unsafe += 1
-    elif pred_tier > opt_tier:
-        direct_overkill += 1
-
-n_test = len(idx_test)
-print(f"  Success rate: {direct_correct/n_test:.3f}")
-print(f"  Avg cost: {direct_cost/n_test:.4f}")
-print(f"  Underkill (pred < optimal): {direct_underkill/n_test:.3f}")
-print(f"  Overkill (pred > optimal): {direct_overkill/n_test:.3f}")
-print(f"  Unsafe misses: {direct_unsafe/n_test:.3f}")
-
-# ─── Comparison: All Routers ────────────────────────────────────────
-print("\n\n" + "="*80)
-print("FINAL COMPARISON: ALL ROUTERS ON TEST SET")
-print("="*80)
-
-# 1. Always frontier
-f_succ = sum(1 for i in idx_test if traces[i]["tier_out"].get(4,False) or traces[i]["tier_out"].get(5,False))
-f_cost = TIER_COST[4] * n_test
-
-# 2. Always cheapest
-c_succ = sum(1 for i in idx_test if traces[i]["tier_out"].get(1,False) or traces[i]["tier_out"].get(2,False))
-c_cost = TIER_COST[1] * n_test
-
-# 3. Heuristic (difficulty + 1)
-h_succ = 0; h_cost = 0.0
-for i in idx_test:
-    t = traces[i]
-    h_tier = min(t["diff"] + 1, 5)
-    if t["tier_out"].get(h_tier, False): h_succ += 1
-    h_cost += TIER_COST[h_tier]
-
-# 4. CARROT (best mu)
-best_mu = 0.7
-carrot_succ = 0; carrot_cost = 0.0; carrot_unsafe = 0
-for i in idx_test:
-    t = traces[i]
-    x = feats_to_vec_safe(t["feats"]).reshape(1, -1)
-    pred = route_carrot(x, tier_classifiers, mu=best_mu)
-    if t["tier_out"].get(pred, False): carrot_succ += 1
-    carrot_cost += TIER_COST[pred]
-    if pred < t["opt"] and not t["tier_out"].get(pred, False):
-        carrot_unsafe += 1
-
-# 5. Direct XGB
-d_succ = direct_correct
-d_cost = direct_cost
-d_unsafe = direct_unsafe
-
-# 6. Oracle (always picks optimal)
-o_succ = sum(1 for i in idx_test if traces[i]["tier_out"].get(traces[i]["opt"], False))
-o_cost = sum(TIER_COST[traces[i]["opt"]] for i in idx_test)
-
-print(f"\n{'Router':<20} {'Success':>10} {'AvgCost':>10} {'CostRed':>10} {'Unsafe':>10}")
-print("-"*60)
-for name, succ, cost, unsafe in [
-    ("always_frontier", f_succ, f_cost, 0),
-    ("always_cheap", c_succ, c_cost, 0),
-    ("heuristic", h_succ, h_cost, 0),
-    (f"CARROT(mu={best_mu})", carrot_succ, carrot_cost, carrot_unsafe),
-    ("direct_xgb", d_succ, d_cost, d_unsafe),
-    ("oracle", o_succ, o_cost, 0),
-]:
-    sr = succ/n_test
-    ac = cost/n_test
-    cr = (1 - cost/f_cost)*100
-    um = unsafe/n_test
-    print(f"{name:<20} {sr:>10.3f} {ac:>10.4f} {cr:>9.1f}% {um:>10.3f}")
-
-# ─── Save Models ────────────────────────────────────────────────────
-print("\n\n[9] Saving models...")
-os.makedirs("/app/router_models", exist_ok=True)
-
-# Save per-tier classifiers
-for tier, clf in tier_classifiers.items():
-    clf.save_model(f"/app/router_models/tier_{tier}_success.json")
-    print(f"  Saved tier_{tier}_success.json")
-
-# Save direct classifier
-direct_clf.save_model("/app/router_models/direct_optimal_tier.json")
-print(f"  Saved direct_optimal_tier.json")
-
-# Save feature keys
-with open("/app/router_models/feat_keys.json", "w") as f:
-    json.dump(FEAT_KEYS, f)
-print(f"  Saved feat_keys.json ({len(FEAT_KEYS)} features)")
-
-# Save tier config
-with open("/app/router_models/tier_config.json", "w") as f:
-    json.dump({"tier_cost": TIER_COST, "tier_str": TIER_STR}, f)
-print(f"  Saved tier_config.json")
-
-print("\n\nDONE! Models saved to /app/router_models/")
-print("Next step: integrate trained router into ACO ModelCascadeRouter._route_learned()")

training/train_router_v2.py

@@ -1,510 +0,0 @@
-#!/usr/bin/env python3
-"""Trained Router v2: Safety-first CARROT with tuned mu + safety floors.
-
-Key insight from v1:
-- Per-tier P(success) classifiers work well individually
-- CARROT routing with mu=0.6 beats heuristic on both quality and cost
-- But success rate drops because CARROT routes cheap for hard tasks
-
-Solution: Add SAFETY FLOORS per task type:
-- legal_regulated: never below tier 4
-- coding/research with legal kw: never below tier 3
-- Use P(success) > threshold as gate, fallback to difficulty-based tier
-- When confidence is low, default to tier 3 (medium)
-"""
-import json, os, sys, random, pickle, uuid
-import numpy as np
-from datetime import datetime
-from collections import defaultdict
-
-TASK_TYPES = ["quick_answer","coding","research","document_drafting",
-              "legal_regulated","tool_heavy","retrieval_heavy",
-              "long_horizon","unknown_ambiguous"]
-TT2IDX = {t:i for i,t in enumerate(TASK_TYPES)}
-
-CODE_KW = ["python","javascript","code","function","bug","debug","refactor",
-           "implement","test","compile","runtime","class","module","async","thread"]
-LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"]
-RESEARCH_KW = ["research","find sources","literature","investigate","compare","analyze","survey"]
-TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
-LONG_KW = ["plan","project","roadmap","orchestrate","multi-step","migrate","pipeline","deploy"]
-MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"]
-
-TIER_STR = {1:0.35,2:0.55,3:0.80,4:0.93,5:0.97}
-TIER_COST = {1:0.05,2:0.15,3:0.75,4:1.0,5:1.5}
-
-TASK_TEMPLATES = {
-    "quick_answer":["What is the capital of France?","Explain quantum computing briefly.",
-        "What is 237*452?","Define photosynthesis.","Who wrote Hamlet?",
-        "What is the speed of light?","List the primary colors.","What is GDP?"],
-    "coding":["Write a Python function to reverse a linked list.",
-        "Fix the bug in this React component.","Refactor auth module to JWT.",
-        "Implement LRU cache in Go.","Debug segfault in C++ thread pool.",
-        "Add unit tests for the payment module.","Optimize this SQL query.",
-        "Create a REST API for user management.","Implement binary search in Rust."],
-    "research":["Research latest transformer advances.",
-        "Find sources comparing LoRA and full FT.",
-        "Investigate data center climate impact.",
-        "Survey privacy-preserving ML techniques.",
-        "Compare reinforcement learning algorithms for robotics."],
-    "document_drafting":["Draft project proposal for ML pipeline.",
-        "Write email to team about deployment.","Create technical report on performance."],
-    "legal_regulated":["Review this contract for liability clauses.",
-        "Check GDPR compliance for data pipeline.","Draft privacy policy section.",
-        "Verify regulatory compliance for medical device software."],
-    "tool_heavy":["Search open issues and create summary.",
-        "Fetch API docs and generate client code.","Query Q3 sales and produce chart."],
-    "retrieval_heavy":["Answer based on 50-page document.",
-        "Find all payment processing mentions.","Retrieve relevant cases for legal query."],
-    "long_horizon":["Plan 3-month roadmap.","Orchestrate multi-region deployment.",
-        "Redesign data architecture end-to-end.","Migrate monolith to microservices."],
-    "unknown_ambiguous":["Help me with this thing.",
-        "I need something about the server.","Can you look into that issue?"],
-}
-
-# Safety floors per task type
-TASK_FLOOR = {
-    "legal_regulated": 4,
-    "long_horizon": 3,
-    "research": 3,
-    "coding": 3,
-    "unknown_ambiguous": 3,
-    "quick_answer": 1,
-    "document_drafting": 2,
-    "tool_heavy": 2,
-    "retrieval_heavy": 2,
-}
-
-def tsp(tier, diff):
-    return TIER_STR[tier] ** (diff * 0.6)
-
-def extract_features(request, task_type, difficulty=3):
-    r = request.lower()
-    f = {
-        "req_len": len(request),
-        "num_words": len(request.split()),
-        "has_code": int(any(k in r for k in CODE_KW)),
-        "n_code": sum(1 for k in CODE_KW if k in r),
-        "has_legal": int(any(k in r for k in LEGAL_KW)),
-        "n_legal": sum(1 for k in LEGAL_KW if k in r),
-        "has_research": int(any(k in r for k in RESEARCH_KW)),
-        "n_research": sum(1 for k in RESEARCH_KW if k in r),
-        "has_tool": int(any(k in r for k in TOOL_KW)),
-        "n_tool": sum(1 for k in TOOL_KW if k in r),
-        "has_long": int(any(k in r for k in LONG_KW)),
-        "has_math": int(any(k in r for k in MATH_KW)),
-        "tt_idx": TT2IDX.get(task_type, 8),
-        "difficulty": difficulty,
-    }
-    for tt in TASK_TYPES:
-        f[f"tt_{tt}"] = int(task_type == tt)
-    return f
-
-def gen_trace(idx, rng):
-    tt = rng.choice(list(TASK_TEMPLATES.keys()))
-    diff = {"quick_answer":1,"document_drafting":2,"tool_heavy":2,"retrieval_heavy":2,
-            "research":3,"coding":3,"unknown_ambiguous":3,"long_horizon":4,"legal_regulated":5}[tt]
-    tier_out = {}
-    for t in range(1,6):
-        tier_out[t] = rng.random() < tsp(t, diff)
-    opt = 5
-    for t in range(1,6):
-        if tier_out[t]:
-            opt = t
-            break
-    if diff <= 2:
-        actual = rng.choices([1,2,3,4,5],weights=[3,4,2,1,0.5])[0]
-    elif diff == 3:
-        actual = rng.choices([1,2,3,4,5],weights=[1,2,4,2,1])[0]
-    elif diff == 4:
-        actual = rng.choices([1,2,3,4,5],weights=[0.5,1,2,4,2])[0]
-    else:
-        actual = rng.choices([1,2,3,4,5],weights=[0.2,0.5,1,3,4])[0]
-    outcome = "success" if tier_out[actual] else "failure"
-    req = rng.choice(TASK_TEMPLATES[tt])
-    feats = extract_features(req, tt, diff)
-    return {"feats":feats,"opt":opt,"actual":actual,"outcome":outcome,
-            "tier_out":tier_out,"tt":tt,"diff":diff,"req":req}
-
-print("="*80)
-print("AGENT COST OPTIMIZER - TRAINED ROUTER v2 (Safety-First CARROT)")
-print("="*80)
-
-print("\n[1] Generating 50K training traces...")
-rng = random.Random(42)
-traces = [gen_trace(i, rng) for i in range(50000)]
-print(f"  Generated {len(traces)} traces")
-
-# Feature matrix
-FEAT_KEYS = sorted(traces[0]["feats"].keys())
-NUM_FEATURES = len(FEAT_KEYS)
-
-def f2v(feats):
-    return np.array([float(feats.get(k, 0.0)) for k in FEAT_KEYS], dtype=np.float32)
-
-X_all = np.array([f2v(t["feats"]) for t in traces])
-y_opt = np.array([t["opt"] for t in traces])
-
-# Per-tier labels
-per_tier_labels = {}
-for tier in range(1, 6):
-    per_tier_labels[tier] = np.array([1 if t["tier_out"].get(tier, False) else 0 for t in traces])
-
-# Train/test split
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import accuracy_score, f1_score
-
-X_train, X_test, idx_train, idx_test = train_test_split(
-    X_all, range(len(traces)), test_size=0.2, random_state=42, stratify=y_opt
-)
-print(f"  Train: {len(X_train)}, Test: {len(X_test)}")
-
-# ─── Train Per-Tier XGBoost Classifiers ────────────────────────────
-print("\n[2] Training per-tier P(success) XGBoost classifiers...")
-import xgboost as xgb
-
-tier_clfs = {}
-for tier in range(1, 6):
-    y_tr = per_tier_labels[tier][idx_train]
-    y_te = per_tier_labels[tier][idx_test]
-
-    # Compute scale_pos_weight for imbalanced classes
-    neg = (y_tr == 0).sum()
-    pos = (y_tr == 1).sum()
-    spw = neg / max(pos, 1)
-
-    clf = xgb.XGBClassifier(
-        n_estimators=150, max_depth=5, learning_rate=0.1,
-        subsample=0.8, colsample_bytree=0.8,
-        scale_pos_weight=min(spw, 5.0),
-        objective="binary:logistic", eval_metric="logloss",
-        random_state=42, verbosity=0,
-    )
-    clf.fit(X_train, y_tr)
-
-    y_pred = clf.predict(X_test)
-    acc = accuracy_score(y_te, y_pred)
-    f1 = f1_score(y_te, y_pred, zero_division=0)
-    tier_clfs[tier] = clf
-    print(f"  Tier {tier}: acc={acc:.3f}, f1={f1:.3f}, spw={spw:.2f}")
-
-# ─── Safety-First CARROT Router ─────────────────────────────────────
-print("\n[3] Building safety-first CARROT router...")
-
-def route_safe_carrot(features_vec, tier_clfs, task_type, mu=0.7, 
-                      success_threshold=0.5, safety_floor=None):
-    """Route with safety floors.
-    
-    1. Compute P(success|tier) for each tier
-    2. Apply safety floor per task type
-    3. Pick cheapest tier where P(success) > threshold
-    4. If none meets threshold, escalate to next tier
-    """
-    if features_vec.ndim == 1:
-        features_vec = features_vec.reshape(1, -1)
-
-    floor = safety_floor or TASK_FLOOR.get(task_type, 2)
-
-    # Get per-tier success probabilities
-    p_success = {}
-    for tier in range(1, 6):
-        p_success[tier] = tier_clfs[tier].predict_proba(features_vec)[0, 1]
-
-    # Strategy: Find cheapest tier at or above floor where P(success) > threshold
-    for tier in range(floor, 6):
-        if p_success[tier] >= success_threshold:
-            return tier, p_success
-
-    # Fallback: if no tier meets threshold at floor, try escalating
-    for tier in range(floor + 1, 6):
-        if p_success[tier] >= success_threshold * 0.8:  # relaxed threshold
-            return tier, p_success
-
-    # Last resort: use CARROT scoring at floor
-    best_tier = floor
-    best_score = float("inf")
-    for tier in range(floor, 6):
-        cost_norm = TIER_COST[tier] / TIER_COST[5]
-        score = mu * (1.0 - p_success[tier]) + (1.0 - mu) * cost_norm
-        if score < best_score:
-            best_score = score
-            best_tier = tier
-
-    return best_tier, p_success
-
-# ─── Evaluate ────────────────────────────────────────────────────────
-print("\n[4] Evaluating all routers on test set...")
-
-n_test = len(idx_test)
-results = {}
-
-# Helper: evaluate a router function
-def eval_router(name, route_fn):
-    succ = 0; cost = 0.0; unsafe = 0; false_done = 0
-    tier_dist = defaultdict(int)
-    for i in idx_test:
-        t = traces[i]
-        x = f2v(t["feats"]).reshape(1, -1)
-        pred, _ = route_fn(x, t)
-        tier_dist[pred] += 1
-        if t["tier_out"].get(pred, False):
-            succ += 1
-        else:
-            if pred < t["opt"]:
-                unsafe += 1
-            if pred >= t["opt"] and not t["tier_out"].get(pred, False):
-                false_done += 1
-        cost += TIER_COST[pred]
-    results[name] = {
-        "success": succ/n_test, "avg_cost": cost/n_test,
-        "unsafe_rate": unsafe/n_test, "false_done": false_done/n_test,
-        "tier_dist": dict(tier_dist),
-    }
-
-# 1. Always frontier
-eval_router("always_frontier", lambda x, t: (4, {}))
-
-# 2. Always cheapest
-eval_router("always_cheap", lambda x, t: (1, {}))
-
-# 3. Heuristic (difficulty + 1)
-eval_router("heuristic_diff+1", lambda x, t: (min(t["diff"]+1, 5), {}))
-
-# 4. Heuristic (task floor only)
-eval_router("heuristic_floor", lambda x, t: (TASK_FLOOR.get(t["tt"], 3), {}))
-
-# 5. CARROT v1 (no safety floors, mu=0.6)
-def carrot_v1(x, t):
-    ps = {tier: tier_clfs[tier].predict_proba(x)[0,1] for tier in range(1,6)}
-    best = 3; best_s = float("inf")
-    for tier in range(1,6):
-        s = 0.6*(1-ps[tier]) + 0.4*(TIER_COST[tier]/TIER_COST[5])
-        if s < best_s: best_s = s; best = tier
-    return best, ps
-eval_router("CARROT_v1_mu0.6", carrot_v1)
-
-# 6. Safety-first CARROT (mu=0.7, threshold=0.5)
-def safe_carrot_050(x, t):
-    return route_safe_carrot(x, tier_clfs, t["tt"], mu=0.7, success_threshold=0.5)
-eval_router("safe_CARROT_t0.50", safe_carrot_050)
-
-# 7. Safety-first CARROT (mu=0.7, threshold=0.6)
-def safe_carrot_060(x, t):
-    return route_safe_carrot(x, tier_clfs, t["tt"], mu=0.7, success_threshold=0.6)
-eval_router("safe_CARROT_t0.60", safe_carrot_060)
-
-# 8. Safety-first CARROT (mu=0.7, threshold=0.65)
-def safe_carrot_065(x, t):
-    return route_safe_carrot(x, tier_clfs, t["tt"], mu=0.7, success_threshold=0.65)
-eval_router("safe_CARROT_t0.65", safe_carrot_065)
-
-# 9. Oracle
-eval_router("oracle", lambda x, t: (t["opt"], {}))
-
-# Print comparison
-print(f"\n{'Router':<25} {'Success':>10} {'AvgCost':>10} {'CostRed':>10} {'Unsafe':>10} {'F-DONE':>10}")
-print("-"*75)
-frontier_cost = results["always_frontier"]["avg_cost"]
-for name, r in sorted(results.items(), key=lambda x: -x[1]["success"]):
-    cr = (1 - r["avg_cost"]/frontier_cost)*100
-    print(f"{name:<25} {r['success']:>10.3f} {r['avg_cost']:>10.4f} {cr:>9.1f}% {r['unsafe_rate']:>10.3f} {r['false_done']:>10.3f}")
-
-# ─── Train Improved Direct Classifier ───────────────────────────────
-print("\n\n[5] Training improved direct classifier (0-indexed)...")
-
-y_train_direct = y_opt[idx_train] - 1
-y_test_direct = y_opt[idx_test] - 1
-
-# Use sample weights: penalize underprediction more
-from sklearn.utils.class_weight import compute_sample_weight
-
-# Custom weight: underkill is 3x worse than overkill
-sample_weights = []
-for i in idx_train:
-    t = traces[i]
-    opt = t["opt"]
-    # Weight by inverse frequency + safety penalty
-    sample_weights.append(1.0)
-sample_weights = np.array(sample_weights)
-
-direct_clf = xgb.XGBClassifier(
-    n_estimators=300, max_depth=6, learning_rate=0.05,
-    subsample=0.8, colsample_bytree=0.8,
-    objective="multi:softmax", num_class=5,
-    eval_metric="mlogloss", random_state=42, verbosity=0,
-)
-direct_clf.fit(X_train, y_train_direct, sample_weight=sample_weights)
-
-y_pred_direct = direct_clf.predict(X_test) + 1  # back to 1-indexed
-acc = accuracy_score(y_opt[idx_test], y_pred_direct)
-print(f"  Direct classifier accuracy: {acc:.3f}")
-
-# Evaluate direct classifier with safety floors
-def direct_safe(x, t):
-    pred = int(direct_clf.predict(x)[0]) + 1
-    floor = TASK_FLOOR.get(t["tt"], 2)
-    return max(pred, floor), {}
-
-eval_router("direct_safe_xgb", direct_safe)
-
-# ─── Feature Importance ─────────────────────────────────────────────
-print("\n\n[6] Feature importance (from direct classifier)...")
-imp = direct_clf.feature_importances_
-for feat, score in sorted(zip(FEAT_KEYS, imp), key=lambda x: -x[1])[:10]:
-    print(f"  {feat:<25}: {score:.4f}")
-
-# ─── Save Models ────────────────────────────────────────────────────
-print("\n\n[7] Saving models...")
-os.makedirs("/app/router_models", exist_ok=True)
-for tier, clf in tier_clfs.items():
-    clf.save_model(f"/app/router_models/tier_{tier}_success.json")
-direct_clf.save_model("/app/router_models/direct_optimal_tier.json")
-with open("/app/router_models/feat_keys.json", "w") as f:
-    json.dump(FEAT_KEYS, f)
-with open("/app/router_models/tier_config.json", "w") as f:
-    json.dump({"tier_cost": TIER_COST, "tier_str": TIER_STR, "task_floor": TASK_FLOOR}, f, indent=2)
-
-# Final print
-print(f"\n\n{'='*80}")
-print("FINAL COMPARISON (ALL ROUTERS)")
-print(f"{'='*80}")
-print(f"\n{'Router':<25} {'Success':>10} {'AvgCost':>10} {'CostRed':>10} {'Unsafe':>10} {'F-DONE':>10}")
-print("-"*75)
-frontier_cost = results["always_frontier"]["avg_cost"]
-for name, r in sorted(results.items(), key=lambda x: (-x[1]["success"], x[1]["avg_cost"])):
-    cr = (1 - r["avg_cost"]/frontier_cost)*100
-    print(f"{name:<25} {r['success']:>10.3f} {r['avg_cost']:>10.4f} {cr:>9.1f}% {r['unsafe_rate']:>10.3f} {r['false_done']:>10.3f}")
-
-print(f"\n\nDONE! Models saved to /app/router_models/")
-
-# ─── RouteLLM-Style Binary Router ────────────────────────────────────
-print("\n\n[8] Training RouteLLM-style binary classifiers...")
-print("  (For each tier pair, train: should we route to cheaper or more expensive tier?)")
-
-# For each tier boundary, train a binary classifier
-# tier_boundary[t] = P(should use tier >= t | query)
-# Route to the first tier where the boundary classifier says "yes, this is enough"
-
-boundary_clfs = {}
-for boundary in range(2, 6):
-    # Label: 1 if optimal_tier < boundary (cheaper tier is sufficient)
-    # 0 if optimal_tier >= boundary (need this tier or higher)
-    y_boundary = np.array([1 if traces[i]["opt"] < boundary else 0 for i in range(len(traces))])
-
-    y_tr = y_boundary[idx_train]
-    y_te = y_boundary[idx_test]
-
-    neg = (y_tr == 0).sum()
-    pos = (y_tr == 1).sum()
-    spw = neg / max(pos, 1)
-
-    clf = xgb.XGBClassifier(
-        n_estimators=150, max_depth=5, learning_rate=0.1,
-        subsample=0.8, colsample_bytree=0.8,
-        scale_pos_weight=min(spw, 3.0),
-        objective="binary:logistic", eval_metric="logloss",
-        random_state=42, verbosity=0,
-    )
-    clf.fit(X_train, y_tr)
-
-    y_pred = clf.predict(X_test)
-    acc = accuracy_score(y_te, y_pred)
-    f1 = f1_score(y_te, y_pred, zero_division=0)
-
-    boundary_clfs[boundary] = clf
-    rate = (y_tr == 0).mean()  # fraction that needs this tier
-    print(f"  Boundary {boundary}: acc={acc:.3f}, f1={f1:.3f}, needs_tier={rate:.3f}")
-
-def route_cascade_binary(x, t):
-    """RouteLLM-style cascade: check each boundary, route to first that passes."""
-    if x.ndim == 1:
-        x = x.reshape(1, -1)
-    floor = TASK_FLOOR.get(t["tt"], 2)
-    
-    # Start at floor, check if we need higher
-    current_tier = floor
-    
-    for boundary in range(floor + 1, 6):
-        # boundary_clfs[boundary] predicts P(optimal < boundary)
-        # If P(optimal < boundary) > threshold, we can stay below boundary
-        # i.e., if P(need tier >= boundary) > threshold, escalate
-        p_need_higher = boundary_clfs[boundary].predict_proba(x)[0, 0]  # P(optimal >= boundary)
-        if p_need_higher > 0.4:  # confidence threshold
-            current_tier = boundary
-        else:
-            break
-    
-    return current_tier, {}
-
-eval_router("cascade_binary_t0.4", route_cascade_binary)
-
-def route_cascade_binary_t050(x, t):
-    if x.ndim == 1: x = x.reshape(1, -1)
-    floor = TASK_FLOOR.get(t["tt"], 2)
-    current_tier = floor
-    for boundary in range(floor + 1, 6):
-        p_need = boundary_clfs[boundary].predict_proba(x)[0, 0]
-        if p_need > 0.5:
-            current_tier = boundary
-        else:
-            break
-    return current_tier, {}
-
-eval_router("cascade_binary_t0.5", route_cascade_binary_t050)
-
-def route_cascade_binary_t030(x, t):
-    if x.ndim == 1: x = x.reshape(1, -1)
-    floor = TASK_FLOOR.get(t["tt"], 2)
-    current_tier = floor
-    for boundary in range(floor + 1, 6):
-        p_need = boundary_clfs[boundary].predict_proba(x)[0, 0]
-        if p_need > 0.3:
-            current_tier = boundary
-        else:
-            break
-    return current_tier, {}
-
-eval_router("cascade_binary_t0.3", route_cascade_binary_t030)
-
-# Save boundary classifiers
-for boundary, clf in boundary_clfs.items():
-    clf.save_model(f"/app/router_models/boundary_{boundary}.json")
-    print(f"  Saved boundary_{boundary}.json")
-
-# ─── Final Final Comparison ───────────────────────────────────────────
-print(f"\n\n{'='*80}")
-print("FINAL COMPARISON v2 (WITH BINARY CASCADE ROUTER)")
-print(f"{'='*80}")
-print(f"\n{'Router':<25} {'Success':>10} {'AvgCost':>10} {'CostRed':>10} {'Unsafe':>10} {'F-DONE':>10}")
-print("-"*75)
-frontier_cost = results["always_frontier"]["avg_cost"]
-for name, r in sorted(results.items(), key=lambda x: (-x[1]["success"], x[1]["avg_cost"])):
-    cr = (1 - r["avg_cost"]/frontier_cost)*100
-    # Only show key results
-    if name in ("oracle","always_frontier","heuristic_diff+1","safe_CARROT_t0.60",
-                "cascade_binary_t0.4","cascade_binary_t0.5","cascade_binary_t0.3",
-                "always_cheap"):
-        print(f"{name:<25} {r['success']:>10.3f} {r['avg_cost']:>10.4f} {cr:>9.1f}% {r['unsafe_rate']:>10.3f} {r['false_done']:>10.3f}")
-
-# Find best Pareto
-print("\n\nPARETO FRONTIER:")
-pareto = []
-for name, r in results.items():
-    if name in ("always_cheap",):
-        continue  # skip dominated
-    dominated = False
-    for name2, r2 in results.items():
-        if name == name2: continue
-        if r2["success"] >= r["success"] and r2["avg_cost"] <= r["avg_cost"]:
-            if r2["success"] > r["success"] or r2["avg_cost"] < r["avg_cost"]:
-                dominated = True; break
-    if not dominated:
-        pareto.append((name, r))
-        cr = (1 - r["avg_cost"]/frontier_cost)*100
-        print(f"  {name:<25} success={r['success']:.3f} cost={r['avg_cost']:.4f} costRed={cr:.1f}%")
-
-# Save all results
-with open("/app/router_models/eval_results.json", "w") as f:
-    json.dump(results, f, indent=2, default=str)
-print(f"\n  Saved eval_results.json")
-print(f"\nDONE!")

training/train_router_v3.py

@@ -1,262 +0,0 @@
-#!/usr/bin/env python3
-"""Trained Router v3: Combined best approach.
-Uses per-tier P(success) classifiers + safety floors + 
-cost-aware routing with ASYMMETRIC penalties (underkill penalized 5x harder than overkill).
-"""
-import json, os, sys, random, pickle, uuid
-import numpy as np
-from datetime import datetime
-from collections import defaultdict
-from typing import Dict, List, Tuple, Any, Optional
-
-TASK_TYPES = ["quick_answer","coding","research","document_drafting",
-              "legal_regulated","tool_heavy","retrieval_heavy",
-              "long_horizon","unknown_ambiguous"]
-TT2IDX = {t:i for i,t in enumerate(TASK_TYPES)}
-
-CODE_KW = ["python","javascript","code","function","bug","debug","refactor",
-           "implement","test","compile","runtime","class","module","async","thread"]
-LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"]
-RESEARCH_KW = ["research","find sources","literature","investigate","compare","analyze","survey"]
-TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
-LONG_KW = ["plan","project","roadmap","orchestrate","multi-step","migrate","pipeline","deploy"]
-MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"]
-
-TIER_STR = {1:0.35,2:0.55,3:0.80,4:0.93,5:0.97}
-TIER_COST = {1:0.05,2:0.15,3:0.75,4:1.0,5:1.5}
-TASK_FLOOR = {"legal_regulated":4,"long_horizon":3,"research":3,"coding":3,
-              "unknown_ambiguous":3,"quick_answer":1,"document_drafting":2,
-              "tool_heavy":2,"retrieval_heavy":2}
-
-TASK_TEMPLATES = {
-    "quick_answer":["What is the capital of France?","Explain quantum computing briefly.",
-        "What is 237*452?","Define photosynthesis.","Who wrote Hamlet?",
-        "What is the speed of light?","List the primary colors.","What is GDP?"],
-    "coding":["Write a Python function to reverse a linked list.",
-        "Fix the bug in this React component.","Refactor auth module to JWT.",
-        "Implement LRU cache in Go.","Debug segfault in C++ thread pool.",
-        "Add unit tests for the payment module.","Optimize this SQL query.",
-        "Create a REST API for user management.","Implement binary search in Rust."],
-    "research":["Research latest transformer advances.",
-        "Find sources comparing LoRA and full FT.",
-        "Investigate data center climate impact.",
-        "Survey privacy-preserving ML techniques.",
-        "Compare reinforcement learning algorithms for robotics."],
-    "document_drafting":["Draft project proposal for ML pipeline.",
-        "Write email to team about deployment.","Create technical report on performance."],
-    "legal_regulated":["Review this contract for liability clauses.",
-        "Check GDPR compliance for data pipeline.","Draft privacy policy section.",
-        "Verify regulatory compliance for medical device software."],
-    "tool_heavy":["Search open issues and create summary.",
-        "Fetch API docs and generate client code.","Query Q3 sales and produce chart."],
-    "retrieval_heavy":["Answer based on 50-page document.",
-        "Find all payment processing mentions.","Retrieve relevant cases for legal query."],
-    "long_horizon":["Plan 3-month roadmap.","Orchestrate multi-region deployment.",
-        "Redesign data architecture end-to-end.","Migrate monolith to microservices."],
-    "unknown_ambiguous":["Help me with this thing.",
-        "I need something about the server.","Can you look into that issue?"],
-}
-
-def tsp(tier, diff):
-    return TIER_STR[tier] ** (diff * 0.6)
-
-def extract_features(request, task_type, difficulty=3):
-    r = request.lower()
-    f = {
-        "req_len": len(request), "num_words": len(request.split()),
-        "has_code": int(any(k in r for k in CODE_KW)),
-        "n_code": sum(1 for k in CODE_KW if k in r),
-        "has_legal": int(any(k in r for k in LEGAL_KW)),
-        "n_legal": sum(1 for k in LEGAL_KW if k in r),
-        "has_research": int(any(k in r for k in RESEARCH_KW)),
-        "n_research": sum(1 for k in RESEARCH_KW if k in r),
-        "has_tool": int(any(k in r for k in TOOL_KW)),
-        "n_tool": sum(1 for k in TOOL_KW if k in r),
-        "has_long": int(any(k in r for k in LONG_KW)),
-        "has_math": int(any(k in r for k in MATH_KW)),
-        "tt_idx": TT2IDX.get(task_type, 8), "difficulty": difficulty,
-    }
-    for tt in TASK_TYPES:
-        f[f"tt_{tt}"] = int(task_type == tt)
-    return f
-
-def gen_trace(idx, rng):
-    tt = rng.choice(list(TASK_TEMPLATES.keys()))
-    diff = {"quick_answer":1,"document_drafting":2,"tool_heavy":2,"retrieval_heavy":2,
-            "research":3,"coding":3,"unknown_ambiguous":3,"long_horizon":4,"legal_regulated":5}[tt]
-    tier_out = {}
-    for t in range(1,6):
-        tier_out[t] = rng.random() < tsp(t, diff)
-    opt = 5
-    for t in range(1,6):
-        if tier_out[t]: opt = t; break
-    if diff <= 2: actual = rng.choices([1,2,3,4,5],weights=[3,4,2,1,0.5])[0]
-    elif diff == 3: actual = rng.choices([1,2,3,4,5],weights=[1,2,4,2,1])[0]
-    elif diff == 4: actual = rng.choices([1,2,3,4,5],weights=[0.5,1,2,4,2])[0]
-    else: actual = rng.choices([1,2,3,4,5],weights=[0.2,0.5,1,3,4])[0]
-    outcome = "success" if tier_out[actual] else "failure"
-    req = rng.choice(TASK_TEMPLATES[tt])
-    feats = extract_features(req, tt, diff)
-    return {"feats":feats,"opt":opt,"actual":actual,"outcome":outcome,
-            "tier_out":tier_out,"tt":tt,"diff":diff,"req":req}
-
-print("="*80)
-print("AGENT COST OPTIMIZER - TRAINED ROUTER v3")
-print("Asymmetric cost scoring: underkill 5x penalty")
-print("="*80)
-
-# ─── Generate ────────────────────────────────────────────────────────
-print("\n[1] Generating 50K training traces...")
-rng = random.Random(42)
-traces = [gen_trace(i, rng) for i in range(50000)]
-FEAT_KEYS = sorted(traces[0]["feats"].keys())
-def f2v(feats):
-    return np.array([float(feats.get(k, 0.0)) for k in FEAT_KEYS], dtype=np.float32)
-
-X_all = np.array([f2v(t["feats"]) for t in traces])
-y_opt = np.array([t["opt"] for t in traces])
-
-per_tier_labels = {}
-for tier in range(1,6):
-    per_tier_labels[tier] = np.array([1 if t["tier_out"].get(tier,False) else 0 for t in traces])
-
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import accuracy_score, f1_score
-import xgboost as xgb
-
-X_train, X_test, idx_train, idx_test = train_test_split(X_all, range(len(traces)), test_size=0.2, random_state=42, stratify=y_opt)
-print(f"  Train: {len(X_train)}, Test: {len(X_test)}")
-
-# ─── Train Per-Tier Classifiers ─────────────────────────────────────
-print("\n[2] Training per-tier P(success) classifiers...")
-tier_clfs = {}
-for tier in range(1,6):
-    y_tr = per_tier_labels[tier][idx_train]
-    neg = (y_tr==0).sum(); pos = (y_tr==1).sum()
-    spw = neg / max(pos,1)
-    clf = xgb.XGBClassifier(n_estimators=150, max_depth=5, learning_rate=0.1,
-        subsample=0.8, colsample_bytree=0.8, scale_pos_weight=min(spw,5.0),
-        objective="binary:logistic", eval_metric="logloss", random_state=42, verbosity=0)
-    clf.fit(X_train, y_tr)
-    y_pred = clf.predict(X_test)
-    acc = accuracy_score(per_tier_labels[tier][idx_test], y_pred)
-    f1 = f1_score(per_tier_labels[tier][idx_test], y_pred, zero_division=0)
-    tier_clfs[tier] = clf
-    print(f"  Tier {tier}: acc={acc:.3f}, f1={f1:.3f}")
-
-# ─── Asymmetric Cost Router ──────────────────────────────────────────
-print("\n[3] Building asymmetric cost router...")
-
-def route_asymmetric(x, task_type, tier_clfs, underkill_penalty=5.0, overkill_penalty=1.0):
-    """Score each tier with asymmetric penalties.
-    
-    score(tier) = P(failure@tier) * underkill_penalty * cost_of_failure
-                + cost(tier) * overkill_penalty
-    
-    Underkill (routing too low) is penalized 5x more than overkill.
-    """
-    if x.ndim == 1:
-        x = x.reshape(1, -1)
-    floor = TASK_FLOOR.get(task_type, 2)
-
-    best_tier = floor
-    best_score = float("inf")
-
-    for tier in range(floor, 6):
-        p_fail = 1.0 - tier_clfs[tier].predict_proba(x)[0, 1]
-        cost_norm = TIER_COST[tier] / TIER_COST[5]  # [0.03, 1.0]
-        
-        # Expected cost of failure (cheap model on hard task)
-        failure_cost = p_fail * underkill_penalty
-        
-        # Cost of using this tier (overkill penalty)
-        tier_cost = cost_norm * overkill_penalty
-        
-        score = failure_cost + tier_cost
-        
-        if score < best_score:
-            best_score = score
-            best_tier = tier
-
-    return best_tier
-
-# ─── Evaluate ─────────────────────────────────────────────────────────
-print("\n[4] Evaluating all routers...")
-n_test = len(idx_test)
-results = {}
-
-def eval_router(name, route_fn):
-    succ = 0; cost = 0.0; unsafe = 0; fd = 0
-    td = defaultdict(int)
-    for i in idx_test:
-        t = traces[i]
-        x = f2v(t["feats"]).reshape(1,-1)
-        pred = route_fn(x, t)
-        td[pred] += 1
-        if t["tier_out"].get(pred, False): succ += 1
-        elif pred < t["opt"]: unsafe += 1
-        elif pred >= t["opt"]: fd += 1
-        cost += TIER_COST[pred]
-    results[name] = {"success":succ/n_test, "avg_cost":cost/n_test,
-                     "unsafe_rate":unsafe/n_test, "false_done":fd/n_test,
-                     "tier_dist":dict(td)}
-
-eval_router("always_frontier", lambda x,t: 4)
-eval_router("always_cheap", lambda x,t: 1)
-eval_router("heuristic_diff+1", lambda x,t: min(t["diff"]+1,5))
-eval_router("heuristic_floor", lambda x,t: TASK_FLOOR.get(t["tt"],2))
-
-for ukp in [3.0, 5.0, 8.0, 10.0, 15.0]:
-    eval_router(f"asymmetric_uk{ukp:.0f}", lambda x,t,uk=ukp: route_asymmetric(x, t["tt"], tier_clfs, underkill_penalty=uk))
-
-eval_router("oracle", lambda x,t: t["opt"])
-
-# Print comparison
-print(f"\n{'Router':<25} {'Success':>10} {'AvgCost':>10} {'CostRed':>10} {'Unsafe':>10} {'F-DONE':>10}")
-print("-"*75)
-fc = results["always_frontier"]["avg_cost"]
-for name, r in sorted(results.items(), key=lambda x: (-x[1]["success"], x[1]["avg_cost"])):
-    cr = (1 - r["avg_cost"]/fc)*100
-    print(f"{name:<25} {r['success']:>10.3f} {r['avg_cost']:>10.4f} {cr:>9.1f}% {r['unsafe_rate']:>10.3f} {r['false_done']:>10.3f}")
-
-# Pareto frontier
-print("\nPARETO FRONTIER:")
-pareto = []
-for name, r in results.items():
-    if name == "always_cheap": continue
-    dominated = False
-    for name2, r2 in results.items():
-        if name == name2: continue
-        if r2["success"] >= r["success"] and r2["avg_cost"] <= r["avg_cost"]:
-            if r2["success"] > r["success"] or r2["avg_cost"] < r["avg_cost"]:
-                dominated = True; break
-    if not dominated:
-        pareto.append((name, r))
-        cr = (1 - r["avg_cost"]/fc)*100
-        print(f"  {name:<25} success={r['success']:.3f} cost={r['avg_cost']:.4f} costRed={cr:.1f}% unsafe={r['unsafe_rate']:.3f}")
-
-# ─── Save Best Model ─────────────────────────────────────────────────
-print("\n[5] Saving models...")
-os.makedirs("/app/router_models", exist_ok=True)
-for tier, clf in tier_clfs.items():
-    clf.save_model(f"/app/router_models/tier_{tier}_success.json")
-with open("/app/router_models/feat_keys.json","w") as f:
-    json.dump(FEAT_KEYS, f)
-with open("/app/router_models/tier_config.json","w") as f:
-    json.dump({"tier_cost":TIER_COST,"tier_str":TIER_STR,"task_floor":TASK_FLOOR}, f, indent=2)
-with open("/app/router_models/eval_results_v3.json","w") as f:
-    json.dump(results, f, indent=2, default=str)
-print(f"  Saved all models to /app/router_models/")
-
-# Find best config
-best_name = ""
-best_score = -float("inf")
-for name, r in results.items():
-    if name in ("oracle","always_cheap"): continue
-    # Composite: success*20 - cost*50 - unsafe*100
-    score = r["success"]*20 - r["avg_cost"]*50 - r["unsafe_rate"]*100
-    if score > best_score:
-        best_score = score
-        best_name = name
-print(f"\n  BEST CONFIG: {best_name} (composite score: {best_score:.2f})")
-print(f"\nDONE!")

training/train_router_v4_production.py

@@ -1,241 +0,0 @@
-#!/usr/bin/env python3
-"""Production Trained Router v4: Difficulty-first + ML refinement.
-
-Architecture:
-1. Predict difficulty (task_type -> difficulty mapping + ML difficulty classifier)
-2. Convert difficulty to base_tier (difficulty + 1, capped at 5)
-3. Apply safety floor per task_type
-4. Use per-tier P(success) classifiers to CONFIRM or ESCALATE
-5. If P(success@base_tier) < 0.65, escalate to next tier
-
-This combines the best of both worlds:
-- Difficulty mapping (heuristic) is reliable and interpretable
-- ML classifiers add a safety net: they catch cases where difficulty is underestimated
-- Safety floors prevent dangerous under-routing on legal/critical tasks
-"""
-import json, os, sys, random, uuid
-import numpy as np
-from datetime import datetime
-from collections import defaultdict
-from typing import Dict, List, Optional, Any
-
-import xgboost as xgb
-
-# ─── Load Models ──────────────────────────────────────────────────────
-MODEL_DIR = "/app/router_models"
-feat_keys = json.load(open(f"{MODEL_DIR}/feat_keys.json"))
-tier_config = json.load(open(f"{MODEL_DIR}/tier_config.json"))
-TIER_COST = {int(k):v for k,v in tier_config["tier_cost"].items()}
-TIER_STR = {int(k):v for k,v in tier_config["tier_str"].items()}
-TASK_FLOOR = tier_config["task_floor"]
-
-tier_clfs = {}
-for tier in range(1, 6):
-    clf = xgb.XGBClassifier()
-    clf.load_model(f"{MODEL_DIR}/tier_{tier}_success.json")
-    tier_clfs[tier] = clf
-
-# ─── Feature Extraction ────────────────────────────────────────────────
-TASK_TYPES = ["quick_answer","coding","research","document_drafting",
-              "legal_regulated","tool_heavy","retrieval_heavy",
-              "long_horizon","unknown_ambiguous"]
-TT2IDX = {t:i for i,t in enumerate(TASK_TYPES)}
-
-CODE_KW = ["python","javascript","code","function","bug","debug","refactor",
-           "implement","test","compile","runtime","class","module","async","thread"]
-LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"]
-RESEARCH_KW = ["research","find sources","literature","investigate","compare","analyze","survey"]
-TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
-LONG_KW = ["plan","project","roadmap","orchestrate","multi-step","migrate","pipeline","deploy"]
-MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"]
-
-def extract_features(request, task_type, difficulty=3):
-    r = request.lower()
-    f = {"req_len":len(request),"num_words":len(request.split()),
-        "has_code":int(any(k in r for k in CODE_KW)),
-        "n_code":sum(1 for k in CODE_KW if k in r),
-        "has_legal":int(any(k in r for k in LEGAL_KW)),
-        "n_legal":sum(1 for k in LEGAL_KW if k in r),
-        "has_research":int(any(k in r for k in RESEARCH_KW)),
-        "n_research":sum(1 for k in RESEARCH_KW if k in r),
-        "has_tool":int(any(k in r for k in TOOL_KW)),
-        "n_tool":sum(1 for k in TOOL_KW if k in r),
-        "has_long":int(any(k in r for k in LONG_KW)),
-        "has_math":int(any(k in r for k in MATH_KW)),
-        "tt_idx":TT2IDX.get(task_type,8),"difficulty":difficulty}
-    for tt in TASK_TYPES:
-        f[f"tt_{tt}"] = int(task_type == tt)
-    return f
-
-def f2v(feats):
-    return np.array([float(feats.get(k,0.0)) for k in feat_keys], dtype=np.float32)
-
-# ─── Production Router ─────────────────────────────────────────────────
-def route_production(request, task_type, difficulty, escalation_threshold=0.65):
-    """Production router: difficulty-first + ML confirmation.
-    
-    Step 1: difficulty -> base_tier (difficulty + 1, capped at 5)
-    Step 2: Apply safety floor (task_type -> minimum tier)
-    Step 3: base_tier = max(base_tier, safety_floor)
-    Step 4: If P(success@base_tier) < escalation_threshold, escalate one tier
-    Step 5: Never exceed tier 5
-    """
-    base_tier = min(difficulty + 1, 5)
-    floor = TASK_FLOOR.get(task_type, 2)
-    base_tier = max(base_tier, floor)
-    
-    # ML confirmation: check if base_tier is likely to succeed
-    feats = extract_features(request, task_type, difficulty)
-    x = f2v(feats).reshape(1, -1)
-    
-    p_success = tier_clfs[base_tier].predict_proba(x)[0, 1]
-    
-    # If P(success) at base_tier is too low, escalate
-    while p_success < escalation_threshold and base_tier < 5:
-        base_tier += 1
-        p_success = tier_clfs[base_tier].predict_proba(x)[0, 1]
-    
-    return base_tier
-
-# ─── Generate Evaluation ────────────────────────────────────────────────
-TASK_TEMPLATES_EVAL = {
-    "quick_answer":["What is the capital of France?","Explain quantum computing briefly.",
-        "What is 237*452?","Define photosynthesis.","Who wrote Hamlet?"],
-    "coding":["Write a Python function to reverse a linked list.",
-        "Fix the bug in this React component.","Refactor auth module to JWT.",
-        "Implement LRU cache in Go.","Debug segfault in C++ thread pool."],
-    "research":["Research latest transformer advances.",
-        "Find sources comparing LoRA and full FT.",
-        "Investigate data center climate impact."],
-    "document_drafting":["Draft project proposal for ML pipeline.",
-        "Write email to team about deployment.","Create technical report on performance."],
-    "legal_regulated":["Review this contract for liability clauses.",
-        "Check GDPR compliance for data pipeline.","Draft privacy policy section."],
-    "tool_heavy":["Search open issues and create summary.",
-        "Fetch API docs and generate client code.","Query Q3 sales and produce chart."],
-    "retrieval_heavy":["Answer based on 50-page document.",
-        "Find all payment processing mentions.","Retrieve relevant cases for legal query."],
-    "long_horizon":["Plan 3-month roadmap.","Orchestrate multi-region deployment.",
-        "Redesign data architecture end-to-end."],
-    "unknown_ambiguous":["Help me with this thing.",
-        "I need something about the server.","Can you look into that issue?"],
-}
-
-def tsp(tier, diff):
-    s = {1:0.35,2:0.55,3:0.80,4:0.93,5:0.97}[tier]
-    return s ** (diff * 0.6)
-
-print("="*80)
-print("PRODUCTION TRAINED ROUTER v4 BENCHMARK")
-print("="*80)
-print("\nArchitecture: difficulty-first + ML confirmation + safety floors")
-
-print("\n[1] Generating 2K eval traces (seed=999)...")
-eval_rng = random.Random(999)
-eval_traces = []
-for i in range(2000):
-    tt = eval_rng.choice(list(TASK_TEMPLATES_EVAL.keys()))
-    diff = {"quick_answer":1,"document_drafting":2,"tool_heavy":2,"retrieval_heavy":2,
-            "research":3,"coding":3,"unknown_ambiguous":3,"long_horizon":4,"legal_regulated":5}[tt]
-    tier_out = {t: eval_rng.random() < tsp(t, diff) for t in range(1,6)}
-    opt = 5
-    for t in range(1,6):
-        if tier_out[t]: opt = t; break
-    req = eval_rng.choice(TASK_TEMPLATES_EVAL[tt])
-    eval_traces.append({"tt":tt,"diff":diff,"opt":opt,"tier_out":tier_out,"req":req})
-print(f"  Generated {len(eval_traces)} traces")
-
-# ─── Evaluate ──────────────────────────────────────────────────────────
-print("\n[2] Evaluating all routers...")
-
-def eval_router(name, route_fn):
-    succ=0; cost=0.0; unsafe=0; fd=0; td=defaultdict(int)
-    for t in eval_traces:
-        pred = route_fn(t)
-        td[pred] += 1
-        if t["tier_out"].get(pred, False): succ += 1
-        elif pred < t["opt"]: unsafe += 1
-        else: fd += 1
-        cost += TIER_COST[pred]
-    n = len(eval_traces)
-    return {"success":succ/n, "avg_cost":cost/n, "unsafe_rate":unsafe/n,
-            "false_done":fd/n, "tier_dist":dict(td)}
-
-# Baselines
-results = {}
-results["always_frontier"] = eval_router("always_frontier", lambda t: 4)
-results["always_cheap"] = eval_router("always_cheap", lambda t: 1)
-results["heuristic_diff+1"] = eval_router("heuristic_diff+1", lambda t: min(t["diff"]+1, 5))
-results["heuristic_floor"] = eval_router("heuristic_floor", lambda t: TASK_FLOOR.get(t["tt"], 2))
-results["oracle"] = eval_router("oracle", lambda t: t["opt"])
-
-# Production router at different escalation thresholds
-for threshold in [0.50, 0.55, 0.60, 0.65, 0.70, 0.75]:
-    name = f"prod_t{threshold:.2f}"
-    results[name] = eval_router(name, 
-        lambda t, th=threshold: route_production(t["req"], t["tt"], t["diff"], escalation_threshold=th))
-
-# Print comparison
-print(f"\n{'Router':<25} {'Success':>10} {'AvgCost':>10} {'CostRed':>10} {'Unsafe':>10} {'F-DONE':>10}")
-print("-"*75)
-fc = results["always_frontier"]["avg_cost"]
-for name, r in sorted(results.items(), key=lambda x: (-x[1]["success"], x[1]["avg_cost"])):
-    cr = (1 - r["avg_cost"]/fc)*100
-    print(f"{name:<25} {r['success']:>10.3f} {r['avg_cost']:>10.4f} {cr:>9.1f}% {r['unsafe_rate']:>10.3f} {r['false_done']:>10.3f}")
-
-# Find best production config
-print("\n\n[3] Finding best production config...")
-best_name = ""
-best_score = -float("inf")
-for name, r in results.items():
-    if name in ("oracle","always_cheap"): continue
-    # Composite: maximize success, minimize cost and unsafe
-    score = r["success"]*20 - r["avg_cost"]*30 - r["unsafe_rate"]*100
-    if score > best_score:
-        best_score = score
-        best_name = name
-
-print(f"  Best: {best_name} (composite: {best_score:.2f})")
-print(f"  Success: {results[best_name]['success']:.3f}")
-print(f"  Avg cost: {results[best_name]['avg_cost']:.4f}")
-fc_val = results["always_frontier"]["avg_cost"]
-cr = (1 - results[best_name]["avg_cost"]/fc_val)*100
-print(f"  Cost reduction vs frontier: {cr:.1f}%")
-print(f"  Unsafe rate: {results[best_name]['unsafe_rate']:.3f}")
-print(f"  False-DONE rate: {results[best_name]['false_done']:.3f}")
-
-# Per-task breakdown for best
-print(f"\n\n[4] Per-task breakdown for {best_name}...")
-for tt in sorted(set(t["tt"] for t in eval_traces)):
-    tt_traces = [t for t in eval_traces if t["tt"] == tt]
-    n_tt = len(tt_traces)
-    if n_tt == 0: continue
-    
-    for rname, rfn in [("frontier", lambda t:4),
-                       ("heuristic", lambda t:min(t["diff"]+1,5)),
-                       ("trained", lambda t:route_production(t["req"],t["tt"],t["diff"],
-                           escalation_threshold=float(best_name.split("t")[1]))),
-                       ("oracle", lambda t:t["opt"])]:
-        succ = sum(1 for t in tt_traces if t["tier_out"].get(rfn(t), False))
-        cost = sum(TIER_COST[rfn(t)] for t in tt_traces)
-        sr = succ/n_tt; ac = cost/n_tt
-        if rname == "frontier":
-            print(f"\n  {tt} (n={n_tt}):")
-        cr = (1 - ac/fc_val)*100
-        print(f"    {rname:<12} success={sr:.3f} cost={ac:.4f} costRed={cr:.1f}%")
-
-# Pareto
-print("\n\nPARETO FRONTIER:")
-for name, r in results.items():
-    if name == "always_cheap": continue
-    dominated = False
-    for name2, r2 in results.items():
-        if name == name2: continue
-        if r2["success"] >= r["success"] and r2["avg_cost"] <= r["avg_cost"]:
-            if r2["success"] > r["success"] or r2["avg_cost"] < r["avg_cost"]:
-                dominated = True; break
-    if not dominated:
-        cr = (1 - r["avg_cost"]/fc_val)*100
-        print(f"  {name:<25} success={r['success']:.3f} cost={r['avg_cost']:.4f} costRed={cr:.1f}% unsafe={r['unsafe_rate']:.3f}")
-
-print(f"\nDONE!")