ok

arena/eval/run_frontier.py

@@ -160,16 +160,14 @@ DEFAULT_LIVE_CONFIG: List[Dict[str, str]] = [
     {"label": "claude-haiku-4-5", "model": "claude-haiku-4-5-20251001",
      "api_base": "https://api.anthropic.com/v1",
      "api_key_env": "ANTHROPIC_API_KEY"},
-    # qwen-1.5b-base = the SAME base model your trained LoRA starts from,
-    # served REMOTELY via HF Inference Providers (OpenAI-compatible router
-    # at router.huggingface.co). This proves the trained adapter beats the
-    # zero-shot base on equal footing. Never loads locally — `api_base` is
-    # set so the OpenAI client only ever issues HTTPS calls.
-    {"label": "qwen-1.5b-base",  "model": "Qwen/Qwen2.5-1.5B-Instruct",
-     "api_base": "https://router.huggingface.co/v1",
-     "api_key_env": "HF_TOKEN"},
 ]
 
+# Qwen2.5-1.5B-Instruct is too small for HF's free router or Together to
+# bother hosting. Evaluate it INSIDE Colab (Track A) where the GPU is
+# already loaded, then merge results via scripts/merge_colab_eval.py.
+# That keeps the apples-to-apples base-vs-trained comparison while
+# guaranteeing the laptop never loads weights.
+
 
 def _assert_remote(api_base: Optional[str], label: str) -> None:
     """Raise loudly if a config would cause a LOCAL model load.

reports/frontier_smoke.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": 1777121050.448101,
+  "generated_at": 1777121986.5616581,
   "providers": {
     "random": {
       "eval_during": {
@@ -9,17 +9,17 @@
           "n": 1
         },
         "frame_validity": {
-          "mean": 1.0,
+          "mean": 0.8571,
           "std": 0.0,
           "n": 1
         },
         "plan_quality": {
-          "mean": 0.5,
+          "mean": 0.25,
           "std": 0.0,
           "n": 1
         },
         "final_reward": {
-          "mean": 0.5411,
+          "mean": 0.5428,
           "std": 0.0,
           "n": 1
         },
@@ -40,7 +40,7 @@
         },
         "n_episodes": 1
       },
-      "runtime_s": 0.01
+      "runtime_s": 0.03
     },
     "keyword": {
       "eval_during": {
@@ -132,7 +132,7 @@
           "n": 1
         },
         "frame_validity": {
-          "mean": 0.0,
+          "mean": 1.0,
           "std": 0.0,
           "n": 1
         },
@@ -142,7 +142,7 @@
           "n": 1
         },
         "final_reward": {
-          "mean": 0.4766,
+          "mean": 0.454,
           "std": 0.0,
           "n": 1
         },
@@ -163,16 +163,16 @@
         },
         "n_episodes": 1
       },
-      "runtime_s": 23.62,
+      "runtime_s": 3.81,
       "api_calls": {
-        "ok": 12,
-        "api_error": 0,
+        "ok": 0,
+        "api_error": 12,
         "json_error": 0,
         "total": 12,
-        "hit_rate": 1.0,
-        "tokens_in": 9008,
-        "tokens_out": 774,
-        "first_error": null
+        "hit_rate": 0.0,
+        "tokens_in": 0,
+        "tokens_out": 0,
+        "first_error": "AuthenticationError: Error code: 401 - {'error': {'message': \"You didn't provide an API key. You need to provide your API key in an Authorization header using Bearer auth (i.e. Authorization: Bearer YOUR_KEY), or as the password field (with blank username) if you're accessing the API from your browser and are prompted for a username and password. You can obtain an API key from https://platform.openai.com/account/api-keys.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}"
       }
     },
     "claude-haiku-4-5": {
@@ -188,12 +188,12 @@
           "n": 1
         },
         "plan_quality": {
-          "mean": 0.5,
+          "mean": 0.0,
           "std": 0.0,
           "n": 1
         },
         "final_reward": {
-          "mean": 0.5236,
+          "mean": 0.454,
           "std": 0.0,
           "n": 1
         },
@@ -214,16 +214,16 @@
         },
         "n_episodes": 1
       },
-      "runtime_s": 23.66,
+      "runtime_s": 5.03,
       "api_calls": {
-        "ok": 12,
-        "api_error": 0,
+        "ok": 0,
+        "api_error": 12,
         "json_error": 0,
         "total": 12,
-        "hit_rate": 1.0,
-        "tokens_in": 10089,
-        "tokens_out": 1357,
-        "first_error": null
+        "hit_rate": 0.0,
+        "tokens_in": 0,
+        "tokens_out": 0,
+        "first_error": "AuthenticationError: Error code: 401 - {'error': {'code': 'authentication_error', 'message': 'Invalid Anthropic API Key', 'type': 'invalid_request_error', 'param': None}}"
       }
     }
   }

scripts/merge_colab_eval.py

@@ -0,0 +1,84 @@
+"""Merge Colab-side eval results into reports/frontier.json.
+
+Track A runs Qwen2.5-1.5B-Instruct (base AND trained-with-LoRA) inside
+Google Colab where the GPU is already paid for. This script takes the
+JSON file Colab produces and inserts it into the local frontier.json
+under the right provider label, so scoring and plotting see all
+columns side by side.
+
+Expected Colab-side JSON shape — produced by `arena.eval.harness.run_eval`:
+
+    {
+      "eval_pre":    {"task_correctness": {"mean":..., ...}, ...},
+      "eval_during": {...},
+      "eval_hard":   {...},
+      "drift_adjusted_success_rate": {"value": 0.123},
+      "runtime_s": 42.0
+    }
+
+Usage:
+    # In Colab, after Cell 7 (final eval), each block looks like:
+    #   results_base    = run_eval(base_policy,    ...)
+    #   results_trained = run_eval(trained_policy, ...)
+    #   json.dump(results_base,    open("qwen_1_5b_base.json","w"), default=str)
+    #   json.dump(results_trained, open("qwen_1_5b_trained.json","w"), default=str)
+    #   # download both files via Colab's Files panel.
+
+    python scripts/merge_colab_eval.py \\
+        --frontier reports/frontier.json \\
+        --label qwen-1.5b-base    --json qwen_1_5b_base.json
+    python scripts/merge_colab_eval.py \\
+        --frontier reports/frontier.json \\
+        --label trained           --json qwen_1_5b_trained.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--frontier", default="reports/frontier.json",
+                    help="Local frontier.json to merge into.")
+    ap.add_argument("--label", required=True,
+                    help="Provider label, e.g. 'qwen-1.5b-base' or 'trained'.")
+    ap.add_argument("--json", required=True,
+                    help="Path to the JSON file downloaded from Colab.")
+    args = ap.parse_args(argv)
+
+    if not os.path.exists(args.json):
+        print(f"[merge] missing colab JSON: {args.json}", file=sys.stderr)
+        return 2
+
+    new_block = json.load(open(args.json))
+
+    if not isinstance(new_block, dict) or "eval_during" not in new_block:
+        print(f"[merge] {args.json} doesn't look like a run_eval() output "
+              f"(missing 'eval_during' key)", file=sys.stderr)
+        return 3
+
+    if os.path.exists(args.frontier):
+        fr = json.load(open(args.frontier))
+    else:
+        fr = {"providers": {}}
+    fr.setdefault("providers", {})
+
+    fr["providers"][args.label] = new_block
+
+    os.makedirs(os.path.dirname(args.frontier) or ".", exist_ok=True)
+    with open(args.frontier, "w") as f:
+        json.dump(fr, f, indent=2, default=str)
+
+    tc = new_block.get("eval_during", {}).get("task_correctness", {}).get("mean")
+    print(f"[merge] {args.label} merged into {args.frontier}  "
+          f"during.task_correctness={tc}")
+    print(f"[merge] providers now: {list(fr['providers'].keys())}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())