Upload folder using huggingface_hub

src/mathcompose/eval/prmbench.py

@@ -106,7 +106,7 @@ def evaluate_batched(
 
     ds = load_dataset(dataset_id, split=split)
     if limit is not None:
-        ds = ds.select(range(min(limit, len(ds))))
+        ds = ds.shuffle(seed=0).select(range(min(limit, len(ds))))
 
     mod_items, mod_gold, mod_cat = [], [], []
     orig_items = []

src/mathcompose/eval/processbench.py

@@ -135,7 +135,8 @@ def evaluate_batched(
     for split in splits:
         ds = load_dataset(dataset_id, split=split)
         if limit is not None:
-            ds = ds.select(range(min(limit, len(ds))))
+            # shuffle first: ProcessBench splits are ordered, so first-N is class-biased
+            ds = ds.shuffle(seed=0).select(range(min(limit, len(ds))))
         items = [(row["problem"], list(row["steps"])) for row in ds]
         golds = [int(row["label"]) for row in ds]
         preds = verify_batch_predictions(

src/mathcompose/train/colab_train.ipynb

@@ -16,9 +16,11 @@
    "execution_count": null,
    "outputs": [],
    "source": [
-    "# 1. GPU deps (+ 4-bit kernels)\n",
+    "# 1. GPU deps (+ 4-bit kernels). Remove Colab's incompatible torchao (breaks PEFT\n",
+    "#    adapter loading; unused by our QLoRA/LoRA path).\n",
     "!pip -q install \"transformers>=5.13,<6\" \"trl>=1.7,<2\" \"peft>=0.19,<0.20\" \\\n",
-    "    \"accelerate>=1.14\" \"datasets>=5.0\" \"huggingface_hub>=1.22\" sympy bitsandbytes"
+    "    \"accelerate>=1.14\" \"datasets>=5.0\" \"huggingface_hub>=1.22\" sympy bitsandbytes\n",
+    "!pip -q uninstall -y torchao 2>/dev/null; echo \"torchao removed (ok if it was absent)\""
    ]
   },
   {
@@ -69,9 +71,8 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## 5. Eval V on ProcessBench \u2014 base vs tuned (now batched \u2192 fast)\n",
-    "Fast first read below (1 split, small limit, Maj@4). For the headline number,\n",
-    "drop `--splits`/raise `--limit` and use `--maj-k 8`."
+    "## 5. Eval V on ProcessBench \u2014 base vs tuned (batched; limited slices are sampled randomly)\n",
+    "Fast first read (1 split, Maj@4). For the headline, drop `--splits`/`--limit`, use `--maj-k 8`."
    ]
   },
   {
@@ -80,10 +81,9 @@
    "execution_count": null,
    "outputs": [],
    "source": [
-    "# fast sanity read (~a few min on A100):\n",
     "!python -m mathcompose.eval.run processbench --tag base  --maj-k 1 --limit 100 --splits gsm8k\n",
     "!python -m mathcompose.eval.run processbench --adapter runs/verifier_v --tag tuned --maj-k 4 --limit 100 --splits gsm8k\n",
-    "# full run (all 4 splits, Maj@8) \u2014 uncomment when the fast read looks right:\n",
+    "# full run when the fast read looks right:\n",
     "# !python -m mathcompose.eval.run processbench --tag base  --maj-k 1\n",
     "# !python -m mathcompose.eval.run processbench --adapter runs/verifier_v --tag tuned --maj-k 8\n",
     "!python -m mathcompose.eval.run prmbench --adapter runs/verifier_v --tag tuned --maj-k 1 --limit 200\n",
@@ -114,8 +114,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## 7. (Later) Model G + composition\n",
-    "Build G's dataset (teacher=claude-opus-4-8), train, then V-reranks-G."
+    "## 7. (Later) Model G + composition"
    ]
   },
   {