anima_closeout v12: smoke + both relay arms TRAINED; the LoRA control died in peft's torchao dispatcher (Colab preinstalls torchao 0.10.0, peft raises wanting >=0.16; nothing here uses torchao -> CELL 0 uninstalls it, probe then returns False and dispatch falls to standard Linear). run_arm is now IDEMPOTENT: an arm with its final-step save present is reused, never retrained — the campaign resumes from local files after any crash.

colab/anima_closeout.ipynb

@@ -139,6 +139,15 @@
     "    sh('pip install -q \"amoe-lora[diffusion] @ '\n",
     "       'git+https://github.com/AbstractEyes/amoe-lora\"')\n",
     "    ANIMA[\"DEPS_DONE\"] = True\n",
+    "# Colab preinstalls torchao 0.10.0; peft's LoRA torchao dispatcher RAISES\n",
+    "# on it ('only versions above 0.16.0 are supported') \u2014 killing the LoRA\n",
+    "# control arms. Nothing in this stack uses torchao; with it ABSENT peft's\n",
+    "# probe returns False and dispatch falls through to the standard Linear.\n",
+    "import importlib.util\n",
+    "if importlib.util.find_spec(\"torchao\"):\n",
+    "    sh(\"pip uninstall -y -q torchao\")\n",
+    "    importlib.invalidate_caches()\n",
+    "    assert importlib.util.find_spec(\"torchao\") is None, \"torchao survived\"\n",
     "import amoe, deepspeed\n",
     "print(f\"amoe {amoe.__version__} | deepspeed {deepspeed.__version__}\")\n",
     "\n",
@@ -506,7 +515,19 @@
     "            lf.write(line)\n",
     "        return proc.wait()\n",
     "\n",
+    "def _arm_max_steps(tag):\n",
+    "    import tomllib\n",
+    "    return tomllib.loads(ARMS[tag].read_text())[\"max_steps\"]\n",
+    "\n",
     "def run_arm(tag, resume=False):\n",
+    "    # IDEMPOTENT: an arm whose FINAL-step save already exists is never\n",
+    "    # retrained \u2014 after any crash the campaign resumes from local files,\n",
+    "    # skipping straight past every finished arm.\n",
+    "    done = sorted(\n",
+    "        glob.glob(str(RUNS / tag / \"*\" / f\"step{_arm_max_steps(tag)}\")))\n",
+    "    if done:\n",
+    "        print(f\"[{tag}] already trained \u2014 reusing {done[-1]}\")\n",
+    "        return Path(done[-1])\n",
     "    log = RUNS / f\"{tag}.log\"\n",
     "    flags = \" --resume_from_checkpoint\" if resume else \"\"\n",
     "    t0 = time.time()\n",