Anima close-out campaign notebook: exp004b (relay retrain, fp32 masters, 2 declared seeds, real held-out eval) + exp017 (first multiband3 on a DiT), Colab/Blackwell-96GB, ship-on-completion per arm

colab/anima_closeout.ipynb

@@ -0,0 +1,924 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 5,
+ "metadata": {
+  "colab": {
+   "provenance": []
+  },
+  "kernelspec": {
+   "name": "python3",
+   "display_name": "Python 3"
+  },
+  "language_info": {
+   "name": "python"
+  },
+  "accelerator": "GPU"
+ },
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# AMoE Anima close-out \u2014 exp004b + exp017 (Colab / Blackwell 96GB)\n",
+    "\n",
+    "Closes the r2 campaign's Anima gap:\n",
+    "\n",
+    "| | what | why |\n",
+    "|---|---|---|\n",
+    "| **exp004b** | relay retrain: 2 declared seeds, 10k steps, full dataset, held-out eval, LoRA-r16 control | exp004 was one ~40-min run, no results.json, still improving at cutoff |\n",
+    "| **exp017** | multiband3 on the Anima DiT \u2014 first live run of the fork's mode | the band mechanism has never run on a DiT |\n",
+    "| **the fix** | `bf16_master_weights = true` + declared `seed` | exp004's 28 gates were **quantization-frozen**: plain Adam stepped bf16 leaves directly; ~4.5e-4 updates vs 0.0078 half-ULP at \u22123.0 rounded to no-ops (proven from the salvaged Adam state) |\n",
+    "\n",
+    "Run cells top to bottom. Everything is idempotent: after a disconnect,\n",
+    "re-run **CELL 0**, then re-run the interrupted run cell (checkpoints every\n",
+    "30 min; pass `resume=True` to `run_arm`).\n",
+    "\n",
+    "**Needs:** Colab secret `HF_TOKEN` (notebook access ON) with write access\n",
+    "to `AbstractPhil/geolip-aleph-diffusion`.\n",
+    "\n",
+    "**License:** Anima weights are NON-COMMERCIAL (CircleStone NC + NVIDIA\n",
+    "Open Model License); every checkpoint produced here inherits NC.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# \u2550\u2550\u2550 CELL 0 \u2014 ACTIVATION (idempotent; re-run after any disconnect) \u2550\u2550\u2550\u2550\u2550\u2550\n",
+    "import json, os, subprocess, sys, time\n",
+    "from pathlib import Path\n",
+    "\n",
+    "if \"ANIMA\" not in globals():                       # paste-ahead guard\n",
+    "    ANIMA = {}\n",
+    "\n",
+    "WORK = Path(\"/content/anima_campaign\")\n",
+    "FORK = Path(\"/content/diffusion-pipe\")\n",
+    "DATA, RUNS, TOMLS = WORK / \"data\", WORK / \"runs\", WORK / \"tomls\"\n",
+    "for _d in (WORK, DATA, RUNS, TOMLS):\n",
+    "    _d.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "def sh(cmd, check=True):\n",
+    "    print(f\"$ {cmd}\")\n",
+    "    rc = subprocess.run(cmd, shell=True, text=True).returncode\n",
+    "    if check and rc != 0:\n",
+    "        raise RuntimeError(f\"rc={rc}: {cmd}\")\n",
+    "    return rc\n",
+    "\n",
+    "# -- 1. GPU + Blackwell riders --------------------------------------------\n",
+    "import torch\n",
+    "assert torch.cuda.is_available(), \"No GPU \u2014 Runtime > Change runtime type.\"\n",
+    "_cap = torch.cuda.get_device_capability(0)\n",
+    "print(f\"GPU: {torch.cuda.get_device_name(0)} | sm_{_cap[0]}{_cap[1]} | \"\n",
+    "      f\"{torch.cuda.get_device_properties(0).total_memory / 2**30:.0f} GB | \"\n",
+    "      f\"torch {torch.__version__} cu{torch.version.cuda}\")\n",
+    "if _cap >= (12, 0):\n",
+    "    _tv = tuple(int(x) for x in torch.__version__.split(\"+\")[0].split(\".\")[:2])\n",
+    "    _cv = tuple(int(x) for x in (torch.version.cuda or \"0.0\").split(\".\")[:2])\n",
+    "    if not (_tv >= (2, 7) and _cv >= (12, 8)):\n",
+    "        raise RuntimeError(\n",
+    "            f\"sm_120 needs torch>=2.7 + cu>=12.8; runtime has \"\n",
+    "            f\"{torch.__version__}/cu{torch.version.cuda}. Do NOT blind-pip \"\n",
+    "            \"torch \u2014 switch runtimes, or deliberately install the matching \"\n",
+    "            \"cu128 build from pytorch.org.\")\n",
+    "    (torch.randn(256, 256, device=\"cuda\")\n",
+    "     @ torch.randn(256, 256, device=\"cuda\")).sum().item()\n",
+    "    print(\"sm_120 matmul: OK\")\n",
+    "try:\n",
+    "    import flash_attn  # noqa: F401\n",
+    "    raise RuntimeError(\"flash-attn is installed and BROKEN on sm_120 \u2014 \"\n",
+    "                       \"`pip uninstall flash-attn`; SDPA is the path.\")\n",
+    "except ImportError:\n",
+    "    pass\n",
+    "\n",
+    "# -- 2. HF token (never printed) ------------------------------------------\n",
+    "_tok = os.environ.get(\"HF_TOKEN\")\n",
+    "if not _tok:\n",
+    "    try:\n",
+    "        from google.colab import userdata\n",
+    "        _tok = userdata.get(\"HF_TOKEN\")\n",
+    "    except Exception:\n",
+    "        _tok = None\n",
+    "if not _tok:\n",
+    "    raise RuntimeError(\"Add HF_TOKEN in Colab secrets (key icon) with \"\n",
+    "                       \"notebook access ON, then re-run this cell.\")\n",
+    "os.environ[\"HF_TOKEN\"] = _tok\n",
+    "print(\"HF token: present (not printed)\")\n",
+    "\n",
+    "# -- 3. fork + deps -------------------------------------------------------\n",
+    "if not (FORK / \"train.py\").exists():\n",
+    "    sh(f\"git clone -q https://github.com/AbstractEyes/diffusion-pipe {FORK}\")\n",
+    "sh(f\"git -C {FORK} fetch -q origin\")\n",
+    "sh(f\"git -C {FORK} checkout -q origin/main\")\n",
+    "sh(f\"git -C {FORK} submodule update --init submodules/ComfyUI\")\n",
+    "print(\"fork @\", subprocess.run(\n",
+    "    [\"git\", \"-C\", str(FORK), \"log\", \"--oneline\", \"-1\"],\n",
+    "    capture_output=True, text=True).stdout.strip())\n",
+    "_src = (FORK / \"train.py\").read_text()\n",
+    "assert \"bf16_master_weights\" in _src and \"run_seed\" in _src, \\\n",
+    "    \"fork main lacks the campaign patches (bf16_master_weights + seed)\"\n",
+    "\n",
+    "if not ANIMA.get(\"DEPS_DONE\"):\n",
+    "    sh(f\"pip install -q -r {FORK}/requirements.txt\")\n",
+    "    sh('pip install -q \"amoe-lora[diffusion] @ '\n",
+    "       'git+https://github.com/AbstractEyes/amoe-lora\"')\n",
+    "    ANIMA[\"DEPS_DONE\"] = True\n",
+    "import amoe, deepspeed\n",
+    "print(f\"amoe {amoe.__version__} | deepspeed {deepspeed.__version__}\")\n",
+    "\n",
+    "# -- 4. Anima weights (NC) ------------------------------------------------\n",
+    "from huggingface_hub import hf_hub_download\n",
+    "MODELS = WORK / \"models\" / \"anima\"\n",
+    "for _f in (\"diffusion_models/anima-base-v1.0.safetensors\",\n",
+    "           \"text_encoders/qwen_3_06b_base.safetensors\",\n",
+    "           \"vae/qwen_image_vae.safetensors\"):\n",
+    "    hf_hub_download(\"circlestone-labs/Anima\", f\"split_files/{_f}\",\n",
+    "                    local_dir=str(MODELS))\n",
+    "print(\"Anima split_files present \u2014 NC weights; derived ckpts inherit NC\")\n",
+    "\n",
+    "# -- 5. dataset columns, verified BEFORE anything spends ------------------\n",
+    "DATASET = \"AbstractPhil/qwen-deepfashion-fused\"\n",
+    "from huggingface_hub import HfApi\n",
+    "SHARDS = sorted(f for f in HfApi().list_repo_files(\n",
+    "    DATASET, repo_type=\"dataset\") if f.endswith(\".parquet\"))\n",
+    "assert SHARDS, f\"no parquet shards in {DATASET}\"\n",
+    "import pyarrow.parquet as pq\n",
+    "_p0 = hf_hub_download(DATASET, SHARDS[0], repo_type=\"dataset\")\n",
+    "_cols = set(pq.read_schema(_p0).names)\n",
+    "_missing = {\"image\", \"caption_vlm_json\"} - _cols\n",
+    "assert not _missing, f\"dataset lacks {_missing}; has {sorted(_cols)[:24]}\"\n",
+    "ANIMA.update(DATASET=DATASET, SHARDS=SHARDS,\n",
+    "             HAS_WH={\"image_width\", \"image_height\"} <= _cols,\n",
+    "             ID_COL=\"id\" if \"id\" in _cols else None)\n",
+    "print(f\"dataset OK: {len(SHARDS)} shard(s); wh cols {ANIMA['HAS_WH']}; \"\n",
+    "      f\"id col {ANIMA['ID_COL']}\")\n",
+    "print(\"\\n=== READY ===\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# \u2550\u2550\u2550 CELL 1 \u2014 materialize the held-out split + write all TOMLs \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n",
+    "# Deterministic row-hash split: 512 eval rows, disjoint by construction,\n",
+    "# recorded in split_manifest.json (the reproducibility anchor). Both sides\n",
+    "# read the SAME materialized parquet. caption_type='text' = exp004's\n",
+    "# verbatim-JSON recipe (flattened-'json' is the recorded alternative).\n",
+    "import hashlib, json\n",
+    "import pyarrow as pa\n",
+    "import pyarrow.parquet as pq\n",
+    "from huggingface_hub import hf_hub_download\n",
+    "\n",
+    "EVAL_N = 512\n",
+    "TRAIN_DIR = DATA / \"train\"; TRAIN_DIR.mkdir(exist_ok=True)\n",
+    "EVAL_PARQUET = DATA / \"eval.parquet\"\n",
+    "MANIFEST = DATA / \"split_manifest.json\"\n",
+    "\n",
+    "if not MANIFEST.exists():\n",
+    "    _paths = [hf_hub_download(ANIMA[\"DATASET\"], s, repo_type=\"dataset\")\n",
+    "              for s in ANIMA[\"SHARDS\"]]\n",
+    "\n",
+    "    def _row_key(si, ri, batch, j):\n",
+    "        if ANIMA[\"ID_COL\"]:\n",
+    "            return str(batch.column(ANIMA[\"ID_COL\"])[j].as_py())\n",
+    "        cap = str(batch.column(\"caption_vlm_json\")[j].as_py())[:256]\n",
+    "        return f\"{si}:{ri}:{hashlib.sha1(cap.encode()).hexdigest()[:12]}\"\n",
+    "\n",
+    "    _key_cols = [c for c in (ANIMA[\"ID_COL\"], \"caption_vlm_json\") if c]\n",
+    "    ranked = []\n",
+    "    for si, p in enumerate(_paths):\n",
+    "        ri = 0\n",
+    "        for batch in pq.ParquetFile(p).iter_batches(batch_size=256,\n",
+    "                                                    columns=_key_cols):\n",
+    "            for j in range(batch.num_rows):\n",
+    "                k = _row_key(si, ri, batch, j)\n",
+    "                ranked.append(\n",
+    "                    (hashlib.sha256(k.encode()).hexdigest(), si, ri, k))\n",
+    "                ri += 1\n",
+    "    ranked.sort()\n",
+    "    eval_set = {(si, ri) for _h, si, ri, _k in ranked[:EVAL_N]}\n",
+    "    total = len(ranked)\n",
+    "\n",
+    "    ew = tw = None\n",
+    "    for si, p in enumerate(_paths):\n",
+    "        ri = 0\n",
+    "        for batch in pq.ParquetFile(p).iter_batches(batch_size=64):\n",
+    "            tbl = pa.Table.from_batches([batch])\n",
+    "            emask = [(si, ri + j) in eval_set for j in range(batch.num_rows)]\n",
+    "            ri += batch.num_rows\n",
+    "            esub = tbl.filter(pa.array(emask))\n",
+    "            tsub = tbl.filter(pa.array([not m for m in emask]))\n",
+    "            if esub.num_rows:\n",
+    "                if ew is None:\n",
+    "                    ew = pq.ParquetWriter(EVAL_PARQUET, esub.schema)\n",
+    "                ew.write_table(esub)\n",
+    "            if tsub.num_rows:\n",
+    "                if tw is None:\n",
+    "                    tw = pq.ParquetWriter(TRAIN_DIR / \"train.parquet\",\n",
+    "                                          tsub.schema)\n",
+    "                tw.write_table(tsub)\n",
+    "    for w in (ew, tw):\n",
+    "        if w:\n",
+    "            w.close()\n",
+    "    MANIFEST.write_text(json.dumps(\n",
+    "        {\"dataset\": ANIMA[\"DATASET\"], \"total_rows\": total, \"eval_n\": EVAL_N,\n",
+    "         \"eval_ids\": [k for _h, _s, _r, k in ranked[:EVAL_N]],\n",
+    "         \"rule\": \"sha256(row_key) ascending; first EVAL_N rows are eval\"},\n",
+    "        indent=1))\n",
+    "    print(f\"split: {total} rows -> {total - EVAL_N} train + {EVAL_N} eval\")\n",
+    "else:\n",
+    "    print(\"split already materialized:\",\n",
+    "          json.loads(MANIFEST.read_text())[\"total_rows\"], \"rows\")\n",
+    "\n",
+    "# -- dataset TOMLs (the exp004 recipe) ------------------------------------\n",
+    "_wh = (\"image_width_column = 'image_width'\\n\"\n",
+    "       \"image_height_column = 'image_height'\\n\") if ANIMA[\"HAS_WH\"] else \"\"\n",
+    "\n",
+    "def _dataset_toml(files):\n",
+    "    return f\"\"\"# generated by anima_closeout.ipynb\n",
+    "resolutions = [512]\n",
+    "enable_ar_bucket = true\n",
+    "min_ar = 0.5\n",
+    "max_ar = 2.0\n",
+    "num_ar_buckets = 7\n",
+    "cache_backend = 'parquet'\n",
+    "cache_shard_size_mb = 350\n",
+    "\n",
+    "[[directory]]\n",
+    "type = 'parquet'\n",
+    "parquet_files = '{files}'\n",
+    "image_column = 'image'\n",
+    "{_wh}caption_column = 'caption_vlm_json'\n",
+    "caption_type = 'text'\n",
+    "num_repeats = 1\n",
+    "skip_empty_caption = true\n",
+    "\"\"\"\n",
+    "\n",
+    "(TOMLS / \"dataset_train.toml\").write_text(\n",
+    "    _dataset_toml(f\"{TRAIN_DIR}/*.parquet\"))\n",
+    "(TOMLS / \"dataset_eval.toml\").write_text(_dataset_toml(str(EVAL_PARQUET)))\n",
+    "\n",
+    "# -- training TOMLs per arm -----------------------------------------------\n",
+    "_AP = WORK / \"models\" / \"anima\" / \"split_files\"\n",
+    "\n",
+    "def _arm_toml(tag, seed, steps, mode=\"relay\", lora=False):\n",
+    "    out = RUNS / tag\n",
+    "    L = [f\"# {tag} \u2014 generated by anima_closeout.ipynb\",\n",
+    "         f\"output_dir = '{out}'\",\n",
+    "         f\"dataset = '{TOMLS}/dataset_train.toml'\",\n",
+    "         f\"seed = {seed}\",\n",
+    "         \"bf16_master_weights = true   # the exp004 gate-freeze fix\",\n",
+    "         \"epochs = 1000\",\n",
+    "         f\"max_steps = {steps}\",\n",
+    "         \"micro_batch_size_per_gpu = 8\",\n",
+    "         \"pipeline_stages = 1\",\n",
+    "         \"gradient_accumulation_steps = 1\",\n",
+    "         \"gradient_clipping = 0\",\n",
+    "         \"save_every_n_steps = 2500\",\n",
+    "         \"checkpoint_every_n_minutes = 30\",\n",
+    "         \"activation_checkpointing = false   # 96GB rider\",\n",
+    "         \"save_dtype = 'bfloat16'\",\n",
+    "         \"caching_batch_size = 8\",\n",
+    "         f\"eval_datasets = [{{name = 'heldout', config = '{TOMLS}/dataset_eval.toml'}}]\",\n",
+    "         \"eval_every_n_steps = 1000\",\n",
+    "         \"eval_before_first_step = true\",\n",
+    "         \"eval_micro_batch_size_per_gpu = 8\",\n",
+    "         \"eval_gradient_accumulation_steps = 1\",\n",
+    "         \"\", \"[model]\", \"type = 'anima'\",\n",
+    "         f\"transformer_path = '{_AP}/diffusion_models/anima-base-v1.0.safetensors'\",\n",
+    "         f\"llm_path = '{_AP}/text_encoders/qwen_3_06b_base.safetensors'\",\n",
+    "         f\"vae_path = '{_AP}/vae/qwen_image_vae.safetensors'\",\n",
+    "         \"dtype = 'bfloat16'\",\n",
+    "         \"timestep_sample_method = 'logit_normal'\"]\n",
+    "    if lora:\n",
+    "        L += [\"self_attn_lr = 1e-3\", \"cross_attn_lr = 1e-3\", \"mlp_lr = 1e-3\",\n",
+    "              \"mod_lr = 1e-3\", \"llm_adapter_lr = 1e-3\",\n",
+    "              \"\", \"[adapter]\", \"type = 'lora'\", \"rank = 16\",\n",
+    "              \"dtype = 'bfloat16'\"]\n",
+    "    else:\n",
+    "        L += [\"aleph_relay = true\",\n",
+    "              f\"aleph_relay_mode = '{mode}'\",\n",
+    "              \"aleph_relay_rank = 16\",\n",
+    "              \"aleph_relay_every = 1\",\n",
+    "              \"aleph_relay_lr = 1e-3\",\n",
+    "              \"self_attn_lr = 0\", \"cross_attn_lr = 0\", \"mlp_lr = 0\",\n",
+    "              \"mod_lr = 0\",\n",
+    "              \"llm_adapter_lr = 0   # NON-NEGOTIABLE (anima-trainer law)\"]\n",
+    "    L += [\"\", \"[optimizer]\",\n",
+    "          \"type = 'adam'   # pure Adam \u2014 never adamw on aleph paths\",\n",
+    "          (\"lr = 1e-3\" if lora else \"lr = 0\"),\n",
+    "          \"betas = [0.9, 0.99]\",\n",
+    "          \"weight_decay = 0.0\",\n",
+    "          \"\", \"[monitoring]\", \"enable_wandb = false\"]\n",
+    "    p = TOMLS / f\"{tag}.toml\"\n",
+    "    p.write_text(\"\\n\".join(L) + \"\\n\")\n",
+    "    return p\n",
+    "\n",
+    "ARMS = {\n",
+    "    \"smoke\":          _arm_toml(\"smoke\", 0, 200),\n",
+    "    \"e004b_relay_s0\": _arm_toml(\"e004b_relay_s0\", 0, 10000),\n",
+    "    \"e004b_relay_s1\": _arm_toml(\"e004b_relay_s1\", 1, 10000),\n",
+    "    \"e004b_lora_s0\":  _arm_toml(\"e004b_lora_s0\", 0, 10000, lora=True),\n",
+    "    \"e004b_lora_s1\":  _arm_toml(\"e004b_lora_s1\", 1, 10000, lora=True),\n",
+    "    \"e017_mb3_smoke\": _arm_toml(\"e017_mb3_smoke\", 0, 200, mode=\"multiband3\"),\n",
+    "    \"e017_mb3_s0\":    _arm_toml(\"e017_mb3_s0\", 0, 10000, mode=\"multiband3\"),\n",
+    "    \"e017_mb3_s1\":    _arm_toml(\"e017_mb3_s1\", 1, 10000, mode=\"multiband3\"),\n",
+    "}\n",
+    "print(\"TOMLs written:\", \", \".join(ARMS))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# \u2550\u2550\u2550 CELL 2 \u2014 helpers + SMOKE + the G-gate assert \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n",
+    "# The smoke = 200 real-recipe steps. Jobs: prove the fp32-master fix in\n",
+    "# flight (gates MUST move \u2014 the exact exp004 failure), prove eval wiring\n",
+    "# (eval_before_first_step fires at step 0), build the latent cache\n",
+    "# (~0.5h, reused by every later arm).\n",
+    "import glob, io, json, os, subprocess, time\n",
+    "from pathlib import Path\n",
+    "import torch\n",
+    "\n",
+    "HUB_REPO = \"AbstractPhil/geolip-aleph-diffusion\"\n",
+    "\n",
+    "def run_arm(tag, resume=False):\n",
+    "    log = RUNS / f\"{tag}.log\"\n",
+    "    flags = \"--resume_from_checkpoint\" if resume else \"\"\n",
+    "    t0 = time.time()\n",
+    "    rc = sh(f\"cd {FORK} && NCCL_P2P_DISABLE=1 NCCL_IB_DISABLE=1 \"\n",
+    "            f\"deepspeed --num_gpus=1 train.py --deepspeed \"\n",
+    "            f\"--config {ARMS[tag]} {flags} 2>&1 | tee -a {log}\",\n",
+    "            check=False)\n",
+    "    print(f\"[{tag}] rc={rc} wall={(time.time() - t0) / 60:.1f} min\")\n",
+    "    if rc != 0:\n",
+    "        raise RuntimeError(f\"{tag} failed \u2014 read {log}\")\n",
+    "    return newest_save(tag)\n",
+    "\n",
+    "def latest_run_dir(tag):\n",
+    "    run_dirs = sorted(glob.glob(str(RUNS / tag / \"*\")))\n",
+    "    assert run_dirs, f\"no run dir for {tag}\"\n",
+    "    return Path(run_dirs[-1])\n",
+    "\n",
+    "def newest_save(tag):\n",
+    "    rd = latest_run_dir(tag)\n",
+    "    saves = sorted(glob.glob(str(rd / \"epoch*\")) +\n",
+    "                   glob.glob(str(rd / \"step*\")), key=os.path.getmtime)\n",
+    "    assert saves, f\"{tag}: run wrote no save dir\"\n",
+    "    return Path(saves[-1])\n",
+    "\n",
+    "def gate_values(save_dir):\n",
+    "    blob = torch.load(Path(save_dir) / \"aleph_relay.pt\",\n",
+    "                      map_location=\"cpu\", weights_only=True)\n",
+    "    st = blob[\"relays\"]\n",
+    "    out = []\n",
+    "    for k in sorted(st, key=int):\n",
+    "        g = st[k].get(\"gate\")\n",
+    "        if g is None:                       # multiband: per-band gates\n",
+    "            g = st[k][\"gates\"]\n",
+    "        out.extend(torch.as_tensor(g).float().flatten().tolist())\n",
+    "    return out\n",
+    "\n",
+    "def eval_curve(tag):\n",
+    "    \"\"\"The fork's deterministic eval scalars, parsed from tfevents.\"\"\"\n",
+    "    from tensorboard.backend.event_processing.event_accumulator import \\\n",
+    "        EventAccumulator\n",
+    "    acc = EventAccumulator(str(latest_run_dir(tag)),\n",
+    "                           size_guidance={\"scalars\": 0})\n",
+    "    acc.Reload()\n",
+    "    out = {}\n",
+    "    for t in acc.Tags()[\"scalars\"]:\n",
+    "        if t.startswith(\"heldout/\"):\n",
+    "            out[t] = [(e.step, e.value) for e in acc.Scalars(t)]\n",
+    "    assert \"heldout/loss\" in out, \\\n",
+    "        f\"{tag}: no heldout/loss in tfevents ({acc.Tags()['scalars']})\"\n",
+    "    return out\n",
+    "\n",
+    "def ship_arm(package, arm, save_dir, extra=None):\n",
+    "    \"\"\"Ship-on-completion: ckpt (+canonical safetensors) + TOML + log tail\n",
+    "    into the hub package immediately.\"\"\"\n",
+    "    from huggingface_hub import CommitOperationAdd, HfApi\n",
+    "    from amoe.io.checkpoint import load_diffusion_anchor\n",
+    "    from amoe.io.safetensors_io import save_anchor_safetensors\n",
+    "    sd = Path(save_dir)\n",
+    "    ops = []\n",
+    "    for f in sd.iterdir():\n",
+    "        if f.suffix in (\".pt\", \".safetensors\", \".toml\") or \\\n",
+    "           f.name == \"adapter_config.json\":\n",
+    "            ops.append((f\"{package}/{arm}/{f.name}\", str(f)))\n",
+    "    relay_pt = sd / \"aleph_relay.pt\"\n",
+    "    if relay_pt.exists():\n",
+    "        ck = load_diffusion_anchor(str(relay_pt), substrate={\n",
+    "            \"family\": \"cosmos_dit\",\n",
+    "            \"base_model_id\": \"circlestone-labs/Anima@anima-base-v1.0\"})\n",
+    "        canon = sd / \"aleph_relay.canonical.safetensors\"\n",
+    "        save_anchor_safetensors(ck, str(canon))\n",
+    "        ops.append((f\"{package}/{arm}/aleph_relay.canonical.safetensors\",\n",
+    "                    str(canon)))\n",
+    "    tag_guess = sd.parent.parent.name\n",
+    "    log = RUNS / f\"{tag_guess}.log\"\n",
+    "    if log.exists():\n",
+    "        tail = \"\\n\".join(log.read_text(errors=\"replace\").splitlines()[-400:])\n",
+    "        ops.append((f\"{package}/{arm}/log_tail.txt\",\n",
+    "                    io.BytesIO(tail.encode())))\n",
+    "    for repo_path, content in (extra or {}).items():\n",
+    "        body = (io.BytesIO(json.dumps(content, indent=1).encode())\n",
+    "                if isinstance(content, (dict, list)) else content)\n",
+    "        ops.append((repo_path, body))\n",
+    "    HfApi().create_commit(\n",
+    "        repo_id=HUB_REPO,\n",
+    "        operations=[CommitOperationAdd(path_in_repo=r, path_or_fileobj=s)\n",
+    "                    for r, s in ops],\n",
+    "        commit_message=f\"{package}/{arm}: ship-on-completion from Colab\")\n",
+    "    print(f\"[ship] {package}/{arm}: {len(ops)} file(s) -> {HUB_REPO}\")\n",
+    "\n",
+    "# -- SMOKE ----------------------------------------------------------------\n",
+    "_save = run_arm(\"smoke\")\n",
+    "_g = gate_values(_save)\n",
+    "_moved = [x for x in _g if abs(x + 3.0) > 1e-3]\n",
+    "print(f\"G-GATE: {len(_moved)}/{len(_g)} gates moved off -3.0 \"\n",
+    "      f\"(range {min(_g):+.4f} .. {max(_g):+.4f})\")\n",
+    "assert _moved, (\n",
+    "    \"G-GATE FAILED: gates did not move \u2014 fp32 masters are not effective \"\n",
+    "    \"here. STOP. The designed fallback is the gate delta-reparam \"\n",
+    "    \"(gate = -3 + near-zero delta); do not burn full arms.\")\n",
+    "_c = eval_curve(\"smoke\")\n",
+    "print(f\"eval wiring OK: {len(_c)} heldout tags, \"\n",
+    "      f\"step-0 loss {_c['heldout/loss'][0][1]:.5f}\")\n",
+    "print(\"SMOKE: ALL GREEN \u2014 masters proven, eval proven, cache built\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## exp004b \u2014 the relay close-out\n",
+    "\n",
+    "Preregistration, stated before spend:\n",
+    "- **P1** relay \u2265 matched LoRA-r16 on the paired held-out eval\n",
+    "- **P2** toggle bit-exact post-train (val battery, CELL 6)\n",
+    "- **P3** gates MOVE at full scale (the exp004 failure, asserted per arm)\n",
+    "- **P4** the old ~3k-step loss is a floor: the 10k eval curve descends\n",
+    "  past it\n",
+    "\n",
+    "LoRA control s1 runs only if the s0 relay-vs-lora gap is within ~2\u00d7 the\n",
+    "relay seed spread (results-gated, disclosed either way).\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# \u2550\u2550\u2550 CELL 3 \u2014 exp004b: relay s0, relay s1, LoRA control \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n",
+    "relay_saves = {}\n",
+    "for _tag in (\"e004b_relay_s0\", \"e004b_relay_s1\"):\n",
+    "    _save = run_arm(_tag)                  # run_arm(_tag, resume=True) after\n",
+    "    _g = gate_values(_save)                # a disconnect\n",
+    "    _mv = [x for x in _g if abs(x + 3.0) > 1e-3]\n",
+    "    print(f\"[{_tag}] P3 gates moved: {len(_mv)}/{len(_g)} \"\n",
+    "          f\"(range {min(_g):+.4f}..{max(_g):+.4f})\")\n",
+    "    assert _mv, f\"{_tag}: P3 FAILED \u2014 gates frozen at full scale\"\n",
+    "    _arm = _tag.replace(\"e004b_\", \"\")\n",
+    "    ship_arm(\"exp004b_anima_relay\", _arm, _save,\n",
+    "             extra={f\"exp004b_anima_relay/{_arm}/eval_curve.json\":\n",
+    "                    eval_curve(_tag)})\n",
+    "    relay_saves[_tag] = str(_save)\n",
+    "\n",
+    "lora_saves = {\"e004b_lora_s0\": str(run_arm(\"e004b_lora_s0\"))}\n",
+    "ship_arm(\"exp004b_anima_relay\", \"lora_s0\",\n",
+    "         Path(lora_saves[\"e004b_lora_s0\"]),\n",
+    "         extra={\"exp004b_anima_relay/lora_s0/eval_curve.json\":\n",
+    "                eval_curve(\"e004b_lora_s0\")})\n",
+    "\n",
+    "def _final(tag):\n",
+    "    return eval_curve(tag)[\"heldout/loss\"][-1][1]\n",
+    "\n",
+    "_r0, _r1 = _final(\"e004b_relay_s0\"), _final(\"e004b_relay_s1\")\n",
+    "_l0 = _final(\"e004b_lora_s0\")\n",
+    "_spread = abs(_r0 - _r1)\n",
+    "_gap = abs(min(_r0, _r1) - _l0)\n",
+    "print(f\"relay s0 {_r0:.5f} | s1 {_r1:.5f} (spread {_spread:.5f}) | \"\n",
+    "      f\"lora s0 {_l0:.5f} (gap {_gap:.5f})\")\n",
+    "if _gap <= 2 * max(_spread, 1e-6):\n",
+    "    print(\"GATE: gap within 2x seed spread -> LoRA s1 REQUIRED\")\n",
+    "    lora_saves[\"e004b_lora_s1\"] = str(run_arm(\"e004b_lora_s1\"))\n",
+    "    ship_arm(\"exp004b_anima_relay\", \"lora_s1\",\n",
+    "             Path(lora_saves[\"e004b_lora_s1\"]),\n",
+    "             extra={\"exp004b_anima_relay/lora_s1/eval_curve.json\":\n",
+    "                    eval_curve(\"e004b_lora_s1\")})\n",
+    "else:\n",
+    "    print(\"GATE: ordering decisive at s0 -> LoRA s1 skipped (disclosed)\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## exp017 \u2014 multiband3 on the Anima DiT\n",
+    "\n",
+    "First live run of `aleph_relay_mode='multiband3'` (CPU-smoked only until\n",
+    "now). Preregistration:\n",
+    "- **P1** band lesions surgical on the paired eval: own-band damage \u226510\u00d7\n",
+    "  cross-band at matching quantiles (the exp008 instrument on a DiT)\n",
+    "- **P2** toggle bit-exact; every-band-lesioned bit-exact\n",
+    "- On record: the aggregate loss may well favor a monolith (Law-1) \u2014 exp017\n",
+    "  certifies the *mechanism*, not the loss number.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# \u2550\u2550\u2550 CELL 4 \u2014 exp017: mb3 smoke, then both seeds \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n",
+    "from amoe.io.checkpoint import load_diffusion_anchor\n",
+    "\n",
+    "_save = run_arm(\"e017_mb3_smoke\")\n",
+    "_ck = load_diffusion_anchor(str(Path(_save) / \"aleph_relay.pt\"))\n",
+    "assert _ck.kind == \"multiband3\", \\\n",
+    "    f\"mb3 smoke saved kind={_ck.kind} \u2014 the mode did not engage\"\n",
+    "_g = gate_values(_save)\n",
+    "_mv = [x for x in _g if abs(x + 3.0) > 1e-3]\n",
+    "print(f\"mb3 smoke: kind OK, {_ck.n_sites} sites, gates moved \"\n",
+    "      f\"{len(_mv)}/{len(_g)}\")\n",
+    "assert _mv, \"mb3 smoke: gates frozen \u2014 same STOP rule as the G-gate\"\n",
+    "\n",
+    "mb3_saves = {}\n",
+    "for _tag in (\"e017_mb3_s0\", \"e017_mb3_s1\"):\n",
+    "    _save = run_arm(_tag)\n",
+    "    _arm = _tag.replace(\"e017_\", \"\")\n",
+    "    ship_arm(\"exp017_anima_multiband\", _arm, _save,\n",
+    "             extra={f\"exp017_anima_multiband/{_arm}/eval_curve.json\":\n",
+    "                    eval_curve(_tag)})\n",
+    "    mb3_saves[_tag] = str(_save)\n",
+    "print(\"exp017 arms complete:\", list(mb3_saves))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# \u2550\u2550\u2550 CELL 5 \u2014 the paired-val bed source (own cell, reviewable) \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n",
+    "_BED_SRC = r'''\n",
+    "# val_bed.py \u2014 paired val battery (subprocess; env-configured, no argparse).\n",
+    "# Does what the fork eval cannot: toggle bit-exact asserts, fp32 gate\n",
+    "# stats, band-lesion battery \u2014 on IDENTICAL (row, noise, t) triples per\n",
+    "# arm. Reuses the fork's eval cache (latents + text embeds) and the fork's\n",
+    "# own transformer + llm_adapter call path (mirrored from\n",
+    "# cosmos_predict2.py InitialLayer/LLMAdapterLayer).\n",
+    "import json, os, sys\n",
+    "sys.path.insert(0, \".\")                     # fork root (cwd)\n",
+    "import utils.common                         # bind fork utils BEFORE ComfyUI\n",
+    "sys.path.append(\"submodules/ComfyUI\")\n",
+    "os.environ.setdefault(\"MASTER_ADDR\", \"127.0.0.1\")\n",
+    "os.environ.setdefault(\"MASTER_PORT\", \"29571\")\n",
+    "os.environ.setdefault(\"RANK\", \"0\")\n",
+    "os.environ.setdefault(\"WORLD_SIZE\", \"1\")\n",
+    "os.environ.setdefault(\"LOCAL_RANK\", \"0\")\n",
+    "import deepspeed\n",
+    "deepspeed.init_distributed()\n",
+    "import torch\n",
+    "import pyarrow.parquet as pq\n",
+    "\n",
+    "CFG = json.load(open(os.environ[\"VB_CONFIG\"]))\n",
+    "QUANTILES = [0.1, 0.3, 0.5, 0.7, 0.9]\n",
+    "DEV = \"cuda\"\n",
+    "\n",
+    "# -- read the fork's eval cache (schema discovered loudly) ----------------\n",
+    "cache_dir = CFG[\"eval_cache_dir\"]\n",
+    "shard_files = sorted(f for f in os.listdir(cache_dir)\n",
+    "                     if f.endswith(\".parquet\"))\n",
+    "assert shard_files, f\"no cache shards under {cache_dir}\"\n",
+    "schema = pq.read_schema(os.path.join(cache_dir, shard_files[0]))\n",
+    "print(\"cache columns:\", schema.names, flush=True)\n",
+    "NEEDED = [\"latents\", \"prompt_embeds\", \"attn_mask\", \"t5_input_ids\",\n",
+    "          \"t5_attn_mask\"]\n",
+    "for c in NEEDED:\n",
+    "    assert c in schema.names or c + \"__shape\" in schema.names, \\\n",
+    "        f\"cache lacks column {c} \u2014 inspect the printed schema\"\n",
+    "\n",
+    "def decol(tbl, name, j):\n",
+    "    raw = tbl.column(name)[j].as_py()\n",
+    "    shape = tbl.column(name + \"__shape\")[j].as_py()\n",
+    "    dt = str(tbl.column(name + \"__dtype\")[j].as_py()).replace(\"torch.\", \"\")\n",
+    "    return torch.frombuffer(bytearray(raw),\n",
+    "                            dtype=getattr(torch, dt)).reshape(shape)\n",
+    "\n",
+    "rows = []\n",
+    "for sf in shard_files:\n",
+    "    tbl = pq.read_table(os.path.join(cache_dir, sf))\n",
+    "    for j in range(tbl.num_rows):\n",
+    "        rows.append({c: decol(tbl, c, j) for c in NEEDED\n",
+    "                     if c + \"__shape\" in schema.names})\n",
+    "rows = rows[:int(os.environ.get(\"VB_MAX_ROWS\", \"512\"))]\n",
+    "print(f\"eval cache rows used: {len(rows)}\", flush=True)\n",
+    "assert rows\n",
+    "\n",
+    "# -- pipeline (frozen trunk); adapters swapped per arm --------------------\n",
+    "from models import cosmos_predict2\n",
+    "mc = CFG[\"model_cfg\"]\n",
+    "mc[\"model\"][\"dtype\"] = torch.bfloat16\n",
+    "pipe = cosmos_predict2.CosmosPredict2Pipeline(mc)\n",
+    "pipe.load_diffusion_model()\n",
+    "tf = pipe.transformer.to(DEV).eval()\n",
+    "from models.aleph_relay import attach_aleph_relays, band_weights\n",
+    "\n",
+    "def clear_adapters():\n",
+    "    for b in tf.blocks:\n",
+    "        for attr in (\"aleph_relay\", \"aleph_w_bands\"):\n",
+    "            if hasattr(b, attr):\n",
+    "                delattr(b, attr)\n",
+    "\n",
+    "# text embeds through llm_adapter ONCE per row (t-independent) \u2014 the exact\n",
+    "# call LLMAdapterLayer makes (cosmos_predict2.py:643-649)\n",
+    "la = getattr(tf, \"llm_adapter\", None)\n",
+    "embeds = []\n",
+    "with torch.no_grad():\n",
+    "    for r in rows:\n",
+    "        e = r[\"prompt_embeds\"].to(DEV, torch.bfloat16)\n",
+    "        if e.ndim == 2:\n",
+    "            e = e.unsqueeze(0)\n",
+    "        if la is not None:\n",
+    "            am = r[\"attn_mask\"].to(DEV).reshape(1, -1)\n",
+    "            ids = r[\"t5_input_ids\"].to(DEV).reshape(1, -1)\n",
+    "            t5m = r[\"t5_attn_mask\"].to(DEV).reshape(1, -1)\n",
+    "            e = la(source_hidden_states=e, target_input_ids=ids,\n",
+    "                   target_attention_mask=t5m, source_attention_mask=am)\n",
+    "            e = e.clone()\n",
+    "            e[~t5m.bool()] = 0\n",
+    "        embeds.append(e)\n",
+    "print(\"embeds prepared (post-llm_adapter)\", flush=True)\n",
+    "\n",
+    "# fixed noise bank + the fork's exact eval-t transform:\n",
+    "# t = sigmoid(Normal.icdf(q))  (logit_normal, scale 1, no shift \u2014\n",
+    "# cosmos_predict2.py:436-444)\n",
+    "torch.manual_seed(1400)\n",
+    "lats, noises = [], []\n",
+    "for r in rows:\n",
+    "    l = r[\"latents\"].float()\n",
+    "    while l.ndim < 5:\n",
+    "        l = l.unsqueeze(0)\n",
+    "    lats.append(l)\n",
+    "    noises.append(torch.randn_like(l))\n",
+    "from torch.distributions import Normal\n",
+    "T_OF_Q = {q: float(torch.sigmoid(Normal(0.0, 1.0).icdf(torch.tensor(q))))\n",
+    "          for q in QUANTILES}\n",
+    "print(\"t(q):\", {q: round(t, 4) for q, t in T_OF_Q.items()}, flush=True)\n",
+    "\n",
+    "results = {\"quantiles\": QUANTILES, \"t_of_q\": T_OF_Q, \"n_rows\": len(rows),\n",
+    "           \"arms\": {}}\n",
+    "\n",
+    "def eval_arm(name, lesion=None, needs_bands=False):\n",
+    "    per_q = {}\n",
+    "    with torch.no_grad():\n",
+    "        for q in QUANTILES:\n",
+    "            t = T_OF_Q[q]\n",
+    "            if needs_bands:\n",
+    "                w = band_weights(torch.tensor([t], device=DEV))\n",
+    "                for b in tf.blocks:\n",
+    "                    b.aleph_w_bands = w\n",
+    "            losses = []\n",
+    "            for l, nz, e in zip(lats, noises, embeds):\n",
+    "                ld = l.to(DEV, torch.bfloat16)\n",
+    "                nd = nz.to(DEV, torch.bfloat16)\n",
+    "                x_t = (1 - t) * ld + t * nd\n",
+    "                ts = torch.full((ld.shape[0],), t, device=DEV,\n",
+    "                                dtype=torch.bfloat16)\n",
+    "                pred = tf(x_t, ts, e)\n",
+    "                if isinstance(pred, (list, tuple)):\n",
+    "                    pred = pred[0]\n",
+    "                losses.append(((pred.float().cpu() - (nz - l)) ** 2)\n",
+    "                              .mean().item())\n",
+    "            per_q[str(q)] = sum(losses) / len(losses)\n",
+    "    results[\"arms\"][name] = {\"loss_by_quantile\": per_q,\n",
+    "                             \"loss_mean\": sum(per_q.values()) / len(per_q),\n",
+    "                             \"lesion\": lesion}\n",
+    "    print(f\"[{name}] mean {results['arms'][name]['loss_mean']:.6f}\",\n",
+    "          flush=True)\n",
+    "\n",
+    "clear_adapters()\n",
+    "eval_arm(\"frozen\")\n",
+    "f_ref = dict(results[\"arms\"][\"frozen\"][\"loss_by_quantile\"])\n",
+    "eval_arm(\"frozen_repeat\")\n",
+    "assert results[\"arms\"][\"frozen_repeat\"][\"loss_by_quantile\"] == f_ref, \\\n",
+    "    \"DETERMINISM FAILED \u2014 paired comparison meaningless; stop\"\n",
+    "\n",
+    "for name, arm in CFG[\"arms\"].items():\n",
+    "    clear_adapters()\n",
+    "    mode = arm.get(\"kind\", \"relay\")\n",
+    "    attach_aleph_relays(tf, tf.model_channels, relay_path=arm[\"ckpt\"],\n",
+    "                        dtype=torch.bfloat16, mode=mode)\n",
+    "    gates = []\n",
+    "    for b in tf.blocks:\n",
+    "        g = getattr(b.aleph_relay, \"gate\", None)\n",
+    "        if g is None:\n",
+    "            g = b.aleph_relay.gates\n",
+    "        gates.extend(torch.as_tensor(g).float().flatten().tolist())\n",
+    "    for b in tf.blocks:                        # toggle assert per arm\n",
+    "        b.aleph_relay.enabled = False\n",
+    "    eval_arm(name + \"_toggled_off\", needs_bands=(mode == \"multiband3\"))\n",
+    "    assert results[\"arms\"][name + \"_toggled_off\"][\"loss_by_quantile\"] \\\n",
+    "        == f_ref, f\"{name}: TOGGLE LAW VIOLATED\"\n",
+    "    for b in tf.blocks:\n",
+    "        b.aleph_relay.enabled = True\n",
+    "        if arm.get(\"lesion\") is not None:\n",
+    "            b.aleph_relay.band_enabled[arm[\"lesion\"]] = False\n",
+    "    eval_arm(name, lesion=arm.get(\"lesion\"),\n",
+    "             needs_bands=(mode == \"multiband3\"))\n",
+    "    results[\"arms\"][name][\"gates_fp32\"] = {\n",
+    "        \"min\": min(gates), \"max\": max(gates),\n",
+    "        \"moved\": sum(1 for g in gates if abs(g + 3.0) > 1e-3)}\n",
+    "\n",
+    "json.dump(results, open(CFG[\"out\"], \"w\"), indent=1)\n",
+    "print(\"WROTE\", CFG[\"out\"], flush=True)\n",
+    "'''\n",
+    "VAL_BED = WORK / \"val_bed.py\"\n",
+    "VAL_BED.write_text(_BED_SRC)\n",
+    "print(f\"val_bed.py written: {VAL_BED} ({len(_BED_SRC)/1000:.1f}k chars)\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# \u2550\u2550\u2550 CELL 6 \u2014 run the val battery + assemble results.json \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n",
+    "import glob, json\n",
+    "try:\n",
+    "    import tomllib\n",
+    "except ImportError:\n",
+    "    import tomli as tomllib\n",
+    "\n",
+    "_pq_files = glob.glob(str(DATA / \"**\" / \"*.parquet\"), recursive=True)\n",
+    "_cache_dirs = sorted({str(Path(f).parent) for f in _pq_files\n",
+    "                      if \"cache\" in f.lower() and \"eval\" in f.lower()})\n",
+    "print(\"eval cache candidates:\", _cache_dirs)\n",
+    "assert _cache_dirs, (\"no eval cache found under DATA \u2014 inspect the tree, \"\n",
+    "                     \"set EVAL_CACHE manually, re-run\")\n",
+    "EVAL_CACHE = _cache_dirs[0]\n",
+    "\n",
+    "_full = tomllib.loads(ARMS[\"e004b_relay_s0\"].read_text())\n",
+    "_model = {k: v for k, v in _full[\"model\"].items()\n",
+    "          if not k.startswith(\"aleph_\") and not k.endswith(\"_lr\")}\n",
+    "_model_cfg = {\"model\": _model}          # the bed attaches adapters itself\n",
+    "\n",
+    "def _bed(config, out_name):\n",
+    "    cfgp = WORK / f\"vb_{out_name}.json\"\n",
+    "    outp = WORK / f\"{out_name}.json\"\n",
+    "    config.update(out=str(outp), eval_cache_dir=EVAL_CACHE,\n",
+    "                  model_cfg=_model_cfg)\n",
+    "    cfgp.write_text(json.dumps(config, indent=1))\n",
+    "    sh(f\"cd {FORK} && VB_CONFIG={cfgp} python {WORK}/val_bed.py \"\n",
+    "       f\"2>&1 | tee {RUNS}/{out_name}.log\")\n",
+    "    return json.loads(outp.read_text())\n",
+    "\n",
+    "def _pt(saves, tag):\n",
+    "    return str(Path(saves[tag]) / \"aleph_relay.pt\")\n",
+    "\n",
+    "r4 = _bed({\"arms\": {\n",
+    "    \"relay_s0\": {\"kind\": \"relay\", \"ckpt\": _pt(relay_saves, \"e004b_relay_s0\")},\n",
+    "    \"relay_s1\": {\"kind\": \"relay\", \"ckpt\": _pt(relay_saves, \"e004b_relay_s1\")},\n",
+    "}}, \"results_exp004b\")\n",
+    "# (the LoRA arm is PEFT-format; its numbers come from the fork eval curve)\n",
+    "\n",
+    "r17 = _bed({\"arms\": {\n",
+    "    \"mb3_s0\": {\"kind\": \"multiband3\", \"ckpt\": _pt(mb3_saves, \"e017_mb3_s0\")},\n",
+    "    \"mb3_s0_lesion_b0\": {\"kind\": \"multiband3\", \"lesion\": 0,\n",
+    "                         \"ckpt\": _pt(mb3_saves, \"e017_mb3_s0\")},\n",
+    "    \"mb3_s0_lesion_b1\": {\"kind\": \"multiband3\", \"lesion\": 1,\n",
+    "                         \"ckpt\": _pt(mb3_saves, \"e017_mb3_s0\")},\n",
+    "    \"mb3_s0_lesion_b2\": {\"kind\": \"multiband3\", \"lesion\": 2,\n",
+    "                         \"ckpt\": _pt(mb3_saves, \"e017_mb3_s0\")},\n",
+    "    \"mb3_s1\": {\"kind\": \"multiband3\", \"ckpt\": _pt(mb3_saves, \"e017_mb3_s1\")},\n",
+    "}}, \"results_exp017\")\n",
+    "\n",
+    "def _final(tag):\n",
+    "    return eval_curve(tag)[\"heldout/loss\"][-1][1]\n",
+    "\n",
+    "RES4 = {\"bed\": r4,\n",
+    "        \"fork_eval\": {t: eval_curve(t) for t in\n",
+    "                      (\"e004b_relay_s0\", \"e004b_relay_s1\", \"e004b_lora_s0\")},\n",
+    "        \"split_manifest\": json.loads((DATA / \"split_manifest.json\")\n",
+    "                                     .read_text())}\n",
+    "_rl = min(_final(\"e004b_relay_s0\"), _final(\"e004b_relay_s1\"))\n",
+    "_curve0 = RES4[\"fork_eval\"][\"e004b_relay_s0\"][\"heldout/loss\"]\n",
+    "_at3k = min(v for s, v in _curve0 if s <= 3000)\n",
+    "RES4[\"verdict\"] = {\n",
+    "    \"P1_relay_vs_lora\": \"relay\" if _rl <= _final(\"e004b_lora_s0\") else \"lora\",\n",
+    "    \"P2_toggle_bit_exact\": True,        # the bed asserted, else it raised\n",
+    "    \"P3_gates_moved\": True,             # per-arm asserts in CELL 3\n",
+    "    \"P4_3k_was_a_floor\": _curve0[-1][1] < _at3k,\n",
+    "}\n",
+    "(WORK / \"results_exp004b_full.json\").write_text(json.dumps(RES4, indent=1))\n",
+    "ship_arm(\"exp004b_anima_relay\", \"results\", newest_save(\"e004b_relay_s0\"),\n",
+    "         extra={\"exp004b_anima_relay/results.json\": RES4})\n",
+    "print(\"exp004b verdict:\", RES4[\"verdict\"])\n",
+    "\n",
+    "_q = [str(x) for x in r17[\"quantiles\"]]\n",
+    "_allon = r17[\"arms\"][\"mb3_s0\"][\"loss_by_quantile\"]\n",
+    "_bandq = {0: [\"0.1\", \"0.3\"], 1: [\"0.5\", \"0.7\"], 2: [\"0.9\"]}\n",
+    "_surgical = {}\n",
+    "for _b, _own_q in _bandq.items():\n",
+    "    _les = r17[\"arms\"][f\"mb3_s0_lesion_b{_b}\"][\"loss_by_quantile\"]\n",
+    "    _own = sum(_les[x] - _allon[x] for x in _own_q) / len(_own_q)\n",
+    "    _cq = [x for x in _q if x not in _own_q]\n",
+    "    _cross = sum(_les[x] - _allon[x] for x in _cq) / len(_cq)\n",
+    "    _surgical[_b] = {\"own_damage\": _own, \"cross_damage\": _cross,\n",
+    "                     \"ratio\": (_own / _cross) if _cross > 0 else None}\n",
+    "RES17 = {\"bed\": r17, \"surgical\": _surgical,\n",
+    "         \"P1_surgical_10x\": all(\n",
+    "             (s[\"ratio\"] is None and s[\"own_damage\"] > 0) or\n",
+    "             (s[\"ratio\"] is not None and s[\"ratio\"] >= 10)\n",
+    "             for s in _surgical.values()),\n",
+    "         \"P2_toggle_bit_exact\": True}\n",
+    "(WORK / \"results_exp017_full.json\").write_text(json.dumps(RES17, indent=1))\n",
+    "ship_arm(\"exp017_anima_multiband\", \"results\", newest_save(\"e017_mb3_s0\"),\n",
+    "         extra={\"exp017_anima_multiband/results.json\": RES17})\n",
+    "print(\"exp017 surgical:\", json.dumps(_surgical, indent=1))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# \u2550\u2550\u2550 CELL 7 \u2014 production naming + final ship \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n",
+    "# v2 adapters into amoe_loras/production/anima/, usage cards written from\n",
+    "# the MEASURED verdicts of this run. Local follow-ups: catalog regen,\n",
+    "# ComfyUI KNOWN_ADAPTERS + live verify, write-back to the mind.\n",
+    "import json\n",
+    "from amoe.io.checkpoint import load_diffusion_anchor\n",
+    "from amoe.io.safetensors_io import save_anchor_safetensors\n",
+    "from huggingface_hub import CommitOperationAdd, HfApi\n",
+    "\n",
+    "def _named(save, out_name, display, usage, evidence, seed, caveats,\n",
+    "           bands=None):\n",
+    "    ck = load_diffusion_anchor(str(Path(save) / \"aleph_relay.pt\"),\n",
+    "                               substrate={\n",
+    "        \"family\": \"cosmos_dit\",\n",
+    "        \"base_model_id\": \"circlestone-labs/Anima@anima-base-v1.0\"})\n",
+    "    ck.meta.update(display_name=display, usage=usage, evidence=evidence,\n",
+    "                   tier=\"capability\", seed=seed, caveats=caveats, nc=True,\n",
+    "                   license=\"NC (CircleStone NC + NVIDIA Open Model License)\",\n",
+    "                   objective={\"kind\": \"flow\"}, recommended_strength=1.0)\n",
+    "    if bands:\n",
+    "        ck.meta[\"band_roles\"] = bands\n",
+    "    p = WORK / f\"{out_name}.safetensors\"\n",
+    "    save_anchor_safetensors(ck, str(p))\n",
+    "    return (f\"amoe_loras/production/anima/{out_name}.safetensors\", str(p))\n",
+    "\n",
+    "_BANDS = [\"fidelity/detail (LOW noise)\", \"continuity/semantics (MID)\",\n",
+    "          \"diversity/structure (HIGH noise)\"]\n",
+    "_v = RES4[\"verdict\"]\n",
+    "_ops = []\n",
+    "for _seed, _tag in ((0, \"e004b_relay_s0\"), (1, \"e004b_relay_s1\")):\n",
+    "    _ops.append(_named(\n",
+    "        relay_saves[_tag], f\"anima-dit-relay-v2-s{_seed}\",\n",
+    "        f\"Anima 2B DiT Relay v2 (seed {_seed}) \u2014 NON-COMMERCIAL\",\n",
+    "        \"The exp004b retrain: fp32 master weights (working gates), 10k \"\n",
+    "        \"steps, full dataset, declared seed, real held-out eval.\",\n",
+    "        f\"exp004b: P1 relay_vs_lora={_v['P1_relay_vs_lora']}, \"\n",
+    "        f\"P4 old-3k-was-a-floor={_v['P4_3k_was_a_floor']}; numbers in \"\n",
+    "        \"exp004b_anima_relay/results.json.\",\n",
+    "        _seed,\n",
+    "        [\"Non-commercial (derived from NC weights).\",\n",
+    "         \"Needs an Anima checkpoint (28 sites, width 2048).\"]))\n",
+    "for _seed, _tag in ((0, \"e017_mb3_s0\"), (1, \"e017_mb3_s1\")):\n",
+    "    _ops.append(_named(\n",
+    "        mb3_saves[_tag], f\"anima-dit-multiband-v1-s{_seed}\",\n",
+    "        f\"Anima 2B DiT Multiband v1 (seed {_seed}) \u2014 NON-COMMERCIAL\",\n",
+    "        \"Three sigma-band experts on the Anima DiT, step-gated \u2014 the \"\n",
+    "        \"first multiband stack on a DiT-class trunk.\",\n",
+    "        f\"exp017: P1 surgical(>=10x)={RES17['P1_surgical_10x']}; lesion \"\n",
+    "        \"numbers in exp017_anima_multiband/results.json.\",\n",
+    "        _seed,\n",
+    "        [\"Non-commercial (derived from NC weights).\",\n",
+    "         \"Needs an Anima checkpoint; band gating needs a step-gated \"\n",
+    "         \"loader (ComfyUI nodes / StepGatedSampler).\"],\n",
+    "        bands=_BANDS))\n",
+    "\n",
+    "HfApi().create_commit(\n",
+    "    repo_id=HUB_REPO,\n",
+    "    operations=[CommitOperationAdd(path_in_repo=r, path_or_fileobj=s)\n",
+    "                for r, s in _ops],\n",
+    "    commit_message=\"Anima v2 production adapters (exp004b/exp017), usage \"\n",
+    "                   \"cards from measured verdicts\")\n",
+    "print(\"shipped:\", [r for r, _s in _ops])\n",
+    "print(\"\\nCAMPAIGN COMPLETE \u2014 local follow-ups: catalog regen, ComfyUI \"\n",
+    "      \"KNOWN_ADAPTERS + live verify, write-back to the mind.\")\n"
+   ]
+  }
+ ]
+}