exp012 shipped: blob v1 honest negative BUT the role-aligned gauge reveals multiband beats monolith ~10% on HIGH-band foreground structure; ANIMA R0b all green (dtype violation caught+fixed)

exp012_blob_supervised/README.md

@@ -0,0 +1,35 @@
+# exp012_blob_supervised — blob supervision v1 + THE ROLE-ALIGNED GAUGE
+# DISCOVERY (CANDIDATE s0; s1 running) + ANIMA R0b CERTIFICATION
+
+**Blob supervision v1** (foreground-weighted LP-x0 on the HIGH band, λ=0.5):
+HONEST NEGATIVE on both instruments — training-side blob gauge moved +0.03%
+(0.111495 vs uniform 0.111528; P1 technical hit, economically nil); the
+image battery (P4) showed mild dilution (grounding +0.1931 vs +0.2135;
+HIGH-lesion LP-dominance 0.0662 vs 0.0727 — not sharpened). The eps term
+dominates at this dose; stronger coupling or pure-band phases are the
+follow-ups.
+
+**THE KEEPER — the role-aligned gauge (in-bed reference gauging):**
+
+| arm | HIGH-band foreground-LP-x0 gauge |
+|---|---|
+| frozen | 0.15549 |
+| monolith r48 | 0.12339 |
+| **uniform multiband** | **0.11153** |
+| **blob multiband** | **0.11150** |
+
+**The first gauge where multiband CLEARLY beats the matched monolith (~10%)**
+— band specialists are better at foreground coarse structure in the
+HIGH-noise band, exactly where the coarse-to-fine role map says they should
+be. Every aggregate-eps comparison had hidden this. Band assignment's payer
+is role-aligned MEASUREMENT — the specialists were already earning.
+
+**ANIMA R0b (bundled: `r0b_probe.py`): ALL GATES GREEN.** 28/28 DiT blocks
+carry relays (14,275,716 bf16 params), freeze-by-lr-0 exact, zero-init +
+toggle exact. The gate CAUGHT a dtype-law violation live (fresh relays
+attached fp32 on the bf16 trunk via a meta-tensor sniff; fixed to the
+DECLARED dtype, fork commit on feat/aleph-adapter). exp004 training is
+ready.
+
+**Caveats.** s0 (s1 running); one λ; one coupling form. Cost ≈ 1.6 GPU-h
++ 0.2h battery + R0b probes.

exp012_blob_supervised/dexp012_blob_supervised.py

@@ -0,0 +1,235 @@
+"""dexp012_blob_supervised.py — exp012: SEGMENTATION-BLOB SUPERVISION on the
+HIGH-noise band (Phil's blobbing role, first real qualitatively-different
+supervision; day-3 plan item 2 — design preregistered there).
+
+Coupling (conservative, controlled): foreground-weighted LOW-PASS x0 loss on
+the HIGH band only:
+    x0_hat = (x_t - sqrt(1-abar_t) * eps_hat) / sqrt(abar_t)
+    L_blob = mean( blob ⊙ (LP(x0_hat) - LP(x0))^2 ),  lambda = 0.5
+routed by the crossfade windows (w2 only). Differs from exp009's inert
+reweighting: x0-space structural prediction at high noise + EXTERNAL
+segmentation information (the rebuilt 100%-non-empty blob targets).
+
+In-bed gauges for ALL arms (blob-mb3 trained here; uniform-mb3 + mono48
+loaded from exp011 ckpts; frozen): common eps gauge per band + BLOB GAUGE
+(foreground LP-x0 error) per band. Prereg P1-P4 in the day-3 plan.
+
+Pod: bash pod2/run_exp012.sh   [DEXP12_SEED=0 DEXP12_LAMBDA=0.5]
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+
+sys.path[:0] = ["pod2", "."]
+
+import torch
+import torch.nn.functional as F
+
+from pod_ledger import ledger_run, note, burn_down
+from d1_substrate import MEM_FRACTION
+from dexp006_sd15core_relay import make_schedule, add_noise
+from dexp008_multiband import (MultibandDelta, MonoDelta, band_weights,
+                               band_of, attach, load_unet, N_BANDS)
+from dexp009_bandroles import lp
+
+N_TRAIN, N_VAL = 4096, 256
+BATCH = int(os.environ.get("DEXP12_BATCH", "16"))
+STEPS = int(os.environ.get("DEXP12_STEPS", "3000"))
+SEED = int(os.environ.get("DEXP12_SEED", "0"))
+LAM = float(os.environ.get("DEXP12_LAMBDA", "0.5"))
+LR, CFG_DROPOUT = 1e-3, 0.1
+D11 = ("/workspace/data/dexp011" if os.path.isdir("/workspace")
+       else "./data/dexp011")
+D11_CKPT = ("/workspace/ckpts2/dexp011" if os.path.isdir("/workspace")
+            else D11)
+DATA_DIR = ("/workspace/data/dexp012" if os.path.isdir("/workspace")
+            else os.path.join(os.environ.get("GEOLIP_DATA", "./data"),
+                              "dexp012"))
+CKPT_DIR = ("/workspace/ckpts2/dexp012" if os.path.isdir("/workspace")
+            else DATA_DIR)
+
+
+def x0_from_eps(noisy, eps, t, acp):
+    a = acp[t].sqrt()[:, None, None, None]
+    s = (1 - acp[t]).sqrt()[:, None, None, None]
+    return (noisy - s * eps) / a.clamp_min(1e-4)
+
+
+def blob_lp_err(x0_hat, x0, blob):
+    """Foreground-weighted LP-x0 squared error, per sample (B,)."""
+    d2 = (lp(x0_hat) - lp(x0)) ** 2                 # (B,4,64,64)
+    m = blob[:, None]                                # (B,1,64,64)
+    denom = m.sum(dim=(1, 2, 3)).clamp_min(1.0) * d2.shape[1]
+    return (d2 * m).sum(dim=(1, 2, 3)) / denom
+
+
+def run(device="cuda"):
+    torch.cuda.set_per_process_memory_fraction(MEM_FRACTION, 0)
+    os.makedirs(CKPT_DIR, exist_ok=True)
+    os.makedirs(DATA_DIR, exist_ok=True)
+    acp = make_schedule(device)
+    cache = torch.load(os.path.join(D11, "cache.pt"), map_location="cpu",
+                       weights_only=True)
+    assert int(cache["blob_empty_count"]) == 0, \
+        "blob targets not the rebuilt ones — run fix_blob_targets first"
+
+    def set_w(wraps, s01):
+        w = band_weights(s01)
+        for wr in wraps:
+            wr.w_bands = w
+        return w
+
+    def loss_of(unet, wraps, lat, ehs, blob, gen):
+        bsz = lat.shape[0]
+        drop = torch.rand(bsz, generator=gen, device=device) < CFG_DROPOUT
+        ehs = ehs.clone()
+        ehs[drop] = 0
+        t = torch.randint(0, 1000, (bsz,), generator=gen, device=device)
+        w = set_w(wraps, t.float() / 1000.0)
+        noise = torch.randn(lat.shape, generator=gen, device=device)
+        noisy = add_noise(lat, noise, t, acp)
+        pred = unet(noisy, t, ehs, return_dict=False)[0]
+        base = ((pred - noise) ** 2).mean(dim=(1, 2, 3))
+        x0h = x0_from_eps(noisy, pred, t, acp)
+        blob_term = blob_lp_err(x0h, lat, blob)
+        return (base + LAM * w[:, 2] * blob_term).mean()
+
+    @torch.no_grad()
+    def val(unet, wraps):
+        """Common + blob gauges per band, identical math for every arm."""
+        per_band = {0: [], 1: [], 2: []}
+        blob_band = {0: [], 1: [], 2: []}
+        tot = []
+        for i in range(0, N_VAL, 32):
+            lat = cache["val_lat"][i:i + 32].to(device)
+            ehs = cache["val_ehs"][i:i + 32].to(device)
+            noise = cache["val_noise"][i:i + 32].to(device)
+            t = cache["val_t"][i:i + 32].to(device)
+            blob = cache["val_blob"][i:i + 32].float().to(device)
+            set_w(wraps, t.float() / 1000.0)
+            noisy = add_noise(lat, noise, t, acp)
+            pred = unet(noisy, t, ehs, return_dict=False)[0]
+            mse = ((pred - noise) ** 2).mean(dim=(1, 2, 3))
+            bg = blob_lp_err(x0_from_eps(noisy, pred, t, acp), lat, blob)
+            tot += mse.tolist()
+            for j, tv in enumerate((t.float() / 1000.0).tolist()):
+                b = band_of(tv)
+                per_band[b].append(mse[j].item())
+                blob_band[b].append(bg[j].item())
+        return (sum(tot) / len(tot),
+                {f"band{b}": round(sum(v) / max(len(v), 1), 6)
+                 for b, v in per_band.items()},
+                {f"band{b}": round(sum(v) / max(len(v), 1), 6)
+                 for b, v in blob_band.items()})
+
+    results = {"config": {"lambda": LAM, "steps": STEPS, "batch": BATCH,
+                          "seed": SEED, "coupling":
+                          "foreground-weighted LP-x0, HIGH band only"}}
+
+    # references gauged IN-BED (the exp009 instrument lesson)
+    with ledger_run(f"dexp012 reference gauges s{SEED}", budget_h=0.4) as h:
+        unet = load_unet(device)
+        v, pb, bb = val(unet, [])
+        results["frozen"] = {"val": v, "per_band": pb, "blob_gauge": bb}
+        del unet
+        torch.cuda.empty_cache()
+        for label, mk, fn in (("uniform_mb3", lambda d: MultibandDelta(d),
+                               "multiband3_s0.pt"),
+                              ("monolith", lambda d: MonoDelta(d),
+                               "monolith_s0.pt")):
+            unet = load_unet(device)
+            mods, wraps = attach(unet, mk)
+            ck = torch.load(os.path.join(D11_CKPT, fn), map_location="cpu",
+                            weights_only=True)
+            for m, sd in zip(mods, ck["mods"]):
+                m.load_state_dict(sd, strict=True)
+            v, pb, bb = val(unet, wraps)
+            results[label] = {"val": v, "per_band": pb, "blob_gauge": bb}
+            del unet, mods
+            torch.cuda.empty_cache()
+        h["verdict"] = "refs gauged"
+
+    with ledger_run(f"dexp012 blob-mb3 s{SEED}", budget_h=2.5) as h:
+        unet = load_unet(device)
+        mods, wraps = attach(unet, lambda d: MultibandDelta(d))
+        for m in mods:
+            m.assert_zero_init()
+        opt = torch.optim.Adam(mods.parameters(), lr=LR, weight_decay=0.0)
+        gen = torch.Generator(device=device).manual_seed(SEED + 42)
+        idx = torch.Generator().manual_seed(SEED + 7)
+        t0 = time.time()
+        for step in range(1, STEPS + 1):
+            sel = torch.randint(0, N_TRAIN, (BATCH,), generator=idx)
+            loss = loss_of(unet, wraps, cache["lat"][sel].to(device),
+                           cache["ehs"][sel].to(device),
+                           cache["blob"][sel].float().to(device), gen)
+            loss.backward()
+            opt.step()
+            opt.zero_grad(set_to_none=True)
+            if step == 50 or step % 500 == 0:
+                print(f"[blob] step {step}: loss {loss.item():.4f} | "
+                      f"{(time.time() - t0) / step:.2f}s/step", flush=True)
+        v, pb, bb = val(unet, wraps)
+        for m in mods:
+            m.enabled = False
+        v_off, _, _ = val(unet, wraps)
+        d = abs(v_off - results["frozen"]["val"])
+        assert d < 1e-9, f"toggle parity broken: {d}"
+        for m in mods:
+            m.enabled = True
+        torch.save({"mods": [m.state_dict() for m in mods]},
+                   os.path.join(CKPT_DIR, f"blob_mb3_s{SEED}.pt"))
+        results["blob_mb3"] = {"val": v, "per_band": pb, "blob_gauge": bb,
+                               "val_toggled_off": v_off}
+        del unet, mods
+        torch.cuda.empty_cache()
+        h["verdict"] = f"val {v:.5f} blobH {bb['band2']}"
+
+    bm, um = results["blob_mb3"], results["uniform_mb3"]
+    results["verdict"] = {
+        "P1_blob_gauge_high_band":
+            {"blob_mb3": bm["blob_gauge"]["band2"],
+             "uniform_mb3": um["blob_gauge"]["band2"],
+             "hit": bm["blob_gauge"]["band2"] < um["blob_gauge"]["band2"]},
+        "P2_common_undegraded":
+            abs(bm["val"] - um["val"]) / um["val"] < 0.005
+            or bm["val"] < um["val"],
+        "common_val": {"blob": bm["val"], "uniform": um["val"],
+                       "mono": results["monolith"]["val"]},
+        "note": "P4 (image battery on the blob ckpt) runs via dexp010 env; "
+                "1-seed CANDIDATE",
+    }
+    with open(os.path.join(DATA_DIR, "results.json" if SEED == 0
+                           else f"results_s{SEED}.json"), "w") as f:
+        json.dump(results, f, indent=2)
+    note(f"dexp012: {json.dumps(results['verdict']['P1_blob_gauge_high_band'])}")
+    print(json.dumps(results["verdict"], indent=2))
+    burn_down()
+    return results
+
+
+def smoke():
+    acp = torch.linspace(0.9999, 0.05, 1000)
+    x = torch.randn(2, 4, 64, 64)
+    n = torch.randn_like(x)
+    t = torch.tensor([100, 900])
+    a = acp[t].sqrt()[:, None, None, None]
+    s = (1 - acp[t]).sqrt()[:, None, None, None]
+    noisy = a * x + s * n
+    x0h = x0_from_eps(noisy, n, t, acp)
+    assert (x0h - x).abs().max() < 1e-4, "x0 inversion wrong"
+    blob = torch.zeros(2, 64, 64)
+    blob[0, 10:30, 10:30] = 1
+    e = blob_lp_err(x0h, x, blob)
+    assert e.shape == (2,) and e[0] < 1e-6 and e[1] == 0, e
+    print("dexp012 smoke PASSED (x0 inversion exact + fg-weighted gauge)")
+
+
+if __name__ == "__main__":
+    if "--run" in sys.argv:
+        run()
+    else:
+        smoke()

exp012_blob_supervised/r0b_probe.py

@@ -0,0 +1,96 @@
+"""r0b_probe.py — ANIMA R0b: the fork-integration gate (runs INSIDE the pod
+clone of diffusion-pipe @ feat/aleph-adapter). Certifies, without a dataset:
+  G1 anima weights load through CosmosPredict2Pipeline (dit_config printed);
+  G2 aleph relays attach post-materialization (site count, param count);
+  G3 DTYPE LAW: relay dtype == block dtype (bf16);
+  G4 param groups: aleph_relay bucket present, trunk freezable by lr=0;
+  G5 zero-init: all-off/enabled-at-init forward delta == 0 through one block.
+Full exp004 training goes to the evening slot per the day-3 plan.
+Run: cd /workspace/geolip2/diffusion-pipe && python3 ../pod2/r0b_probe.py
+"""
+from __future__ import annotations
+
+import sys
+
+import torch
+
+sys.path.insert(0, ".")
+import utils.common  # bind the fork utils BEFORE ComfyUI (regular-pkg shadowing)
+sys.path.append("submodules/ComfyUI")
+
+CFG = {
+    "model": {
+        "type": "anima",
+        "transformer_path":
+            "/workspace/models/anima/split_files/diffusion_models/"
+            "anima-base-v1.0.safetensors",
+        "llm_path": "/workspace/models/anima/split_files/text_encoders/"
+                    "qwen_3_06b_base.safetensors",
+        "vae_path": "/workspace/models/anima/split_files/vae/"
+                    "qwen_image_vae.safetensors",
+        "dtype": torch.bfloat16,
+        "aleph_relay": True,
+        "aleph_relay_every": 1,
+        "aleph_relay_lr": 1e-3,
+        "self_attn_lr": 0, "cross_attn_lr": 0, "mlp_lr": 0, "mod_lr": 0,
+        "llm_adapter_lr": 0,
+    },
+    "optimizer": {"lr": 0},
+    "reentrant_activation_checkpointing": False,
+}
+
+
+def main():
+    import os
+    for k, v in (("MASTER_ADDR", "127.0.0.1"), ("MASTER_PORT", "29571"),
+                 ("RANK", "0"), ("WORLD_SIZE", "1"), ("LOCAL_RANK", "0")):
+        os.environ.setdefault(k, v)
+    import deepspeed
+    deepspeed.init_distributed()
+    from models.cosmos_predict2 import CosmosPredict2Pipeline
+    pipe = CosmosPredict2Pipeline(CFG)
+    print("[G1] pipeline constructed (text encoder loaded, name="
+          f"{pipe.name})", flush=True)
+    pipe.load_diffusion_model()
+    tr = pipe.transformer
+    relays = [getattr(b, "aleph_relay", None) for b in tr.blocks]
+    n_sites = sum(r is not None for r in relays)
+    n_params = sum(p.numel() for r in relays if r is not None
+                   for p in r.parameters())
+    print(f"[G2] {n_sites}/{len(tr.blocks)} blocks carry relays, "
+          f"{n_params:,} adapter params", flush=True)
+    assert n_sites == len(tr.blocks) > 0
+
+    r0 = next(r for r in relays if r is not None)
+    b0 = tr.blocks[0]
+    bdt = next(p for n, p in b0.named_parameters()
+               if "aleph_relay" not in n).dtype
+    rdt = next(r0.parameters()).dtype
+    print(f"[G3] block dtype {bdt} | relay dtype {rdt}", flush=True)
+    assert rdt == bdt == torch.bfloat16, "dtype law violated"
+
+    params = [p for p in tr.parameters()]
+    for p, (n, _) in zip(params, tr.named_parameters()):
+        pass
+    groups = pipe.get_param_groups(
+        [p for p in tr.parameters() if hasattr(p, "original_name")])
+    n_trainable = sum(p.numel() for g in groups for p in g["params"])
+    print(f"[G4] {len(groups)} param groups, trainable {n_trainable:,} "
+          f"(should == adapter count {n_params:,})", flush=True)
+    assert n_trainable == n_params, "freeze-by-lr-0 leaked trunk params"
+
+    r0.assert_zero_init()
+    x = torch.randn(1, 2, 4, 4, tr.model_channels, dtype=bdt)
+    with torch.no_grad():
+        out_on = r0(x)
+        r0.enabled = False
+        out_off = r0(x)
+        r0.enabled = True
+    assert torch.equal(out_on, x) and out_off is x, "zero-init/toggle broken"
+    print("[G5] zero-init + toggle exact on the DiT relay", flush=True)
+    print("R0B ALL GATES GREEN — fork integration certified; exp004 train "
+          "is evening-slot ready", flush=True)
+
+
+if __name__ == "__main__":
+    main()

exp012_blob_supervised/results.json

@@ -0,0 +1,76 @@
+{
+  "config": {
+    "lambda": 0.5,
+    "steps": 3000,
+    "batch": 16,
+    "seed": 0,
+    "coupling": "foreground-weighted LP-x0, HIGH band only"
+  },
+  "frozen": {
+    "val": 0.14243539798007987,
+    "per_band": {
+      "band0": 0.293931,
+      "band1": 0.071764,
+      "band2": 0.010422
+    },
+    "blob_gauge": {
+      "band0": 0.002656,
+      "band1": 0.021497,
+      "band2": 0.15549
+    }
+  },
+  "uniform_mb3": {
+    "val": 0.1352152533709159,
+    "per_band": {
+      "band0": 0.280633,
+      "band1": 0.066728,
+      "band2": 0.009562
+    },
+    "blob_gauge": {
+      "band0": 0.001986,
+      "band1": 0.015805,
+      "band2": 0.111528
+    }
+  },
+  "monolith": {
+    "val": 0.1341835820844608,
+    "per_band": {
+      "band0": 0.278893,
+      "band1": 0.065772,
+      "band2": 0.009564
+    },
+    "blob_gauge": {
+      "band0": 0.001943,
+      "band1": 0.015434,
+      "band2": 0.123388
+    }
+  },
+  "blob_mb3": {
+    "val": 0.13531021448216052,
+    "per_band": {
+      "band0": 0.280604,
+      "band1": 0.066751,
+      "band2": 0.009977
+    },
+    "blob_gauge": {
+      "band0": 0.001991,
+      "band1": 0.015801,
+      "band2": 0.111495
+    },
+    "val_toggled_off": 0.14243539798007987
+  },
+  "verdict": {
+    "P1_blob_gauge_high_band": {
+      "blob_mb3": 0.111495,
+      "uniform_mb3": 0.111528,
+      "hit": true
+    },
+    "P2_common_undegraded": true,
+    "common_val": {
+      "blob": 0.13531021448216052,
+      "uniform": 0.1352152533709159,
+      "mono": 0.1341835820844608
+    },
+    "note": "P4 (image battery on the blob ckpt) runs via dexp010 env; 1-seed CANDIDATE"
+  }
+}

exp012_blob_supervised/results_blob_mb3_s0.json

@@ -0,0 +1,75 @@
+{
+  "config": {
+    "steps": 30,
+    "guidance": 7.5,
+    "seed": 1234,
+    "ckpt": "blob_mb3_s0",
+    "sampler": "DDIM step-gated"
+  },
+  "all_on": {
+    "round_trip": {
+      "cond": 0.7236,
+      "shuffled": 0.5305,
+      "gap": 0.1931
+    },
+    "diversity": 0.24882
+  },
+  "lesion_band0": {
+    "round_trip": {
+      "cond": 0.7071,
+      "shuffled": 0.529,
+      "gap": 0.1781
+    },
+    "vs_all_on": {
+      "lp_change": 0.003908,
+      "hp_change": 0.003013,
+      "total_change": 0.011374
+    },
+    "diversity": 0.26582
+  },
+  "lesion_band1": {
+    "round_trip": {
+      "cond": 0.7046,
+      "shuffled": 0.5425,
+      "gap": 0.1621
+    },
+    "vs_all_on": {
+      "lp_change": 0.021867,
+      "hp_change": 0.004201,
+      "total_change": 0.03785
+    },
+    "diversity": 0.29886
+  },
+  "lesion_band2": {
+    "round_trip": {
+      "cond": 0.7024,
+      "shuffled": 0.5551,
+      "gap": 0.1474
+    },
+    "vs_all_on": {
+      "lp_change": 0.066186,
+      "hp_change": 0.00481,
+      "total_change": 0.086765
+    },
+    "diversity": 0.29093
+  },
+  "frozen": {
+    "round_trip": {
+      "cond": 0.6692,
+      "shuffled": 0.5446,
+      "gap": 0.1247
+    },
+    "vs_all_on": {
+      "lp_change": 0.074492,
+      "hp_change": 0.005201,
+      "total_change": 0.09682
+    },
+    "diversity": 0.33191
+  },
+  "prereg": {
+    "G2_high_band_moves_LP_most": true,
+    "G2_low_band_moves_HP_most": false,
+    "G3_high_lesion_cuts_diversity": false,
+    "note": "image-space coarse-to-fine test; 1-seed battery"
+  }
+}