0.2.0: amoe.diffusion subsystem (relay/multiband/StepGatedSampler, dtype law, align grounded-negative, conditioning law), safetensors I/O + amoe-convert, diffusion invariants; lineage corrected to the audited 19-package record
Browse files- README.md +101 -14
- docs/distributed.md +22 -1
- examples/attach_diffusion.py +32 -0
- pyproject.toml +6 -2
- src/amoe/__init__.py +5 -1
- src/amoe/binding/diffusion.py +197 -0
- src/amoe/diagnostics/diffusion.py +64 -0
- src/amoe/diffusion/__init__.py +29 -0
- src/amoe/diffusion/core/__init__.py +3 -0
- src/amoe/diffusion/core/cond.py +52 -0
- src/amoe/diffusion/core/multiband.py +109 -0
- src/amoe/diffusion/core/relay.py +90 -0
- src/amoe/diffusion/laws.py +53 -0
- src/amoe/diffusion/runtime/__init__.py +2 -0
- src/amoe/diffusion/runtime/attach.py +195 -0
- src/amoe/diffusion/runtime/sampler.py +120 -0
- src/amoe/diffusion/train/__init__.py +4 -0
- src/amoe/diffusion/train/aligner.py +26 -0
- src/amoe/diffusion/train/config.py +44 -0
- src/amoe/diffusion/train/data.py +97 -0
- src/amoe/diffusion/train/objectives.py +80 -0
- src/amoe/diffusion/train/trainer.py +196 -0
- src/amoe/io/checkpoint.py +122 -0
- src/amoe/io/convert.py +86 -0
- src/amoe/io/safetensors_io.py +110 -0
- src/amoe/testing/diffusion_invariants.py +334 -0
- tests/test_invariants.py +2 -0
README.md
CHANGED
|
@@ -4,34 +4,39 @@ tags:
|
|
| 4 |
- adapters
|
| 5 |
- mixture-of-experts
|
| 6 |
- aleph
|
|
|
|
| 7 |
---
|
| 8 |
|
| 9 |
# amoe-lora — aleph mixture-of-experts adapters
|
| 10 |
|
| 11 |
Train, **attach**, align, **detach** small aleph-addressed adapters on
|
| 12 |
-
a frozen
|
| 13 |
architecture's real failure modes built in as first-class API.
|
| 14 |
|
| 15 |
-
Productized from the geolip-aleph research
|
| 16 |
-
([research record](https://huggingface.co/AbstractPhil/geolip-aleph-qwen-3.5-0.8b-instruct) ·
|
|
|
|
| 17 |
[working caption adapter](https://huggingface.co/AbstractPhil/qwen3.5-0.8b-relay-caption)).
|
| 18 |
"LoRA-style" refers to the attach/detach usage pattern, not the math:
|
| 19 |
these are 16-slot patch heads reading the residual stream through a
|
| 20 |
closed-form sinh/cosh address over a 64-atom S³ codebook (~261k
|
| 21 |
params/block at d=1024).
|
| 22 |
|
| 23 |
-
**Status: 0.
|
| 24 |
toggle / detach) and checkpoint I/O are implemented and invariant-
|
| 25 |
-
tested (CPU CI, no downloads)
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
## Install
|
| 31 |
|
| 32 |
```
|
| 33 |
pip install torch # >= 2.1
|
| 34 |
-
pip install -e . #
|
| 35 |
```
|
| 36 |
|
| 37 |
## The five verbs
|
|
@@ -86,6 +91,72 @@ adapter trained on more draws than the space holds distinct questions
|
|
| 86 |
memorizes the space and posts fake generalization — measured in the
|
| 87 |
research line, twice).
|
| 88 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
## House laws (encoded, not optional)
|
| 90 |
|
| 91 |
Pure Adam only (`laws.make_optimizer`); dense signed dispatch with an
|
|
@@ -101,7 +172,14 @@ the research record.
|
|
| 101 |
- **Training**: the trainer is DDP-aware (rank-sharded sampling,
|
| 102 |
`find_unused_parameters=False`; launch with `torchrun`). Correct by
|
| 103 |
construction; multi-GPU smoke is **deferred** (single-GPU-verified).
|
| 104 |
-
- **
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
`docs/distributed.md` for why and the sketch.
|
| 106 |
|
| 107 |
## Maturity
|
|
@@ -113,7 +191,13 @@ the research record.
|
|
| 113 |
| checkpoint I/O + legacy import | implemented; round-trip verified against the shipped campaign checkpoints |
|
| 114 |
| train / align | reference-grade (the certified campaign recipes, transplanted); single-GPU |
|
| 115 |
| DDP | implemented, untested on >1 GPU |
|
| 116 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
|
| 118 |
## Loading the campaign's shipped artifacts
|
| 119 |
|
|
@@ -128,6 +212,9 @@ ck = load_anchor(hf_hub_download(
|
|
| 128 |
|
| 129 |
Architecture and every default in this package trace to the
|
| 130 |
[geolip-aleph-qwen-3.5-0.8b-instruct](https://huggingface.co/AbstractPhil/geolip-aleph-qwen-3.5-0.8b-instruct)
|
| 131 |
-
campaign (
|
| 132 |
-
|
| 133 |
-
[Qwen2.5 predecessor](https://huggingface.co/AbstractPhil/geolip-aleph-qwen)
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
- adapters
|
| 5 |
- mixture-of-experts
|
| 6 |
- aleph
|
| 7 |
+
- diffusion
|
| 8 |
---
|
| 9 |
|
| 10 |
# amoe-lora — aleph mixture-of-experts adapters
|
| 11 |
|
| 12 |
Train, **attach**, align, **detach** small aleph-addressed adapters on
|
| 13 |
+
a frozen trunk — with the honesty diagnostics that found this
|
| 14 |
architecture's real failure modes built in as first-class API.
|
| 15 |
|
| 16 |
+
Productized from the geolip-aleph research lines
|
| 17 |
+
([LM research record](https://huggingface.co/AbstractPhil/geolip-aleph-qwen-3.5-0.8b-instruct) ·
|
| 18 |
+
[diffusion research record](https://huggingface.co/AbstractPhil/geolip-aleph-diffusion) ·
|
| 19 |
[working caption adapter](https://huggingface.co/AbstractPhil/qwen3.5-0.8b-relay-caption)).
|
| 20 |
"LoRA-style" refers to the attach/detach usage pattern, not the math:
|
| 21 |
these are 16-slot patch heads reading the residual stream through a
|
| 22 |
closed-form sinh/cosh address over a 64-atom S³ codebook (~261k
|
| 23 |
params/block at d=1024).
|
| 24 |
|
| 25 |
+
**Status: 0.2 — working-state framework.** The runtime verbs (attach /
|
| 26 |
toggle / detach) and checkpoint I/O are implemented and invariant-
|
| 27 |
+
tested (CPU CI, no downloads), now on BOTH substrate families: language
|
| 28 |
+
trunks (`amoe`) and diffusion denoisers (`amoe.diffusion` — SD1.5-class
|
| 29 |
+
UNets, SDXL, Cosmos-Predict2/Anima DiT). 0.2 also ends the safetensors
|
| 30 |
+
non-goal: anchors save/load `.safetensors` with a ComfyUI-ready key
|
| 31 |
+
layout. The train recipes are reference-grade transplants of the
|
| 32 |
+
certified campaign recipes — runnable, single-GPU-verified, not yet
|
| 33 |
+
optimized or deep-tested. See *Maturity* below.
|
| 34 |
|
| 35 |
## Install
|
| 36 |
|
| 37 |
```
|
| 38 |
pip install torch # >= 2.1
|
| 39 |
+
pip install -e . # extras: .[hf] (LM), .[diffusion] (diffusers+safetensors)
|
| 40 |
```
|
| 41 |
|
| 42 |
## The five verbs
|
|
|
|
| 91 |
memorizes the space and posts fake generalization — measured in the
|
| 92 |
research line, twice).
|
| 93 |
|
| 94 |
+
## Diffusion (new in 0.2)
|
| 95 |
+
|
| 96 |
+
The same verbs on a frozen denoiser — the productized r2 campaign
|
| 97 |
+
([geolip-aleph-diffusion](https://huggingface.co/AbstractPhil/geolip-aleph-diffusion):
|
| 98 |
+
relay certified relay-favorable 3-for-3 substrates, the multiband
|
| 99 |
+
mechanism with surgical band lesions, the step-gated controller, and
|
| 100 |
+
the **conditioning law**).
|
| 101 |
+
|
| 102 |
+
```python
|
| 103 |
+
# pip install amoe-lora[diffusion]
|
| 104 |
+
import torch, amoe.diffusion as ad
|
| 105 |
+
from diffusers import StableDiffusionPipeline
|
| 106 |
+
from huggingface_hub import hf_hub_download
|
| 107 |
+
|
| 108 |
+
pipe = StableDiffusionPipeline.from_pretrained(
|
| 109 |
+
"stable-diffusion-v1-5/stable-diffusion-v1-5",
|
| 110 |
+
torch_dtype=torch.float32).to("cuda")
|
| 111 |
+
|
| 112 |
+
h = ad.attach(pipe.unet, hf_hub_download(
|
| 113 |
+
"AbstractPhil/aleph-diffusion-adapters", "sd15/mb3_s0.safetensors"))
|
| 114 |
+
img = ad.sample(pipe, h, "a lighthouse at dusk", seed=7) # step-gated
|
| 115 |
+
|
| 116 |
+
with h.lesion_band(2): # generate without the HIGH band
|
| 117 |
+
img_no_high = ad.sample(pipe, h, "a lighthouse at dusk", seed=7)
|
| 118 |
+
base = h.detach() # bit-exact or raises
|
| 119 |
+
```
|
| 120 |
+
|
| 121 |
+
Training on a folder of images is five lines:
|
| 122 |
+
|
| 123 |
+
```python
|
| 124 |
+
cache = ad.data.build_cache("photos/", pipe, captions="photos/captions.txt")
|
| 125 |
+
cfg = ad.TrainConfig(name="mystyle", objective="flow",
|
| 126 |
+
adapter="multiband3", blob=False)
|
| 127 |
+
ck = ad.train(pipe.unet, cache, cfg)
|
| 128 |
+
ck.save("mystyle.anchor.safetensors")
|
| 129 |
+
```
|
| 130 |
+
|
| 131 |
+
Three certified deviations from the LM defaults, each encoded as law
|
| 132 |
+
with its receipt (`amoe.diffusion.laws`):
|
| 133 |
+
|
| 134 |
+
- **Dtype law**: adapters attach at the DECLARED trunk dtype (bf16
|
| 135 |
+
relays on a bf16 DiT), never a sniffed one — first-param sniffing can
|
| 136 |
+
read a still-meta fp32 tensor. Judged gauges stay fp32.
|
| 137 |
+
- **`align()` is a grounded negative**: comparative routing on
|
| 138 |
+
diffusion was falsified at 2 seeds (state+sigma keys route nothing;
|
| 139 |
+
address-as-key dies against the repeated-key null in both raw and
|
| 140 |
+
M-hat forms). `ad.align()` raises with the record. The certified
|
| 141 |
+
alternative is STRUCTURAL banding — cosine crossfade windows on the
|
| 142 |
+
sigma axis (`adapter="multiband3"`), whose band lesions are surgical
|
| 143 |
+
(own-band damage 50–200× cross-band, 2 seeds).
|
| 144 |
+
- **Conditioning law**: blob/structural supervision pays where x0
|
| 145 |
+
recovery is linear (flow: `x0 = x_t − σ·v`; ~125–200× the eps
|
| 146 |
+
effect) and is inert on eps — `blob=True` with `objective="eps"`
|
| 147 |
+
refuses unless forced.
|
| 148 |
+
|
| 149 |
+
### safetensors + ComfyUI
|
| 150 |
+
|
| 151 |
+
`ck.save("x.safetensors")` writes the canonical flat layout
|
| 152 |
+
(`blocks.{site}.{param}`) with the full meta as safetensors metadata;
|
| 153 |
+
`amoe-convert path.pt --substrate sd15` converts every legacy campaign
|
| 154 |
+
shape with bitwise verification. A `comfyui-amoe` custom-node package
|
| 155 |
+
(loader / attach / band-toggle / detach, with step gating installed as
|
| 156 |
+
a pre-forward hook so ANY stock KSampler becomes step-gated) is the
|
| 157 |
+
next-turn deliverable; the node consumes the canonical layout and
|
| 158 |
+
asserts the anchor's site-count + width signature before patching.
|
| 159 |
+
|
| 160 |
## House laws (encoded, not optional)
|
| 161 |
|
| 162 |
Pure Adam only (`laws.make_optimizer`); dense signed dispatch with an
|
|
|
|
| 172 |
- **Training**: the trainer is DDP-aware (rank-sharded sampling,
|
| 173 |
`find_unused_parameters=False`; launch with `torchrun`). Correct by
|
| 174 |
construction; multi-GPU smoke is **deferred** (single-GPU-verified).
|
| 175 |
+
- **Diffusion — the declared split (the standard paradigm)**: native
|
| 176 |
+
`amoe.diffusion.train` = single-GPU + DDP at framework level;
|
| 177 |
+
the PRODUCTION multi-GPU path is the
|
| 178 |
+
[diffusion-pipe fork](https://github.com/AbstractEyes/diffusion-pipe)
|
| 179 |
+
(DeepSpeed pipeline engine, aleph relays baked in, proven end-to-end
|
| 180 |
+
on the Anima 2B DiT — r2 exp004). Use the fork when you have real
|
| 181 |
+
scale; use the native trainer for the certified single-card recipes.
|
| 182 |
+
- **FSDP / tensor parallel**: not implemented; see
|
| 183 |
`docs/distributed.md` for why and the sketch.
|
| 184 |
|
| 185 |
## Maturity
|
|
|
|
| 191 |
| checkpoint I/O + legacy import | implemented; round-trip verified against the shipped campaign checkpoints |
|
| 192 |
| train / align | reference-grade (the certified campaign recipes, transplanted); single-GPU |
|
| 193 |
| DDP | implemented, untested on >1 GPU |
|
| 194 |
+
| diffusion core (relay/multiband/windows) | ported verbatim from the certified r2 beds; invariant-tested |
|
| 195 |
+
| diffusion attach / lesions / detach | implemented, invariant-tested incl. dtype law (CPU CI) |
|
| 196 |
+
| diffusion train (eps/flow/blob/roles) | reference-grade transplant; single-GPU-verified recipes |
|
| 197 |
+
| StepGatedSampler | eps/DDIM = the exp010-proven path; flow/Euler = same warp as the beds |
|
| 198 |
+
| safetensors I/O + amoe-convert | implemented in 0.2 (bitwise-verified round trips); ComfyUI node = next turn |
|
| 199 |
+
| SDXL binding | enumeration implemented; site count PINNED AT FIRST REAL RUN (never guessed) |
|
| 200 |
+
| FSDP/TP, token-level gating, streaming data | documented non-goals |
|
| 201 |
|
| 202 |
## Loading the campaign's shipped artifacts
|
| 203 |
|
|
|
|
| 212 |
|
| 213 |
Architecture and every default in this package trace to the
|
| 214 |
[geolip-aleph-qwen-3.5-0.8b-instruct](https://huggingface.co/AbstractPhil/geolip-aleph-qwen-3.5-0.8b-instruct)
|
| 215 |
+
campaign (19 experiment packages, raw ledgers, adversarial audits, one
|
| 216 |
+
published self-retraction plus a corrected overclaim), its
|
| 217 |
+
[Qwen2.5 predecessor](https://huggingface.co/AbstractPhil/geolip-aleph-qwen),
|
| 218 |
+
and — for the 0.2 diffusion subsystem — the
|
| 219 |
+
[geolip-aleph-diffusion](https://huggingface.co/AbstractPhil/geolip-aleph-diffusion)
|
| 220 |
+
campaign (16 experiment packages, 2-seed program, the conditioning law).
|
docs/distributed.md
CHANGED
|
@@ -1,4 +1,25 @@
|
|
| 1 |
-
# Distributed story (0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
## Implemented
|
| 4 |
|
|
|
|
| 1 |
+
# Distributed story (0.2)
|
| 2 |
+
|
| 3 |
+
## The diffusion split (the standard paradigm, 0.2)
|
| 4 |
+
|
| 5 |
+
Diffusion training has TWO sanctioned paths, by design:
|
| 6 |
+
|
| 7 |
+
1. **Native `amoe.diffusion.train`** — single-GPU + the same DDP
|
| 8 |
+
posture as the LM trainer below (rank-sharded cache sampling,
|
| 9 |
+
`find_unused_parameters=False`, rank-0 saves). This is the
|
| 10 |
+
framework-level path for the certified single-card recipes;
|
| 11 |
+
correct-by-construction, multi-GPU smoke deferred.
|
| 12 |
+
2. **The diffusion-pipe fork**
|
| 13 |
+
(https://github.com/AbstractEyes/diffusion-pipe) — the PRODUCTION
|
| 14 |
+
multi-GPU trainer: DeepSpeed pipeline engine with the aleph relays
|
| 15 |
+
baked into the model integrations (attach at declared dtype,
|
| 16 |
+
freeze-by-lr-0 trunk, plain-Adam branch, adapter-only saves in the
|
| 17 |
+
amoe anchor format). Proven end-to-end on the Anima 2B DiT
|
| 18 |
+
(r2 exp004). Multiband w_bands plumbing is working at
|
| 19 |
+
`pipeline_stages=1`; stages>1 is documented-deferred.
|
| 20 |
+
|
| 21 |
+
No FSDP on either path, same reasons as below.
|
| 22 |
+
|
| 23 |
|
| 24 |
## Implemented
|
| 25 |
|
examples/attach_diffusion.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# attach_diffusion.py — the diffusion quickstart, runnable as-is on a
|
| 2 |
+
# CUDA machine with `pip install amoe-lora[diffusion]` (Colab: paste the
|
| 3 |
+
# cell; no argparse, no __file__).
|
| 4 |
+
import torch
|
| 5 |
+
|
| 6 |
+
import amoe.diffusion as ad
|
| 7 |
+
|
| 8 |
+
from diffusers import StableDiffusionPipeline
|
| 9 |
+
from huggingface_hub import hf_hub_download
|
| 10 |
+
|
| 11 |
+
pipe = StableDiffusionPipeline.from_pretrained(
|
| 12 |
+
"stable-diffusion-v1-5/stable-diffusion-v1-5",
|
| 13 |
+
torch_dtype=torch.float32).to("cuda")
|
| 14 |
+
|
| 15 |
+
# The certified multiband stack (exp008 s0; 2-seed surgical band lesions).
|
| 16 |
+
anchor = hf_hub_download("AbstractPhil/aleph-diffusion-adapters",
|
| 17 |
+
"sd15/mb3_s0.safetensors")
|
| 18 |
+
h = ad.attach(pipe.unet, anchor)
|
| 19 |
+
print(h.gates()) # gate health at load
|
| 20 |
+
|
| 21 |
+
img = ad.sample(pipe, h, "a lighthouse at dusk, film photo", seed=7)
|
| 22 |
+
img.save("lighthouse.png")
|
| 23 |
+
|
| 24 |
+
# Band lesion: generate WITHOUT the HIGH-noise/structure band — the
|
| 25 |
+
# exp010 image-space instrument (lesioning HIGH cut grounding most and
|
| 26 |
+
# was 14x LP-dominated: it carries coarse structure).
|
| 27 |
+
with h.lesion_band(2):
|
| 28 |
+
ad.sample(pipe, h, "a lighthouse at dusk, film photo",
|
| 29 |
+
seed=7).save("lighthouse_no_high.png")
|
| 30 |
+
|
| 31 |
+
base = h.detach() # verified bit-exact, or it raises
|
| 32 |
+
print("detached clean")
|
pyproject.toml
CHANGED
|
@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
|
|
| 4 |
|
| 5 |
[project]
|
| 6 |
name = "amoe-lora"
|
| 7 |
-
version = "0.
|
| 8 |
-
description = "Aleph mixture-of-experts adapters: train, attach, align, detach — with the honesty diagnostics built in"
|
| 9 |
readme = "README.md"
|
| 10 |
requires-python = ">=3.10"
|
| 11 |
license = { text = "Apache-2.0" }
|
|
@@ -14,6 +14,10 @@ dependencies = ["torch>=2.1"]
|
|
| 14 |
|
| 15 |
[project.optional-dependencies]
|
| 16 |
hf = ["transformers>=4.57", "huggingface_hub"]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
[tool.setuptools.packages.find]
|
| 19 |
where = ["src"]
|
|
|
|
| 4 |
|
| 5 |
[project]
|
| 6 |
name = "amoe-lora"
|
| 7 |
+
version = "0.2.0"
|
| 8 |
+
description = "Aleph mixture-of-experts adapters: train, attach, align, detach — with the honesty diagnostics built in. 0.2 adds the diffusion subsystem (amoe.diffusion) and safetensors I/O."
|
| 9 |
readme = "README.md"
|
| 10 |
requires-python = ">=3.10"
|
| 11 |
license = { text = "Apache-2.0" }
|
|
|
|
| 14 |
|
| 15 |
[project.optional-dependencies]
|
| 16 |
hf = ["transformers>=4.57", "huggingface_hub"]
|
| 17 |
+
diffusion = ["diffusers>=0.30", "safetensors>=0.4"]
|
| 18 |
+
|
| 19 |
+
[project.scripts]
|
| 20 |
+
amoe-convert = "amoe.io.convert:main"
|
| 21 |
|
| 22 |
[tool.setuptools.packages.find]
|
| 23 |
where = ["src"]
|
src/amoe/__init__.py
CHANGED
|
@@ -16,4 +16,8 @@ from .train.trainer import train
|
|
| 16 |
from .train.aligner import align
|
| 17 |
from . import laws
|
| 18 |
|
| 19 |
-
__version__ = "0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
from .train.aligner import align
|
| 17 |
from . import laws
|
| 18 |
|
| 19 |
+
__version__ = "0.2.0"
|
| 20 |
+
|
| 21 |
+
# The diffusion subsystem (amoe.diffusion) is imported lazily — its verbs
|
| 22 |
+
# live under `import amoe.diffusion as ad`. Core is pure torch; diffusers
|
| 23 |
+
# is only touched by bindings/samplers/data that need it.
|
src/amoe/binding/diffusion.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Diffusion model binding: where do the adapters attach on a denoiser?
|
| 2 |
+
|
| 3 |
+
Same substrate-shim discipline as resolver.py — everything
|
| 4 |
+
architecture-specific lives here and only here. Diffusion sites are
|
| 5 |
+
SCATTERED (heterogeneous widths across down/mid/up), so the binding
|
| 6 |
+
speaks (qualified_name, module, width) triples and replaces modules by
|
| 7 |
+
attribute walk, not by swapping one ModuleList.
|
| 8 |
+
|
| 9 |
+
DTYPE LAW: `declared_dtype` is the trunk dtype AFTER materialization.
|
| 10 |
+
It refuses meta-device models — first-param sniffing during init can
|
| 11 |
+
read a still-meta fp32 tensor (the R0b catch, fork f4f1c46). Attach
|
| 12 |
+
after weights are real, or pass dtype= explicitly.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
from dataclasses import dataclass, field
|
| 17 |
+
from typing import Callable, Protocol
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
import torch.nn as nn
|
| 21 |
+
|
| 22 |
+
from .resolver import REGISTRY
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class DiffusionBinding(Protocol):
|
| 26 |
+
name: str
|
| 27 |
+
expected_sites: int | None
|
| 28 |
+
|
| 29 |
+
def sites(self, model) -> list[tuple[str, nn.Module, int]]: ...
|
| 30 |
+
def replace(self, model, name: str, new: nn.Module) -> None: ...
|
| 31 |
+
def declared_dtype(self, model) -> torch.dtype: ...
|
| 32 |
+
def probe(self, model) -> "torch.Tensor | None": ...
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def walk_replace(root: nn.Module, name: str, new: nn.Module) -> None:
|
| 36 |
+
"""setattr/index walk to a qualified module name (the certified
|
| 37 |
+
attach pattern from the r2 beds)."""
|
| 38 |
+
parent = root
|
| 39 |
+
parts = name.split(".")
|
| 40 |
+
for p in parts[:-1]:
|
| 41 |
+
parent = getattr(parent, p) if not p.isdigit() else parent[int(p)]
|
| 42 |
+
last = parts[-1]
|
| 43 |
+
if last.isdigit():
|
| 44 |
+
parent[int(last)] = new
|
| 45 |
+
else:
|
| 46 |
+
setattr(parent, last, new)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _declared_dtype(model) -> torch.dtype:
|
| 50 |
+
metas = [n for n, p in model.named_parameters() if p.is_meta]
|
| 51 |
+
if metas:
|
| 52 |
+
raise RuntimeError(
|
| 53 |
+
"dtype law: model has meta-device parameters "
|
| 54 |
+
f"(e.g. {metas[0]}) — a dtype read here can lie (the R0b "
|
| 55 |
+
"catch). Materialize the weights first, or pass dtype= "
|
| 56 |
+
"explicitly at attach.")
|
| 57 |
+
dt = getattr(model, "dtype", None)
|
| 58 |
+
if isinstance(dt, torch.dtype):
|
| 59 |
+
return dt
|
| 60 |
+
return next(model.parameters()).dtype
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
@dataclass
|
| 64 |
+
class UNetBinding:
|
| 65 |
+
"""diffusers UNet2DConditionModel — every BasicTransformerBlock.
|
| 66 |
+
SD15 certifies at 16 sites (asserted). SDXL's count is PINNED AT THE
|
| 67 |
+
FIRST REAL RUN (deferred-by-design F1): expected_sites=None means
|
| 68 |
+
'record, do not assert a guess'."""
|
| 69 |
+
name: str = "sd15_unet"
|
| 70 |
+
expected_sites: "int | None" = 16
|
| 71 |
+
|
| 72 |
+
def sites(self, model):
|
| 73 |
+
from diffusers.models.attention import BasicTransformerBlock
|
| 74 |
+
out = []
|
| 75 |
+
for name, mod in model.named_modules():
|
| 76 |
+
if isinstance(mod, BasicTransformerBlock):
|
| 77 |
+
out.append((name, mod, mod.norm1.normalized_shape[0]))
|
| 78 |
+
if self.expected_sites is not None:
|
| 79 |
+
assert len(out) == self.expected_sites, (
|
| 80 |
+
f"{self.name}: enumerated {len(out)} BasicTransformerBlocks, "
|
| 81 |
+
f"expected {self.expected_sites}")
|
| 82 |
+
assert out, f"{self.name}: no BasicTransformerBlocks found"
|
| 83 |
+
return out
|
| 84 |
+
|
| 85 |
+
def replace(self, model, name, new):
|
| 86 |
+
walk_replace(model, name, new)
|
| 87 |
+
|
| 88 |
+
def declared_dtype(self, model):
|
| 89 |
+
return _declared_dtype(model)
|
| 90 |
+
|
| 91 |
+
def probe(self, model):
|
| 92 |
+
"""Deterministic denoiser fingerprint on a tiny fixed input,
|
| 93 |
+
built from the LIVE config (never from memory — F4)."""
|
| 94 |
+
cfg = model.config
|
| 95 |
+
n_down = len(cfg.block_out_channels)
|
| 96 |
+
size = 2 ** (n_down - 1) * 4 # divisible through the UNet
|
| 97 |
+
g = torch.Generator().manual_seed(1400)
|
| 98 |
+
dev = next(model.parameters()).device
|
| 99 |
+
dt = self.declared_dtype(model)
|
| 100 |
+
x = torch.randn(1, cfg.in_channels, size, size, generator=g).to(dev, dt)
|
| 101 |
+
ehs = torch.randn(1, 8, cfg.cross_attention_dim, generator=g).to(dev, dt)
|
| 102 |
+
kwargs = {}
|
| 103 |
+
if getattr(cfg, "addition_embed_type", None) == "text_time":
|
| 104 |
+
# SDXL-style added conditioning, shapes from the live config
|
| 105 |
+
ta = torch.randn(1, cfg.projection_class_embeddings_input_dim
|
| 106 |
+
- 6 * cfg.addition_time_embed_dim,
|
| 107 |
+
generator=g).to(dev, dt)
|
| 108 |
+
tids = torch.tensor([[size * 8, size * 8, 0, 0, size * 8, size * 8]],
|
| 109 |
+
device=dev)
|
| 110 |
+
kwargs["added_cond_kwargs"] = {"text_embeds": ta, "time_ids": tids}
|
| 111 |
+
with torch.no_grad():
|
| 112 |
+
out = model(x, 17, ehs, return_dict=False, **kwargs)[0]
|
| 113 |
+
return out.detach().float().cpu()
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
@dataclass
|
| 117 |
+
class DiTBinding:
|
| 118 |
+
"""Cosmos-Predict2/Anima-style DiT: a `blocks` ModuleList of uniform
|
| 119 |
+
width. The production trainer for this family is the diffusion-pipe
|
| 120 |
+
fork (which carries its own P-INIT/P-TOGGLE gates); the in-package
|
| 121 |
+
probe is None — detach verification degrades to structural restore
|
| 122 |
+
with a warning (documented F4 deferral)."""
|
| 123 |
+
name: str = "cosmos_dit"
|
| 124 |
+
expected_sites: "int | None" = None
|
| 125 |
+
|
| 126 |
+
def sites(self, model):
|
| 127 |
+
blocks = getattr(model, "blocks", None)
|
| 128 |
+
assert isinstance(blocks, nn.ModuleList) and len(blocks) > 0, (
|
| 129 |
+
"cosmos_dit binding expects a `blocks` ModuleList")
|
| 130 |
+
d = None
|
| 131 |
+
for attr in ("model_channels", "hidden_size", "dim"):
|
| 132 |
+
v = getattr(getattr(model, "config", model), attr,
|
| 133 |
+
getattr(model, attr, None))
|
| 134 |
+
if isinstance(v, int):
|
| 135 |
+
d = v
|
| 136 |
+
break
|
| 137 |
+
if d is None:
|
| 138 |
+
lin = next(m for m in blocks[0].modules()
|
| 139 |
+
if isinstance(m, nn.Linear))
|
| 140 |
+
d = lin.in_features
|
| 141 |
+
return [(f"blocks.{i}", b, d) for i, b in enumerate(blocks)]
|
| 142 |
+
|
| 143 |
+
def replace(self, model, name, new):
|
| 144 |
+
walk_replace(model, name, new)
|
| 145 |
+
|
| 146 |
+
def declared_dtype(self, model):
|
| 147 |
+
return _declared_dtype(model)
|
| 148 |
+
|
| 149 |
+
def probe(self, model):
|
| 150 |
+
return None
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
DIFF_REGISTRY: dict[str, Callable[[], "DiffusionBinding"]] = {}
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def register_diffusion(key: str):
|
| 157 |
+
def deco(fn):
|
| 158 |
+
DIFF_REGISTRY[key] = fn
|
| 159 |
+
REGISTRY[key] = fn # discoverable from the one registry
|
| 160 |
+
return fn
|
| 161 |
+
return deco
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
@register_diffusion("sd15_unet")
|
| 165 |
+
def _sd15():
|
| 166 |
+
return UNetBinding(name="sd15_unet", expected_sites=16)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
@register_diffusion("sdxl_unet")
|
| 170 |
+
def _sdxl():
|
| 171 |
+
return UNetBinding(name="sdxl_unet", expected_sites=None) # F1
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
@register_diffusion("cosmos_dit")
|
| 175 |
+
def _cosmos():
|
| 176 |
+
return DiTBinding()
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def resolve_diffusion(model, binding=None) -> DiffusionBinding:
|
| 180 |
+
if binding is not None:
|
| 181 |
+
if isinstance(binding, str):
|
| 182 |
+
if binding in DIFF_REGISTRY:
|
| 183 |
+
return DIFF_REGISTRY[binding]()
|
| 184 |
+
raise ValueError(f"unknown diffusion binding '{binding}'; "
|
| 185 |
+
f"known: {sorted(DIFF_REGISTRY)}")
|
| 186 |
+
return binding
|
| 187 |
+
cls = type(model).__name__
|
| 188 |
+
if cls == "UNet2DConditionModel":
|
| 189 |
+
cad = int(getattr(model.config, "cross_attention_dim", 0) or 0)
|
| 190 |
+
return DIFF_REGISTRY["sdxl_unet" if cad >= 2048 else "sd15_unet"]()
|
| 191 |
+
if isinstance(getattr(model, "blocks", None), nn.ModuleList):
|
| 192 |
+
return DIFF_REGISTRY["cosmos_dit"]()
|
| 193 |
+
cands = [f"{n} (len {len(m)})" for n, m in model.named_modules()
|
| 194 |
+
if isinstance(m, nn.ModuleList) and len(m) >= 4]
|
| 195 |
+
raise ValueError(
|
| 196 |
+
"amoe.diffusion could not resolve an attach site — pass "
|
| 197 |
+
f"binding=<DiffusionBinding|key>. ModuleList candidates: {cands}")
|
src/amoe/diagnostics/diffusion.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Diffusion diagnostics — the campaign's honesty instruments as API.
|
| 2 |
+
|
| 3 |
+
lesion_report : the exp008/exp010 band-lesion battery shape — run YOUR
|
| 4 |
+
gauge under each single-band lesion and read whether
|
| 5 |
+
specialization is surgical (own-band damage >> cross).
|
| 6 |
+
foreground_gauge: the exp012 role-aligned payer instrument — HIGH-band
|
| 7 |
+
foreground-LP-x0 error (the gauge that showed multiband
|
| 8 |
+
beating the matched monolith ~10% when every
|
| 9 |
+
aggregate-eps comparison was blind to it).
|
| 10 |
+
gate_stats : gate health (sigmoid means; the substrate-dependent
|
| 11 |
+
dynamics finding — core grew, lune shrank).
|
| 12 |
+
diagnose : one-call summary with the loud flags.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
|
| 18 |
+
from ..diffusion.laws import N_BANDS
|
| 19 |
+
from ..diffusion.train.objectives import blob_lp_err
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def lesion_report(handle, eval_fn) -> dict:
|
| 23 |
+
"""eval_fn() -> float, evaluated under all_on and each single-band
|
| 24 |
+
lesion. Multiband anchors only. Returns {'all_on': x,
|
| 25 |
+
'lesion_band0': ..., 'surgical': bool} where surgical means every
|
| 26 |
+
lesion moved the gauge (the exp008 signature reads per-band gauges;
|
| 27 |
+
with a single gauge this reports the monotone lesion profile the
|
| 28 |
+
exp010 battery certified)."""
|
| 29 |
+
out = {"all_on": float(eval_fn())}
|
| 30 |
+
for b in range(N_BANDS):
|
| 31 |
+
with handle.lesion_band(b):
|
| 32 |
+
out[f"lesion_band{b}"] = float(eval_fn())
|
| 33 |
+
deltas = [out[f"lesion_band{b}"] - out["all_on"] for b in range(N_BANDS)]
|
| 34 |
+
out["lesion_deltas"] = [round(d, 6) for d in deltas]
|
| 35 |
+
return out
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def foreground_gauge(x0_hat: torch.Tensor, x0: torch.Tensor,
|
| 39 |
+
blob: torch.Tensor) -> float:
|
| 40 |
+
"""Mean foreground-LP-x0 error (lower is better). Judge in fp32 (law)."""
|
| 41 |
+
return float(blob_lp_err(x0_hat.float(), x0.float(),
|
| 42 |
+
blob.float()).mean())
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@torch.no_grad()
|
| 46 |
+
def gate_stats(handle) -> dict:
|
| 47 |
+
return handle.gates()
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@torch.no_grad()
|
| 51 |
+
def diagnose(handle) -> dict:
|
| 52 |
+
"""One-call health readout: gates, amplitude telemetry (if armed),
|
| 53 |
+
and the standing scope note on gate bands."""
|
| 54 |
+
rep = {"kind": handle.kind, "n_sites": len(handle.names),
|
| 55 |
+
**gate_stats(handle)}
|
| 56 |
+
amp = handle.amplitude()
|
| 57 |
+
if amp:
|
| 58 |
+
vals = list(amp.values())
|
| 59 |
+
rep["delta_ratio_mean"] = round(sum(vals) / len(vals), 6)
|
| 60 |
+
rep["note"] = ("gate dynamics are SUBSTRATE-DEPENDENT on diffusion "
|
| 61 |
+
"(exp006: core grew 0.047->0.070; exp001: lune shrank) "
|
| 62 |
+
"— read direction against your own frozen baseline, "
|
| 63 |
+
"not against the LM band")
|
| 64 |
+
return rep
|
src/amoe/diffusion/__init__.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""amoe.diffusion — the aleph adapter verbs on diffusion trunks (0.2).
|
| 2 |
+
|
| 3 |
+
Same five-verb grammar as the text line, three certified deviations
|
| 4 |
+
(see .laws): the dtype law, structural banding instead of comparative
|
| 5 |
+
routing (align() is a grounded negative), and the declared multi-GPU
|
| 6 |
+
split (native train = single-GPU/DDP; production multi-GPU = the
|
| 7 |
+
diffusion-pipe fork).
|
| 8 |
+
|
| 9 |
+
Quickstart:
|
| 10 |
+
import amoe.diffusion as ad
|
| 11 |
+
h = ad.attach(pipe.unet, "mb3_s0.safetensors")
|
| 12 |
+
img = ad.sample(pipe, h, "a lighthouse at dusk", seed=7)
|
| 13 |
+
with h.lesion_band(2): ...
|
| 14 |
+
base = h.detach() # bit-exact or raises
|
| 15 |
+
"""
|
| 16 |
+
from ..io.checkpoint import (DiffusionAnchorCheckpoint,
|
| 17 |
+
load_diffusion_anchor)
|
| 18 |
+
from .core.cond import AlephCondAdapter
|
| 19 |
+
from .core.multiband import (BandBlockWrap, MultibandDelta, band_of,
|
| 20 |
+
band_weights)
|
| 21 |
+
from .core.relay import BlockWithRelay, RelayPatch2D
|
| 22 |
+
from .runtime.attach import DiffusionAttachHandle, attach, detach
|
| 23 |
+
from .runtime.sampler import StepGatedSampler, sample
|
| 24 |
+
from .train.aligner import align
|
| 25 |
+
from .train.config import DiffusionTrainConfig, RelaySpec
|
| 26 |
+
from .train.config import DiffusionTrainConfig as TrainConfig
|
| 27 |
+
from .train.trainer import ConditioningLawWarning, train
|
| 28 |
+
from . import laws
|
| 29 |
+
from .train import data
|
src/amoe/diffusion/core/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .cond import AlephCondAdapter
|
| 2 |
+
from .multiband import BandBlockWrap, MultibandDelta, band_of, band_weights
|
| 3 |
+
from .relay import BlockWithRelay, RelayPatch2D
|
src/amoe/diffusion/core/cond.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AlephCondAdapter — conditioning-stream address injection (research-grade
|
| 2 |
+
export; verbatim port of pod2/aleph_diffusion_core.py).
|
| 3 |
+
|
| 4 |
+
STATUS (read before using): exp002 certified this channel as a Law-2 honest
|
| 5 |
+
negative IN CONTEXT — appended to full text conditioning the address is
|
| 6 |
+
marginally inert (real-vs-deranged −0.0009). But addr-only steering is real
|
| 7 |
+
(+0.0287 after 2k steps, gates GREW) — the channel is redundant-in-context,
|
| 8 |
+
not dead. The open redesign target is COMPLEMENTARITY: inject where text is
|
| 9 |
+
not (reduced-text regimes, sigma-gated, non-text streams). Shipped as a
|
| 10 |
+
component for that line of work, not as a quickstart verb.
|
| 11 |
+
"""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import torch
|
| 15 |
+
import torch.nn as nn
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class AlephCondAdapter(nn.Module):
|
| 19 |
+
"""Frozen [n_addr, addr_dim] address -> n_addr conditioning positions.
|
| 20 |
+
Everything zero-init => tokens are EXACT zeros at init and when
|
| 21 |
+
disabled (the position-presence offset is handled by the bed: SD15
|
| 22 |
+
masked-append/length-toggle, Anima write-into-padding)."""
|
| 23 |
+
|
| 24 |
+
def __init__(self, cond_dim: int, addr_dim: int = 128, n_addr: int = 32):
|
| 25 |
+
super().__init__()
|
| 26 |
+
self.cond_dim, self.addr_dim, self.n_addr = cond_dim, addr_dim, n_addr
|
| 27 |
+
self.addr_proj = nn.Linear(addr_dim, cond_dim)
|
| 28 |
+
nn.init.zeros_(self.addr_proj.weight)
|
| 29 |
+
nn.init.zeros_(self.addr_proj.bias)
|
| 30 |
+
self.pos_table = nn.Parameter(torch.zeros(n_addr, cond_dim))
|
| 31 |
+
self.gate = nn.Parameter(torch.tensor(-3.0))
|
| 32 |
+
self.enabled = True
|
| 33 |
+
|
| 34 |
+
def assert_zero_init(self):
|
| 35 |
+
assert self.addr_proj.weight.abs().max().item() == 0.0
|
| 36 |
+
assert self.addr_proj.bias.abs().max().item() == 0.0
|
| 37 |
+
assert self.pos_table.abs().max().item() == 0.0
|
| 38 |
+
|
| 39 |
+
def tokens(self, addr: torch.Tensor) -> torch.Tensor:
|
| 40 |
+
"""addr [B, n_addr, addr_dim] -> [B, n_addr, cond_dim]."""
|
| 41 |
+
assert addr.shape[-2:] == (self.n_addr, self.addr_dim), addr.shape
|
| 42 |
+
if not self.enabled:
|
| 43 |
+
return addr.new_zeros(*addr.shape[:-1], self.cond_dim)
|
| 44 |
+
return torch.sigmoid(self.gate) * (self.addr_proj(addr) + self.pos_table)
|
| 45 |
+
|
| 46 |
+
@classmethod
|
| 47 |
+
def from_state_dict(cls, sd: dict) -> "AlephCondAdapter":
|
| 48 |
+
cond_dim, addr_dim = sd["addr_proj.weight"].shape
|
| 49 |
+
n_addr = sd["pos_table"].shape[0]
|
| 50 |
+
m = cls(cond_dim, addr_dim=addr_dim, n_addr=n_addr)
|
| 51 |
+
m.load_state_dict(sd, strict=True)
|
| 52 |
+
return m
|
src/amoe/diffusion/core/multiband.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MultibandDelta — the certified band-assignment mechanism (verbatim port
|
| 2 |
+
of pod2/dexp008_multiband.py; exp008: band lesions surgical 3/3 at both
|
| 3 |
+
seeds, own-band damage 50–200× cross-band; exp012's role-aligned gauge:
|
| 4 |
+
multiband beats the matched monolith ~10% on HIGH-band foreground
|
| 5 |
+
structure, 2-seed).
|
| 6 |
+
|
| 7 |
+
Per site: N_BANDS rank-r zero-init deltas (W+b, own gates −3.0), combined
|
| 8 |
+
by smooth COSINE CROSSFADE WINDOWS on s01 — STRUCTURAL/positional gating
|
| 9 |
+
on the sigma axis, dense and differentiable, no selection event (law 2 of
|
| 10 |
+
amoe.diffusion.laws). Per-band gradients arise from the windows: a sample
|
| 11 |
+
at s01=0.9 trains (almost) only the HIGH expert.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import math
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
import torch.nn as nn
|
| 19 |
+
|
| 20 |
+
from ..laws import BAND_EDGES, N_BANDS, XFADE
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def band_weights(s01: torch.Tensor) -> torch.Tensor:
|
| 24 |
+
"""Smooth structural band windows: (B, N_BANDS), sum to 1 everywhere.
|
| 25 |
+
Cosine crossfade of half-width XFADE around each edge — positional
|
| 26 |
+
gating on the sigma axis, never comparative."""
|
| 27 |
+
def ramp(x): # 0 below -XFADE, 1 above +XFADE, smooth
|
| 28 |
+
t = ((x / XFADE).clamp(-1, 1) + 1) / 2
|
| 29 |
+
return 0.5 - 0.5 * torch.cos(t * math.pi)
|
| 30 |
+
e1, e2 = BAND_EDGES
|
| 31 |
+
up1, up2 = ramp(s01 - e1), ramp(s01 - e2)
|
| 32 |
+
low = 1 - up1
|
| 33 |
+
mid = up1 * (1 - up2)
|
| 34 |
+
high = up1 * up2
|
| 35 |
+
return torch.stack([low, mid, high], dim=-1)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def band_of(s01: float) -> int:
|
| 39 |
+
e1, e2 = BAND_EDGES
|
| 40 |
+
return 0 if s01 <= e1 else (1 if s01 <= e2 else 2)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class MultibandDelta(nn.Module):
|
| 44 |
+
"""Three band experts per site, window-combined; per-expert enable
|
| 45 |
+
flags (band lesions) + global enable (toggle law, code-path skip)."""
|
| 46 |
+
|
| 47 |
+
def __init__(self, d: int, r: int = 16):
|
| 48 |
+
super().__init__()
|
| 49 |
+
self.d, self.r = d, r
|
| 50 |
+
self.down = nn.ModuleList(nn.Linear(d, r, bias=False)
|
| 51 |
+
for _ in range(N_BANDS))
|
| 52 |
+
self.up = nn.ModuleList(nn.Linear(r, d) for _ in range(N_BANDS))
|
| 53 |
+
for dn, up in zip(self.down, self.up):
|
| 54 |
+
nn.init.orthogonal_(dn.weight)
|
| 55 |
+
nn.init.zeros_(up.weight)
|
| 56 |
+
nn.init.zeros_(up.bias)
|
| 57 |
+
self.gates = nn.Parameter(torch.full((N_BANDS,), -3.0))
|
| 58 |
+
self.enabled = True
|
| 59 |
+
self.band_enabled = [True] * N_BANDS
|
| 60 |
+
|
| 61 |
+
def assert_zero_init(self):
|
| 62 |
+
for up in self.up:
|
| 63 |
+
assert up.weight.abs().max().item() == 0.0
|
| 64 |
+
assert up.bias.abs().max().item() == 0.0
|
| 65 |
+
|
| 66 |
+
def forward(self, x, w_bands): # w_bands: (B, N_BANDS)
|
| 67 |
+
if not self.enabled:
|
| 68 |
+
return x
|
| 69 |
+
if w_bands is None:
|
| 70 |
+
raise RuntimeError(
|
| 71 |
+
"MultibandDelta needs band windows — set them per step "
|
| 72 |
+
"(handle.set_band_windows / StepGatedSampler) or per "
|
| 73 |
+
"training batch from the sampled sigmas")
|
| 74 |
+
g = torch.sigmoid(self.gates)
|
| 75 |
+
delta = 0
|
| 76 |
+
w = w_bands.view(w_bands.shape[0],
|
| 77 |
+
*([1] * (x.ndim - 2)), N_BANDS)
|
| 78 |
+
for b in range(N_BANDS):
|
| 79 |
+
if not self.band_enabled[b]:
|
| 80 |
+
continue
|
| 81 |
+
delta = delta + g[b] * w[..., b:b + 1] * self.up[b](
|
| 82 |
+
self.down[b](x))
|
| 83 |
+
return x + delta if not isinstance(delta, int) else x
|
| 84 |
+
|
| 85 |
+
@classmethod
|
| 86 |
+
def from_state_dict(cls, sd: dict) -> "MultibandDelta":
|
| 87 |
+
d = sd["down.0.weight"].shape[1]
|
| 88 |
+
r = sd["down.0.weight"].shape[0]
|
| 89 |
+
m = cls(d, r=r)
|
| 90 |
+
m.load_state_dict(sd, strict=True)
|
| 91 |
+
return m
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class BandBlockWrap(nn.Module):
|
| 95 |
+
"""Wrap one trunk block with a window-consuming module. `w_bands` is
|
| 96 |
+
set externally (per training batch, or per sampling step by the
|
| 97 |
+
controller). Tuple outputs pass through."""
|
| 98 |
+
|
| 99 |
+
def __init__(self, block: nn.Module, mod: nn.Module):
|
| 100 |
+
super().__init__()
|
| 101 |
+
self.block = block
|
| 102 |
+
self.mod = mod
|
| 103 |
+
self.w_bands = None
|
| 104 |
+
|
| 105 |
+
def forward(self, *args, **kwargs):
|
| 106 |
+
out = self.block(*args, **kwargs)
|
| 107 |
+
h = out[0] if isinstance(out, tuple) else out
|
| 108 |
+
h = self.mod(h, self.w_bands)
|
| 109 |
+
return (h,) + out[1:] if isinstance(out, tuple) else h
|
src/amoe/diffusion/core/relay.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""RelayPatch2D — the certified per-block residual relay for diffusion
|
| 2 |
+
token streams (verbatim port of the r2 bed, pod2/aleph_diffusion_core.py;
|
| 3 |
+
certified relay-favorable 3-for-3 substrates: Lune flow / SD15-core eps /
|
| 4 |
+
Anima DiT flow — exp001/exp006/exp004).
|
| 5 |
+
|
| 6 |
+
Differences from the text-side RelayPatchwork (amoe.core.adapter), each a
|
| 7 |
+
paid-for law:
|
| 8 |
+
* trailing-dim generalization: forward works on any [..., d] stream;
|
| 9 |
+
* zero-init output weight AND bias (exp006 bias-leak law) — a fresh
|
| 10 |
+
relay is EXACTLY inert, P-INIT bit-exact;
|
| 11 |
+
* `enabled=False` returns x via CODE-PATH SKIP (`return x`, `is x`) —
|
| 12 |
+
the toggle law is bit-exact at every dtype;
|
| 13 |
+
* dims-from-state-dict reconstruction (never trust constructor defaults).
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
import torch.nn as nn
|
| 19 |
+
|
| 20 |
+
from ...core.address import AlephAddress
|
| 21 |
+
from ...core.adapter import SquaredReLU
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class RelayPatch2D(nn.Module):
|
| 25 |
+
"""forward(x[..., d]) = x + sigmoid(gate) * consume(m_hat(proj(x) slots))."""
|
| 26 |
+
|
| 27 |
+
def __init__(self, d: int, n_slots: int = 16, K: int = 64,
|
| 28 |
+
tau: float = 0.1, hidden: int = 178):
|
| 29 |
+
super().__init__()
|
| 30 |
+
self.d, self.n_slots, self.hidden = d, n_slots, hidden
|
| 31 |
+
self.proj = nn.Linear(d, n_slots * 4, bias=False)
|
| 32 |
+
nn.init.orthogonal_(self.proj.weight)
|
| 33 |
+
self.addr = AlephAddress(K, 4, tau)
|
| 34 |
+
self.consume = nn.Sequential(
|
| 35 |
+
nn.Linear(n_slots * 4, hidden), SquaredReLU(),
|
| 36 |
+
nn.LayerNorm(hidden), nn.Linear(hidden, d))
|
| 37 |
+
nn.init.zeros_(self.consume[-1].weight) # zero-init out: weight
|
| 38 |
+
nn.init.zeros_(self.consume[-1].bias) # AND bias (exp006 law)
|
| 39 |
+
self.gate = nn.Parameter(torch.tensor(-3.0))
|
| 40 |
+
self.enabled = True # plain attr, not a buffer
|
| 41 |
+
self.telemetry = False # read-only ratio stash
|
| 42 |
+
self.last_delta_ratio = None
|
| 43 |
+
|
| 44 |
+
def assert_zero_init(self):
|
| 45 |
+
w, b = self.consume[-1].weight, self.consume[-1].bias
|
| 46 |
+
assert w.abs().max().item() == 0.0, "out weight not zero-init"
|
| 47 |
+
assert b.abs().max().item() == 0.0, "out BIAS not zero-init (leak law)"
|
| 48 |
+
|
| 49 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 50 |
+
if not self.enabled:
|
| 51 |
+
return x
|
| 52 |
+
lead = x.shape[:-1]
|
| 53 |
+
slots = self.proj(x).view(*lead, self.n_slots, 4)
|
| 54 |
+
feats = self.addr.m_hat(slots).reshape(*lead, self.n_slots * 4)
|
| 55 |
+
delta = torch.sigmoid(self.gate) * self.consume(feats)
|
| 56 |
+
if self.telemetry:
|
| 57 |
+
with torch.no_grad():
|
| 58 |
+
self.last_delta_ratio = (
|
| 59 |
+
delta.norm() / x.norm().clamp_min(1e-12)).item()
|
| 60 |
+
return x + delta
|
| 61 |
+
|
| 62 |
+
@classmethod
|
| 63 |
+
def from_state_dict(cls, sd: dict) -> "RelayPatch2D":
|
| 64 |
+
"""Rebuild dims FROM the state dict, then strict-load."""
|
| 65 |
+
d = sd["proj.weight"].shape[1]
|
| 66 |
+
n_slots = sd["proj.weight"].shape[0] // 4
|
| 67 |
+
hidden = sd["consume.0.weight"].shape[0]
|
| 68 |
+
K = sd["addr.codebook"].shape[0]
|
| 69 |
+
m = cls(d, n_slots=n_slots, K=K, hidden=hidden)
|
| 70 |
+
m.load_state_dict(sd, strict=True)
|
| 71 |
+
return m
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
class BlockWithRelay(nn.Module):
|
| 75 |
+
"""Wrap one trunk block; the relay reads its hidden-states output.
|
| 76 |
+
Tuple outputs pass through (hybrid-safe, as amoe.core BlockWithAdapter).
|
| 77 |
+
Carries a `w_bands` slot for wrap-uniformity with multiband sites —
|
| 78 |
+
the relay ignores it."""
|
| 79 |
+
|
| 80 |
+
def __init__(self, block: nn.Module, relay: nn.Module):
|
| 81 |
+
super().__init__()
|
| 82 |
+
self.block = block
|
| 83 |
+
self.mod = relay
|
| 84 |
+
self.w_bands = None # unused by relays; uniform wrap API
|
| 85 |
+
|
| 86 |
+
def forward(self, *args, **kwargs):
|
| 87 |
+
out = self.block(*args, **kwargs)
|
| 88 |
+
if isinstance(out, tuple):
|
| 89 |
+
return (self.mod(out[0]),) + out[1:]
|
| 90 |
+
return self.mod(out)
|
src/amoe/diffusion/laws.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""amoe.diffusion.laws — the diffusion-line laws, encoded with their receipts.
|
| 2 |
+
|
| 3 |
+
Every rule was paid for on the r2 rented card; the receipts live in the
|
| 4 |
+
research record (https://huggingface.co/AbstractPhil/geolip-aleph-diffusion).
|
| 5 |
+
|
| 6 |
+
1. DTYPE LAW (supersedes amoe's fp32 default ON DIFFUSION TRUNKS) — adapters
|
| 7 |
+
attach at the DECLARED trunk dtype, never a sniffed one: fp32 adapters on
|
| 8 |
+
a bf16 trunk spin fp32 noise into the environment, and first-param
|
| 9 |
+
sniffing can read a still-meta fp32 tensor (the R0b integration-gate
|
| 10 |
+
catch, diffusion-pipe fork f4f1c46). Judged gauges stay fp32.
|
| 11 |
+
2. STRUCTURAL BANDING, NOT COMPARATIVE ROUTING — the certified partition of
|
| 12 |
+
the sigma axis is cosine crossfade windows (positional, dense,
|
| 13 |
+
differentiable; no selection event). Comparative routing on diffusion was
|
| 14 |
+
falsified 2-seed: state+sigma dispatch keys found nothing to route on
|
| 15 |
+
(exp007), and address-as-key died against the repeated-key null in both
|
| 16 |
+
raw-flattened and M-hat-bottlenecked forms (exp014/exp015).
|
| 17 |
+
3. CONDITIONING LAW — structural/blob supervision pays where x0 recovery is
|
| 18 |
+
LINEAR (flow: x0 = x_t − σ·v; ~125–200× the eps effect, 2-seed) and is
|
| 19 |
+
inert on eps, which divides by a vanishing sqrt(alpha_bar) exactly in
|
| 20 |
+
the supervised band (exp012 vs exp013). blob-on-eps therefore refuses
|
| 21 |
+
unless forced. Operating point λ≈1 (3-point dose curve, exp013).
|
| 22 |
+
4. ZERO-INIT W **AND** B — the exp006 bias-leak law: output weight and bias
|
| 23 |
+
both zero so a fresh adapter is EXACTLY inert (P-INIT bit-exact).
|
| 24 |
+
5. TOGGLE LAW — disabled adapters return x via CODE-PATH SKIP (`return x`,
|
| 25 |
+
not a zero-mul), bit-exact at every dtype. Held post-train in every
|
| 26 |
+
experiment of the line.
|
| 27 |
+
6. PURE ADAM, NEVER ADAMW — inherited program-wide (amoe.laws law 1).
|
| 28 |
+
"""
|
| 29 |
+
from __future__ import annotations
|
| 30 |
+
|
| 31 |
+
from ..laws import make_optimizer, pin_precision # noqa: F401 (the constructors)
|
| 32 |
+
|
| 33 |
+
# The certified band constants (exp008; edges/xfade are LAW, not config):
|
| 34 |
+
N_BANDS = 3
|
| 35 |
+
BAND_EDGES = (0.35, 0.75) # LOW [0,.35] | MID (.35,.75] | HIGH (.75,1]
|
| 36 |
+
XFADE = 0.06 # cosine crossfade half-width
|
| 37 |
+
BAND_ROLES = ("fidelity/detail (LOW noise)",
|
| 38 |
+
"continuity/semantics (MID)",
|
| 39 |
+
"diversity/structure (HIGH noise)")
|
| 40 |
+
|
| 41 |
+
BLOB_LAMBDA = 1.0 # the dose-curve operating point (exp013, 3 points)
|
| 42 |
+
FLOW_SHIFT = 2.5 # the trained sigma warp of the flow substrate
|
| 43 |
+
|
| 44 |
+
# The negatives record (why align()/dispatch is not a diffusion verb in 0.2):
|
| 45 |
+
ROUTING_NEGATIVES = (
|
| 46 |
+
"exp007: A=4 bank with dense signed state+sigma dispatch ties/loses to a "
|
| 47 |
+
"matched monolith under uniform pressure; sigma-band usage flat.",
|
| 48 |
+
"exp014: text/address dispatch keys within 0.07% of the state-key control; "
|
| 49 |
+
"raw-flattened [32,128] address KILLS routing (disease geometry).",
|
| 50 |
+
"exp015: M-hat slot-bottlenecked address key — routing excess 2.5e-06 over "
|
| 51 |
+
"the REPEATED-key null, match advantage ~0, both seeds. The frozen "
|
| 52 |
+
"address-as-key form is CLOSED; the open form is TE+AMoE joint training.",
|
| 53 |
+
)
|
src/amoe/diffusion/runtime/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .attach import DiffusionAttachHandle, attach, detach
|
| 2 |
+
from .sampler import StepGatedSampler, sample
|
src/amoe/diffusion/runtime/attach.py
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""attach / DiffusionAttachHandle / detach — the diffusion runtime verbs.
|
| 2 |
+
|
| 3 |
+
attach(model, anchor) wraps every bound site (RelayPatch2D or
|
| 4 |
+
MultibandDelta by anchor kind) at the DECLARED trunk dtype (dtype law).
|
| 5 |
+
detach(verify=True) restores the retained modules and asserts BIT-EXACT
|
| 6 |
+
equality with a pre-attach probe where the binding provides one
|
| 7 |
+
(UNet bindings do; the DiT binding defers to the diffusion-pipe gates —
|
| 8 |
+
documented F4 — and degrades to structural restore with a warning).
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import contextlib
|
| 13 |
+
import warnings
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn as nn
|
| 17 |
+
|
| 18 |
+
from ...binding.diffusion import resolve_diffusion
|
| 19 |
+
from ...io.checkpoint import DiffusionAnchorCheckpoint, load_diffusion_anchor
|
| 20 |
+
from ..core.multiband import BandBlockWrap, MultibandDelta, band_weights
|
| 21 |
+
from ..core.relay import BlockWithRelay, RelayPatch2D
|
| 22 |
+
from ..laws import N_BANDS
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class DiffusionAttachHandle:
|
| 26 |
+
def __init__(self, model, binding, originals, names, wraps, fingerprint):
|
| 27 |
+
self.model = model
|
| 28 |
+
self.binding = binding
|
| 29 |
+
self._originals = originals # [(qualified_name, module)]
|
| 30 |
+
self.names = list(names)
|
| 31 |
+
self._wraps = wraps # BlockWithRelay | BandBlockWrap
|
| 32 |
+
self._fingerprint = fingerprint
|
| 33 |
+
|
| 34 |
+
@property
|
| 35 |
+
def kind(self) -> str:
|
| 36 |
+
return ("multiband3"
|
| 37 |
+
if isinstance(self._wraps[0].mod, MultibandDelta) else "relay")
|
| 38 |
+
|
| 39 |
+
def modules(self):
|
| 40 |
+
return [w.mod for w in self._wraps]
|
| 41 |
+
|
| 42 |
+
# -- masking / toggling (never renormalizes anything) ---------------
|
| 43 |
+
def set_mask(self, mask: dict[str, bool]) -> None:
|
| 44 |
+
for n, w in zip(self.names, self._wraps):
|
| 45 |
+
w.mod.enabled = mask.get(n, True)
|
| 46 |
+
|
| 47 |
+
@contextlib.contextmanager
|
| 48 |
+
def only(self, *names):
|
| 49 |
+
prev = [w.mod.enabled for w in self._wraps]
|
| 50 |
+
self.set_mask({n: n in names for n in self.names})
|
| 51 |
+
try:
|
| 52 |
+
yield self
|
| 53 |
+
finally:
|
| 54 |
+
for w, e in zip(self._wraps, prev):
|
| 55 |
+
w.mod.enabled = e
|
| 56 |
+
|
| 57 |
+
@contextlib.contextmanager
|
| 58 |
+
def all_off(self):
|
| 59 |
+
prev = [w.mod.enabled for w in self._wraps]
|
| 60 |
+
self.set_mask({n: False for n in self.names})
|
| 61 |
+
try:
|
| 62 |
+
yield self
|
| 63 |
+
finally:
|
| 64 |
+
for w, e in zip(self._wraps, prev):
|
| 65 |
+
w.mod.enabled = e
|
| 66 |
+
|
| 67 |
+
@contextlib.contextmanager
|
| 68 |
+
def lesion_band(self, *bands: int):
|
| 69 |
+
"""Disable band expert(s) at every site (the exp008/exp010 lesion
|
| 70 |
+
instrument). Multiband anchors only."""
|
| 71 |
+
if self.kind != "multiband3":
|
| 72 |
+
raise TypeError("lesion_band is a multiband instrument; this "
|
| 73 |
+
f"anchor is kind={self.kind}")
|
| 74 |
+
bad = [b for b in bands if not 0 <= b < N_BANDS]
|
| 75 |
+
if bad:
|
| 76 |
+
raise ValueError(f"bands out of range: {bad}")
|
| 77 |
+
prev = [list(w.mod.band_enabled) for w in self._wraps]
|
| 78 |
+
for w in self._wraps:
|
| 79 |
+
for b in bands:
|
| 80 |
+
w.mod.band_enabled[b] = False
|
| 81 |
+
try:
|
| 82 |
+
yield self
|
| 83 |
+
finally:
|
| 84 |
+
for w, p in zip(self._wraps, prev):
|
| 85 |
+
w.mod.band_enabled = p
|
| 86 |
+
|
| 87 |
+
# -- band windows (the step gate) ------------------------------------
|
| 88 |
+
def set_band_windows(self, s01: torch.Tensor) -> None:
|
| 89 |
+
"""s01: (B,) noise levels in [0,1] -> cosine crossfade windows on
|
| 90 |
+
every wrap. No-op for relay anchors (windows unused)."""
|
| 91 |
+
w = band_weights(s01)
|
| 92 |
+
for wr in self._wraps:
|
| 93 |
+
wr.w_bands = w
|
| 94 |
+
|
| 95 |
+
# -- telemetry --------------------------------------------------------
|
| 96 |
+
def telemetry(self, on: bool) -> None:
|
| 97 |
+
for w in self._wraps:
|
| 98 |
+
if hasattr(w.mod, "telemetry"):
|
| 99 |
+
w.mod.telemetry = on
|
| 100 |
+
|
| 101 |
+
def amplitude(self) -> dict[str, float]:
|
| 102 |
+
out = {}
|
| 103 |
+
for n, w in zip(self.names, self._wraps):
|
| 104 |
+
r = getattr(w.mod, "last_delta_ratio", None)
|
| 105 |
+
if r is not None:
|
| 106 |
+
out[n] = round(float(r), 6)
|
| 107 |
+
return out
|
| 108 |
+
|
| 109 |
+
def gates(self) -> dict:
|
| 110 |
+
vals = []
|
| 111 |
+
for w in self._wraps:
|
| 112 |
+
m = w.mod
|
| 113 |
+
if hasattr(m, "gate"):
|
| 114 |
+
vals.append(torch.sigmoid(m.gate).item())
|
| 115 |
+
elif hasattr(m, "gates"):
|
| 116 |
+
vals.extend(torch.sigmoid(m.gates).tolist())
|
| 117 |
+
return {"gate_mean": round(sum(vals) / max(len(vals), 1), 5),
|
| 118 |
+
"gate_min": round(min(vals), 5) if vals else None,
|
| 119 |
+
"gate_max": round(max(vals), 5) if vals else None}
|
| 120 |
+
|
| 121 |
+
# -- detach -----------------------------------------------------------
|
| 122 |
+
def detach(self, *, verify: bool = True) -> nn.Module:
|
| 123 |
+
for name, mod in self._originals:
|
| 124 |
+
self.binding.replace(self.model, name, mod)
|
| 125 |
+
if verify:
|
| 126 |
+
if self._fingerprint is None:
|
| 127 |
+
warnings.warn(
|
| 128 |
+
"detach: no pre-attach fingerprint for this binding "
|
| 129 |
+
"(probe deferred — F4); structural restore only. The "
|
| 130 |
+
"diffusion-pipe path carries its own parity gates.")
|
| 131 |
+
else:
|
| 132 |
+
post = self.binding.probe(self.model)
|
| 133 |
+
if not torch.equal(post, self._fingerprint):
|
| 134 |
+
raise RuntimeError(
|
| 135 |
+
"detach verification FAILED: post-detach output is "
|
| 136 |
+
"not bit-exact with the pre-attach fingerprint")
|
| 137 |
+
return self.model
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def attach(model, anchor, *, binding=None, dtype: "torch.dtype | None" = None,
|
| 141 |
+
strict: bool = True) -> DiffusionAttachHandle:
|
| 142 |
+
b = resolve_diffusion(model, binding)
|
| 143 |
+
sites = b.sites(model)
|
| 144 |
+
|
| 145 |
+
ck = (anchor if isinstance(anchor, DiffusionAnchorCheckpoint)
|
| 146 |
+
else load_diffusion_anchor(anchor))
|
| 147 |
+
kind = ck.kind
|
| 148 |
+
if kind not in ("relay", "multiband3"):
|
| 149 |
+
raise ValueError(
|
| 150 |
+
f"anchor kind '{kind}' is not attachable — 'mono'/'bank' are "
|
| 151 |
+
"research-record controls (monolith falsifier / falsified "
|
| 152 |
+
"dispatch bank), loadable for analysis via "
|
| 153 |
+
"load_diffusion_anchor only. See "
|
| 154 |
+
"amoe.diffusion.laws.ROUTING_NEGATIVES.")
|
| 155 |
+
if ck.n_sites != len(sites):
|
| 156 |
+
raise ValueError(
|
| 157 |
+
f"anchor carries {ck.n_sites} sites, binding '{b.name}' "
|
| 158 |
+
f"enumerates {len(sites)} — wrong substrate?")
|
| 159 |
+
if strict:
|
| 160 |
+
fam = ck.meta.get("substrate", {}).get("family")
|
| 161 |
+
if fam and fam not in b.name:
|
| 162 |
+
raise ValueError(
|
| 163 |
+
f"anchor substrate family '{fam}' vs binding '{b.name}' "
|
| 164 |
+
"(pass strict=False to override)")
|
| 165 |
+
|
| 166 |
+
dt = dtype if dtype is not None else b.declared_dtype(model) # dtype law
|
| 167 |
+
fingerprint = b.probe(model)
|
| 168 |
+
|
| 169 |
+
originals, wraps, names = [], [], []
|
| 170 |
+
for i, (name, block, d) in enumerate(sites):
|
| 171 |
+
sd = ck.per_site(i)
|
| 172 |
+
if kind == "relay":
|
| 173 |
+
mod = RelayPatch2D.from_state_dict(sd)
|
| 174 |
+
assert mod.d == d, (f"site {i} ({name}): anchor width {mod.d} "
|
| 175 |
+
f"!= trunk width {d}")
|
| 176 |
+
wrap = BlockWithRelay(block, mod)
|
| 177 |
+
else:
|
| 178 |
+
mod = MultibandDelta.from_state_dict(sd)
|
| 179 |
+
assert mod.d == d, (f"site {i} ({name}): anchor width {mod.d} "
|
| 180 |
+
f"!= trunk width {d}")
|
| 181 |
+
wrap = BandBlockWrap(block, mod)
|
| 182 |
+
p0 = next(block.parameters(), None)
|
| 183 |
+
dev = p0.device if p0 is not None else torch.device("cpu")
|
| 184 |
+
mod.to(device=dev, dtype=dt)
|
| 185 |
+
b.replace(model, name, wrap)
|
| 186 |
+
originals.append((name, block))
|
| 187 |
+
wraps.append(wrap)
|
| 188 |
+
names.append(name)
|
| 189 |
+
return DiffusionAttachHandle(model, b, originals, names, wraps,
|
| 190 |
+
fingerprint)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def detach(handle: DiffusionAttachHandle, *,
|
| 194 |
+
verify: bool = True) -> nn.Module:
|
| 195 |
+
return handle.detach(verify=verify)
|
src/amoe/diffusion/runtime/sampler.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""StepGatedSampler — the controller (exp010): per-step band gating at
|
| 2 |
+
inference. Each denoising step sets the crossfade windows from the
|
| 3 |
+
CURRENT noise level, activating band experts coarse-to-fine across the
|
| 4 |
+
trajectory exactly as trained. Image space sees what eps-MSE could not:
|
| 5 |
+
on the certified mb3 stack the controller moved grounding +0.089 over
|
| 6 |
+
frozen, with monotone band-lesion signatures (exp010, image-space).
|
| 7 |
+
|
| 8 |
+
Two objectives:
|
| 9 |
+
"eps" — DDIM on the stock SD15-style schedule (the exp010-proven path);
|
| 10 |
+
"flow" — Euler on the SHIFT-warped sigma grid (the trained warp of the
|
| 11 |
+
flow substrate, exp013).
|
| 12 |
+
|
| 13 |
+
Works for relay anchors too — the windows are simply unused.
|
| 14 |
+
Uncond convention follows the certified beds: ZEROED conditioning (the
|
| 15 |
+
CFG-dropout convention the adapters were trained with), not an
|
| 16 |
+
empty-prompt embedding.
|
| 17 |
+
"""
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import torch
|
| 21 |
+
|
| 22 |
+
from ..laws import FLOW_SHIFT
|
| 23 |
+
from .attach import DiffusionAttachHandle
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _default_ddim():
|
| 27 |
+
from diffusers import DDIMScheduler
|
| 28 |
+
# The stock SD15 schedule, constructed explicitly (no download):
|
| 29 |
+
return DDIMScheduler(num_train_timesteps=1000, beta_start=0.00085,
|
| 30 |
+
beta_end=0.012, beta_schedule="scaled_linear",
|
| 31 |
+
clip_sample=False, set_alpha_to_one=False,
|
| 32 |
+
steps_offset=1, prediction_type="epsilon")
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class StepGatedSampler:
|
| 36 |
+
def __init__(self, model, handle: DiffusionAttachHandle, *,
|
| 37 |
+
scheduler=None, objective: str = "eps",
|
| 38 |
+
shift: float = FLOW_SHIFT, device=None,
|
| 39 |
+
latent_channels: int = 4):
|
| 40 |
+
assert objective in ("eps", "flow"), objective
|
| 41 |
+
self.model, self.handle = model, handle
|
| 42 |
+
self.objective, self.shift = objective, shift
|
| 43 |
+
self.latent_channels = latent_channels
|
| 44 |
+
self.device = device or next(model.parameters()).device
|
| 45 |
+
self.sched = None
|
| 46 |
+
if objective == "eps":
|
| 47 |
+
self.sched = scheduler or _default_ddim()
|
| 48 |
+
|
| 49 |
+
@torch.no_grad()
|
| 50 |
+
def sample_latents(self, ehs_cond: torch.Tensor,
|
| 51 |
+
ehs_uncond: "torch.Tensor | None" = None, *,
|
| 52 |
+
seed: int, steps: int = 30, guidance: float = 7.5,
|
| 53 |
+
latent_size: tuple = (64, 64)) -> torch.Tensor:
|
| 54 |
+
B = ehs_cond.shape[0]
|
| 55 |
+
dev = self.device
|
| 56 |
+
dt = ehs_cond.dtype
|
| 57 |
+
g = torch.Generator(device=dev).manual_seed(seed)
|
| 58 |
+
x = torch.randn(B, self.latent_channels, *latent_size,
|
| 59 |
+
generator=g, device=dev, dtype=dt)
|
| 60 |
+
unc = (torch.zeros_like(ehs_cond) if ehs_uncond is None
|
| 61 |
+
else ehs_uncond)
|
| 62 |
+
ehs_in = torch.cat([ehs_cond, unc], dim=0)
|
| 63 |
+
|
| 64 |
+
if self.objective == "eps":
|
| 65 |
+
self.sched.set_timesteps(steps, device=dev)
|
| 66 |
+
x = x * self.sched.init_noise_sigma
|
| 67 |
+
for t in self.sched.timesteps:
|
| 68 |
+
s01 = torch.full((2 * B,), float(t) / 1000.0, device=dev)
|
| 69 |
+
self.handle.set_band_windows(s01) # the step gate
|
| 70 |
+
xin = self.sched.scale_model_input(torch.cat([x, x]), t)
|
| 71 |
+
pred = self.model(xin, t, ehs_in, return_dict=False)[0]
|
| 72 |
+
p_c, p_u = pred.chunk(2)
|
| 73 |
+
pred = p_u + guidance * (p_c - p_u)
|
| 74 |
+
x = self.sched.step(pred, t, x).prev_sample
|
| 75 |
+
return x
|
| 76 |
+
|
| 77 |
+
# flow: Euler from s=1 -> 0 on the SHIFT-warped grid (exp013 warp)
|
| 78 |
+
u = torch.linspace(1.0, 0.0, steps + 1, device=dev)
|
| 79 |
+
s_grid = (self.shift * u) / (1 + (self.shift - 1) * u)
|
| 80 |
+
for i in range(steps):
|
| 81 |
+
s, s_next = s_grid[i], s_grid[i + 1]
|
| 82 |
+
s01 = torch.full((2 * B,), float(s), device=dev)
|
| 83 |
+
self.handle.set_band_windows(s01) # the step gate
|
| 84 |
+
v = self.model(torch.cat([x, x]), s01[:1].expand(2 * B) * 1000,
|
| 85 |
+
ehs_in, return_dict=False)[0]
|
| 86 |
+
v_c, v_u = v.chunk(2)
|
| 87 |
+
v = v_u + guidance * (v_c - v_u)
|
| 88 |
+
x = x + (s_next - s) * v # dx/ds = v
|
| 89 |
+
return x
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
@torch.no_grad()
|
| 93 |
+
def sample(pipe, handle: DiffusionAttachHandle, prompt: str, *,
|
| 94 |
+
negative_prompt: "str | None" = None, seed: int = 0,
|
| 95 |
+
steps: int = 30, guidance: float = 7.5, objective: str = "eps"):
|
| 96 |
+
"""Ease-of-use path for a diffusers StableDiffusionPipeline whose UNet
|
| 97 |
+
carries an attached anchor: encode -> step-gated sample -> decode."""
|
| 98 |
+
dev = pipe.device if pipe.device.type != "meta" else "cuda"
|
| 99 |
+
tok = pipe.tokenizer(prompt, padding="max_length", truncation=True,
|
| 100 |
+
max_length=pipe.tokenizer.model_max_length,
|
| 101 |
+
return_tensors="pt")
|
| 102 |
+
ehs = pipe.text_encoder(tok.input_ids.to(dev))[0]
|
| 103 |
+
unc = None
|
| 104 |
+
if negative_prompt is not None:
|
| 105 |
+
ntok = pipe.tokenizer(negative_prompt, padding="max_length",
|
| 106 |
+
truncation=True,
|
| 107 |
+
max_length=pipe.tokenizer.model_max_length,
|
| 108 |
+
return_tensors="pt")
|
| 109 |
+
unc = pipe.text_encoder(ntok.input_ids.to(dev))[0]
|
| 110 |
+
sz = pipe.unet.config.sample_size
|
| 111 |
+
sampler = StepGatedSampler(pipe.unet, handle, scheduler=None,
|
| 112 |
+
objective=objective, device=dev)
|
| 113 |
+
lat = sampler.sample_latents(ehs, unc, seed=seed, steps=steps,
|
| 114 |
+
guidance=guidance, latent_size=(sz, sz))
|
| 115 |
+
img = pipe.vae.decode(lat / pipe.vae.config.scaling_factor).sample
|
| 116 |
+
img = (img / 2 + 0.5).clamp(0, 1)
|
| 117 |
+
from PIL import Image
|
| 118 |
+
import numpy as np
|
| 119 |
+
arr = (img[0].permute(1, 2, 0).float().cpu().numpy() * 255).astype("uint8")
|
| 120 |
+
return Image.fromarray(np.asarray(arr))
|
src/amoe/diffusion/train/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .aligner import align
|
| 2 |
+
from .config import DiffusionTrainConfig, RelaySpec
|
| 3 |
+
from .trainer import ConditioningLawWarning, train
|
| 4 |
+
from . import data, objectives
|
src/amoe/diffusion/train/aligner.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""align() — a grounded negative on diffusion, kept for API symmetry.
|
| 2 |
+
|
| 3 |
+
The text line's align() trains dispatch keys over frozen anchors. On
|
| 4 |
+
diffusion that entire family was falsified at 2 seeds — see
|
| 5 |
+
amoe.diffusion.laws.ROUTING_NEGATIVES (exp007: state+sigma keys find
|
| 6 |
+
nothing to route on; exp014/exp015: address-as-key dead against the
|
| 7 |
+
repeated-key null in both raw and M-hat-bottlenecked forms).
|
| 8 |
+
|
| 9 |
+
The certified replacement for comparative routing on the sigma axis is
|
| 10 |
+
STRUCTURAL banding (adapter='multiband3' in train(); exp008 lesions
|
| 11 |
+
surgical 3/3 both seeds). The open research form — TE+AMoE joint
|
| 12 |
+
training, where the text encoder adapter learns WITH the dispatch — is
|
| 13 |
+
queued science (history/open_questions.md), not shippable API.
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
from ..laws import ROUTING_NEGATIVES
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def align(*args, **kwargs):
|
| 21 |
+
raise NotImplementedError(
|
| 22 |
+
"amoe.diffusion.align: comparative dispatch on diffusion is a "
|
| 23 |
+
"2-seed falsified line — there is nothing honest to align here. "
|
| 24 |
+
"Use train(adapter='multiband3') for the certified structural "
|
| 25 |
+
"banding, or follow the open TE+AMoE joint-training question. "
|
| 26 |
+
"The record:\n - " + "\n - ".join(ROUTING_NEGATIVES))
|
src/amoe/diffusion/train/config.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""DiffusionTrainConfig — the certified recipes as the only knobs.
|
| 2 |
+
|
| 3 |
+
Deliberately ABSENT (laws, not options): optimizer choice (pure Adam,
|
| 4 |
+
wd=0), top-k/load-balancing (no comparative selectors), band edges/xfade
|
| 5 |
+
(law constants, exp008).
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from dataclasses import dataclass, field
|
| 10 |
+
|
| 11 |
+
from ..laws import BLOB_LAMBDA, FLOW_SHIFT
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@dataclass
|
| 15 |
+
class RelaySpec:
|
| 16 |
+
n_slots: int = 16
|
| 17 |
+
K: int = 64
|
| 18 |
+
tau: float = 0.1
|
| 19 |
+
hidden: int = 178
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass
|
| 23 |
+
class DiffusionTrainConfig:
|
| 24 |
+
name: str = "anchor"
|
| 25 |
+
objective: str = "eps" # "eps" | "flow"
|
| 26 |
+
shift: float = FLOW_SHIFT # flow sigma warp (exp013)
|
| 27 |
+
adapter: str = "relay" # "relay" | "multiband3"
|
| 28 |
+
relay: RelaySpec = field(default_factory=RelaySpec)
|
| 29 |
+
rank: int = 16 # multiband3 per-band rank
|
| 30 |
+
band_roles: bool = False # exp009 HP/LP role pressure (measured
|
| 31 |
+
# inert at gauge scale — kept honest)
|
| 32 |
+
blob: bool = False # foreground-LP-x0 supervision
|
| 33 |
+
blob_lambda: float = BLOB_LAMBDA # λ≈1: the 3-point dose-curve operating
|
| 34 |
+
# point (exp013); 0.5/2.0 are the
|
| 35 |
+
# mapped under/over-couple arms
|
| 36 |
+
force_blob_on_eps: bool = False # conditioning law: eps refuses blob
|
| 37 |
+
# unless forced (exp012, 2-seed inert)
|
| 38 |
+
steps: int = 3000
|
| 39 |
+
batch_size: int = 16
|
| 40 |
+
lr: float = 1e-3
|
| 41 |
+
cfg_dropout: float = 0.1
|
| 42 |
+
seed: int = 0
|
| 43 |
+
log_every: int = 50
|
| 44 |
+
base_schedule_id: str = "stable-diffusion-v1-5/stable-diffusion-v1-5"
|
src/amoe/diffusion/train/data.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""build_cache — images + captions -> the trainer's cache dict.
|
| 2 |
+
|
| 3 |
+
The trainer is cache-fed (the pod-bed discipline): latents and text
|
| 4 |
+
features are encoded ONCE, then training never touches the VAE/TE again.
|
| 5 |
+
The campaign's caches load directly too — any dict with lat/ehs
|
| 6 |
+
(+ optional blob, val_lat/val_ehs) works.
|
| 7 |
+
|
| 8 |
+
Blob targets: the campaign rasterized entity polygons from the fused
|
| 9 |
+
dataset (research record, exp011). The generic path here accepts a
|
| 10 |
+
directory of binary masks (PNG, same stem as each image), resized to the
|
| 11 |
+
latent grid. Empty masks are counted and reported LOUDLY (the exp011
|
| 12 |
+
schema lesson: an empty blob target is a silent zero).
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
import torch
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@torch.no_grad()
|
| 22 |
+
def build_cache(image_dir, pipe, *, captions=None, n_val: int = 256,
|
| 23 |
+
size: int = 512, blob_masks=None, device: str = "cuda",
|
| 24 |
+
batch: int = 8) -> dict:
|
| 25 |
+
"""image_dir: folder of images. pipe: a diffusers pipeline carrying
|
| 26 |
+
vae/text_encoder/tokenizer. captions: None | txt file (one caption per
|
| 27 |
+
image, sorted order) | dict stem->caption. blob_masks: None | folder
|
| 28 |
+
of binary PNG masks (same stems)."""
|
| 29 |
+
from PIL import Image
|
| 30 |
+
import numpy as np
|
| 31 |
+
|
| 32 |
+
paths = sorted(p for p in Path(image_dir).iterdir()
|
| 33 |
+
if p.suffix.lower() in (".png", ".jpg", ".jpeg", ".webp"))
|
| 34 |
+
assert paths, f"no images under {image_dir}"
|
| 35 |
+
if isinstance(captions, (str, Path)):
|
| 36 |
+
lines = Path(captions).read_text(encoding="utf-8").splitlines()
|
| 37 |
+
assert len(lines) >= len(paths), \
|
| 38 |
+
f"{len(lines)} captions for {len(paths)} images"
|
| 39 |
+
cap_of = {p.stem: lines[i] for i, p in enumerate(paths)}
|
| 40 |
+
elif isinstance(captions, dict):
|
| 41 |
+
cap_of = captions
|
| 42 |
+
else:
|
| 43 |
+
cap_of = {p.stem: "" for p in paths}
|
| 44 |
+
|
| 45 |
+
vae, te, tok = pipe.vae, pipe.text_encoder, pipe.tokenizer
|
| 46 |
+
sf = vae.config.scaling_factor
|
| 47 |
+
lat_sz = size // (2 ** (len(vae.config.block_out_channels) - 1))
|
| 48 |
+
|
| 49 |
+
lats, ehss, blobs, empty = [], [], [], 0
|
| 50 |
+
for i in range(0, len(paths), batch):
|
| 51 |
+
chunk = paths[i:i + batch]
|
| 52 |
+
imgs = []
|
| 53 |
+
for p in chunk:
|
| 54 |
+
im = Image.open(p).convert("RGB").resize((size, size))
|
| 55 |
+
imgs.append(torch.from_numpy(
|
| 56 |
+
np.asarray(im)).float().permute(2, 0, 1) / 127.5 - 1)
|
| 57 |
+
x = torch.stack(imgs).to(device)
|
| 58 |
+
lats.append((vae.encode(x).latent_dist.sample(
|
| 59 |
+
generator=torch.Generator(device=device).manual_seed(1400 + i))
|
| 60 |
+
* sf).cpu())
|
| 61 |
+
t = tok([cap_of.get(p.stem, "") for p in chunk],
|
| 62 |
+
padding="max_length", truncation=True,
|
| 63 |
+
max_length=tok.model_max_length, return_tensors="pt")
|
| 64 |
+
ehss.append(te(t.input_ids.to(device))[0].cpu())
|
| 65 |
+
if blob_masks is not None:
|
| 66 |
+
for p in chunk:
|
| 67 |
+
mp = Path(blob_masks) / f"{p.stem}.png"
|
| 68 |
+
if mp.exists():
|
| 69 |
+
m = Image.open(mp).convert("L").resize(
|
| 70 |
+
(lat_sz, lat_sz), Image.NEAREST)
|
| 71 |
+
mm = (torch.from_numpy(np.asarray(m)) > 127).float()
|
| 72 |
+
else:
|
| 73 |
+
mm = torch.zeros(lat_sz, lat_sz)
|
| 74 |
+
if mm.sum() == 0:
|
| 75 |
+
empty += 1
|
| 76 |
+
blobs.append(mm)
|
| 77 |
+
|
| 78 |
+
lat = torch.cat(lats)
|
| 79 |
+
ehs = torch.cat(ehss)
|
| 80 |
+
cache = {"lat": lat[:-n_val] if n_val else lat,
|
| 81 |
+
"ehs": ehs[:-n_val] if n_val else ehs}
|
| 82 |
+
if n_val:
|
| 83 |
+
cache["val_lat"] = lat[-n_val:]
|
| 84 |
+
cache["val_ehs"] = ehs[-n_val:]
|
| 85 |
+
if blob_masks is not None:
|
| 86 |
+
bl = torch.stack(blobs)
|
| 87 |
+
cache["blob"] = bl[:-n_val] if n_val else bl
|
| 88 |
+
if n_val:
|
| 89 |
+
cache["val_blob"] = bl[-n_val:]
|
| 90 |
+
cache["blob_empty_count"] = empty
|
| 91 |
+
if empty:
|
| 92 |
+
print(f"WARNING: {empty}/{len(paths)} blob masks are EMPTY — "
|
| 93 |
+
"empty targets train nothing and read as silent zeros "
|
| 94 |
+
"(the exp011 schema lesson)", flush=True)
|
| 95 |
+
print(f"cache: {cache['lat'].shape[0]} train rows"
|
| 96 |
+
+ (f" + {n_val} val" if n_val else ""), flush=True)
|
| 97 |
+
return cache
|
src/amoe/diffusion/train/objectives.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Objectives — the certified loss pieces, ported verbatim.
|
| 2 |
+
|
| 3 |
+
eps : stock epsilon-MSE on the shipped schedule (exp006 recipe).
|
| 4 |
+
flow : rectified-flow v-MSE with the SHIFT warp; x0 recovery is EXACT AND
|
| 5 |
+
LINEAR at all sigma (x0 = x_t − σ·v) — the mechanism behind the
|
| 6 |
+
conditioning law (exp013).
|
| 7 |
+
blob : foreground-LP-x0 coupling, weighted by the HIGH-band window
|
| 8 |
+
(exp012/exp013; pays ~125–200× more on flow than eps).
|
| 9 |
+
roles: HP/LP frequency-reweighted band pressure (exp009 — measured
|
| 10 |
+
directional-but-negligible; shipped for completeness, honest).
|
| 11 |
+
"""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import torch
|
| 15 |
+
import torch.nn.functional as F
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
# ── frequency filters (dexp009) ─────────────────────────────────────────
|
| 19 |
+
|
| 20 |
+
def hp(x):
|
| 21 |
+
"""High-pass: x − avgpool3(x) (finer-detail component)."""
|
| 22 |
+
return x - F.avg_pool2d(x, 3, stride=1, padding=1)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def lp(x):
|
| 26 |
+
"""Low-pass: avgpool7 (coarse structure)."""
|
| 27 |
+
return F.avg_pool2d(x, 7, stride=1, padding=3)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
# ── eps path (dexp006) ──────────────────────────────────────────────────
|
| 31 |
+
|
| 32 |
+
def make_schedule(base_schedule_id: str, device):
|
| 33 |
+
"""Stock training schedule (alphas_cumprod) from the shipped
|
| 34 |
+
scheduler config."""
|
| 35 |
+
from diffusers import DDPMScheduler
|
| 36 |
+
sch = DDPMScheduler.from_pretrained(base_schedule_id,
|
| 37 |
+
subfolder="scheduler")
|
| 38 |
+
assert sch.config.prediction_type == "epsilon", \
|
| 39 |
+
sch.config.prediction_type
|
| 40 |
+
return sch.alphas_cumprod.to(device)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def add_noise(lat, noise, t, acp):
|
| 44 |
+
a = acp[t].sqrt()[:, None, None, None]
|
| 45 |
+
s = (1 - acp[t]).sqrt()[:, None, None, None]
|
| 46 |
+
return a * lat + s * noise
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# ── flow path (dexp013) ─────────────────────────────────────────────────
|
| 50 |
+
|
| 51 |
+
def warp_sigma(u: torch.Tensor, shift: float) -> torch.Tensor:
|
| 52 |
+
return (shift * u) / (1 + (shift - 1) * u)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def flow_pieces(lat, s, noise):
|
| 56 |
+
"""x_t and the v target; x0 = x_t − σ·v holds exactly."""
|
| 57 |
+
s4 = s[:, None, None, None]
|
| 58 |
+
return noise * s4 + lat * (1 - s4), noise - lat # x_t, v
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
# ── blob coupling (dexp012/013) ─────────────────────────────────────────
|
| 62 |
+
|
| 63 |
+
def blob_lp_err(x0_hat, x0, blob):
|
| 64 |
+
"""Foreground-masked LP-x0 error, per-sample (B,). blob: (B, H, W)
|
| 65 |
+
binary mask on the latent grid."""
|
| 66 |
+
d2 = (lp(x0_hat) - lp(x0)) ** 2
|
| 67 |
+
m = blob[:, None]
|
| 68 |
+
denom = m.sum(dim=(1, 2, 3)).clamp_min(1.0) * d2.shape[1]
|
| 69 |
+
return (d2 * m).sum(dim=(1, 2, 3)) / denom
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
# ── role pressure (dexp009) ─────────────────────────────────────────────
|
| 73 |
+
|
| 74 |
+
def role_losses(pred, target, lam: float = 0.5):
|
| 75 |
+
"""Per-sample (B,) losses for each band role: LOW +HP, MID std,
|
| 76 |
+
HIGH +LP."""
|
| 77 |
+
base = ((pred - target) ** 2).mean(dim=(1, 2, 3))
|
| 78 |
+
low = base + lam * ((hp(pred) - hp(target)) ** 2).mean(dim=(1, 2, 3))
|
| 79 |
+
high = base + lam * ((lp(pred) - lp(target)) ** 2).mean(dim=(1, 2, 3))
|
| 80 |
+
return low, base, high
|
src/amoe/diffusion/train/trainer.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""train() — the certified diffusion recipes as a verb.
|
| 2 |
+
|
| 3 |
+
Reference-grade transplant of the campaign recipes: exp006 (eps on the
|
| 4 |
+
stock schedule) and exp013 (flow with the SHIFT warp, optional blob
|
| 5 |
+
coupling at the λ≈1 operating point), on a frozen trunk with fresh
|
| 6 |
+
zero-init adapters, pure Adam wd=0.
|
| 7 |
+
|
| 8 |
+
DDP posture mirrors amoe.train: rank-sharded batch sampling, rank-0
|
| 9 |
+
logging, correct-by-construction — multi-GPU smoke deferred (F6). The
|
| 10 |
+
production multi-GPU path for this family is the diffusion-pipe fork
|
| 11 |
+
(DeepSpeed; proven end-to-end on Anima, exp004). No FSDP.
|
| 12 |
+
|
| 13 |
+
CONDITIONING LAW (encoded): blob supervision on objective='eps' REFUSES —
|
| 14 |
+
it was measured inert-to-negative at 2 seeds exactly where the eps
|
| 15 |
+
parameterization divides by a vanishing sqrt(alpha_bar) (exp012). Pass
|
| 16 |
+
force_blob_on_eps=True to run the control anyway (a ConditioningLawWarning
|
| 17 |
+
fires).
|
| 18 |
+
"""
|
| 19 |
+
from __future__ import annotations
|
| 20 |
+
|
| 21 |
+
import warnings
|
| 22 |
+
|
| 23 |
+
import torch
|
| 24 |
+
import torch.nn as nn
|
| 25 |
+
|
| 26 |
+
from ...binding.diffusion import resolve_diffusion
|
| 27 |
+
from ...io.checkpoint import DiffusionAnchorCheckpoint
|
| 28 |
+
from ..core.multiband import BandBlockWrap, MultibandDelta, band_weights
|
| 29 |
+
from ..core.relay import BlockWithRelay, RelayPatch2D
|
| 30 |
+
from ..laws import make_optimizer, pin_precision
|
| 31 |
+
from .config import DiffusionTrainConfig
|
| 32 |
+
from .objectives import (add_noise, blob_lp_err, flow_pieces, make_schedule,
|
| 33 |
+
role_losses, warp_sigma)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class ConditioningLawWarning(UserWarning):
|
| 37 |
+
pass
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _dist():
|
| 41 |
+
try:
|
| 42 |
+
import torch.distributed as dist
|
| 43 |
+
if dist.is_available() and dist.is_initialized():
|
| 44 |
+
return dist.get_rank(), dist.get_world_size()
|
| 45 |
+
except Exception: # noqa: BLE001
|
| 46 |
+
pass
|
| 47 |
+
return 0, 1
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def train(model, cache: dict, config: "DiffusionTrainConfig | None" = None,
|
| 51 |
+
*, binding=None, device: str = "cuda"
|
| 52 |
+
) -> DiffusionAnchorCheckpoint:
|
| 53 |
+
cfg = config or DiffusionTrainConfig()
|
| 54 |
+
assert cfg.objective in ("eps", "flow"), cfg.objective
|
| 55 |
+
assert cfg.adapter in ("relay", "multiband3"), cfg.adapter
|
| 56 |
+
if cfg.blob and cfg.objective == "eps":
|
| 57 |
+
if not cfg.force_blob_on_eps:
|
| 58 |
+
raise ValueError(
|
| 59 |
+
"conditioning law: blob supervision on the eps objective "
|
| 60 |
+
"was measured inert at 2 seeds (exp012) — the eps x0 "
|
| 61 |
+
"recovery divides by a vanishing sqrt(alpha_bar) exactly "
|
| 62 |
+
"in the supervised band. Use objective='flow' (or v-pred "
|
| 63 |
+
"when available), or set force_blob_on_eps=True to run "
|
| 64 |
+
"the control.")
|
| 65 |
+
warnings.warn("blob-on-eps forced: this reproduces the exp012 "
|
| 66 |
+
"CONTROL arm, not a paying configuration",
|
| 67 |
+
ConditioningLawWarning)
|
| 68 |
+
if cfg.blob and "blob" not in cache:
|
| 69 |
+
raise ValueError("cfg.blob=True but the cache has no 'blob' masks "
|
| 70 |
+
"(see amoe.diffusion.data.build_cache)")
|
| 71 |
+
|
| 72 |
+
rank, world = _dist()
|
| 73 |
+
pin_precision() # judged gauges stay fp32 (law)
|
| 74 |
+
b = resolve_diffusion(model, binding)
|
| 75 |
+
dt = b.declared_dtype(model) # dtype law
|
| 76 |
+
model.requires_grad_(False)
|
| 77 |
+
model.eval()
|
| 78 |
+
if hasattr(model, "enable_gradient_checkpointing"):
|
| 79 |
+
model.enable_gradient_checkpointing()
|
| 80 |
+
|
| 81 |
+
sites = b.sites(model)
|
| 82 |
+
originals, wraps, mods = [], [], nn.ModuleList()
|
| 83 |
+
for name, block, d in sites:
|
| 84 |
+
if cfg.adapter == "relay":
|
| 85 |
+
s = cfg.relay
|
| 86 |
+
m = RelayPatch2D(d, n_slots=s.n_slots, K=s.K, tau=s.tau,
|
| 87 |
+
hidden=s.hidden)
|
| 88 |
+
m.assert_zero_init()
|
| 89 |
+
wrap = BlockWithRelay(block, m)
|
| 90 |
+
else:
|
| 91 |
+
m = MultibandDelta(d, r=cfg.rank)
|
| 92 |
+
m.assert_zero_init()
|
| 93 |
+
wrap = BandBlockWrap(block, m)
|
| 94 |
+
p0 = next(block.parameters(), None)
|
| 95 |
+
m.to(device=p0.device if p0 is not None else device, dtype=dt)
|
| 96 |
+
b.replace(model, name, wrap)
|
| 97 |
+
originals.append((name, block))
|
| 98 |
+
wraps.append(wrap)
|
| 99 |
+
mods.append(m)
|
| 100 |
+
|
| 101 |
+
ddp_model = model
|
| 102 |
+
if world > 1:
|
| 103 |
+
ddp_model = torch.nn.parallel.DistributedDataParallel(
|
| 104 |
+
model, find_unused_parameters=False)
|
| 105 |
+
|
| 106 |
+
opt = make_optimizer([p for p in mods.parameters()], lr=cfg.lr)
|
| 107 |
+
acp = (make_schedule(cfg.base_schedule_id, device)
|
| 108 |
+
if cfg.objective == "eps" else None)
|
| 109 |
+
lat_all, ehs_all = cache["lat"], cache["ehs"]
|
| 110 |
+
blob_all = cache.get("blob")
|
| 111 |
+
n = lat_all.shape[0]
|
| 112 |
+
g = torch.Generator().manual_seed(cfg.seed * 1000 + rank)
|
| 113 |
+
gd = torch.Generator(device=device).manual_seed(cfg.seed * 1000 + rank)
|
| 114 |
+
|
| 115 |
+
def set_w(s01):
|
| 116 |
+
w = band_weights(s01)
|
| 117 |
+
for wr in wraps:
|
| 118 |
+
wr.w_bands = w
|
| 119 |
+
return w
|
| 120 |
+
|
| 121 |
+
for step in range(cfg.steps):
|
| 122 |
+
idx = torch.randint(rank, n, (cfg.batch_size,), generator=g)
|
| 123 |
+
idx = idx - (idx % world) + rank if world > 1 else idx
|
| 124 |
+
idx = idx.clamp(0, n - 1)
|
| 125 |
+
lat = lat_all[idx].to(device, torch.float32)
|
| 126 |
+
ehs = ehs_all[idx].to(device, torch.float32)
|
| 127 |
+
bsz = lat.shape[0]
|
| 128 |
+
drop = torch.rand(bsz, generator=gd, device=device) < cfg.cfg_dropout
|
| 129 |
+
ehs = ehs.clone()
|
| 130 |
+
ehs[drop] = 0
|
| 131 |
+
noise = torch.randn(lat.shape, generator=gd, device=device)
|
| 132 |
+
|
| 133 |
+
if cfg.objective == "eps":
|
| 134 |
+
t = torch.randint(0, 1000, (bsz,), generator=gd, device=device)
|
| 135 |
+
s01 = 1 - acp[t] # noise level proxy in [0,1]
|
| 136 |
+
w = set_w(s01) if cfg.adapter == "multiband3" else None
|
| 137 |
+
x_t = add_noise(lat, noise, t, acp)
|
| 138 |
+
pred = ddp_model(x_t.to(dt), t, ehs.to(dt),
|
| 139 |
+
return_dict=False)[0].float()
|
| 140 |
+
target = noise
|
| 141 |
+
x0_hat = None
|
| 142 |
+
else:
|
| 143 |
+
u = torch.rand(bsz, generator=gd, device=device)
|
| 144 |
+
s = warp_sigma(u, cfg.shift)
|
| 145 |
+
w = set_w(s) if cfg.adapter == "multiband3" else None
|
| 146 |
+
x_t, v = flow_pieces(lat, s, noise)
|
| 147 |
+
pred = ddp_model(x_t.to(dt), s * 1000, ehs.to(dt),
|
| 148 |
+
return_dict=False)[0].float()
|
| 149 |
+
target = v
|
| 150 |
+
x0_hat = x_t - s[:, None, None, None] * pred # EXACT linear
|
| 151 |
+
|
| 152 |
+
if cfg.band_roles and cfg.adapter == "multiband3":
|
| 153 |
+
low, base, high = role_losses(pred, target)
|
| 154 |
+
loss_vec = w[:, 0] * low + w[:, 1] * base + w[:, 2] * high
|
| 155 |
+
else:
|
| 156 |
+
loss_vec = ((pred - target) ** 2).mean(dim=(1, 2, 3))
|
| 157 |
+
if cfg.blob and x0_hat is not None:
|
| 158 |
+
blob = blob_all[idx].to(device, torch.float32)
|
| 159 |
+
wb = w[:, 2] if w is not None else torch.ones_like(loss_vec)
|
| 160 |
+
loss_vec = loss_vec + cfg.blob_lambda * wb * blob_lp_err(
|
| 161 |
+
x0_hat, lat, blob)
|
| 162 |
+
loss = loss_vec.mean()
|
| 163 |
+
opt.zero_grad(set_to_none=True)
|
| 164 |
+
loss.backward()
|
| 165 |
+
opt.step()
|
| 166 |
+
if rank == 0 and (step % cfg.log_every == 0 or
|
| 167 |
+
step == cfg.steps - 1):
|
| 168 |
+
print(f"[amoe.diffusion.train {cfg.name}] step {step} "
|
| 169 |
+
f"loss {loss.item():.5f}", flush=True)
|
| 170 |
+
|
| 171 |
+
adapters = {}
|
| 172 |
+
for i, m in enumerate(mods):
|
| 173 |
+
for k, v in m.state_dict().items():
|
| 174 |
+
adapters[f"{i}.{k}"] = v.detach().cpu()
|
| 175 |
+
meta = {
|
| 176 |
+
"name": cfg.name,
|
| 177 |
+
"substrate": {"family": b.name, "n_sites": len(sites),
|
| 178 |
+
"site_names": [s[0] for s in sites],
|
| 179 |
+
"widths": [s[2] for s in sites]},
|
| 180 |
+
"adapter": ({"kind": "relay", **vars(cfg.relay)}
|
| 181 |
+
if cfg.adapter == "relay"
|
| 182 |
+
else {"kind": "multiband3", "rank": cfg.rank}),
|
| 183 |
+
"objective": ({"kind": "eps"} if cfg.objective == "eps"
|
| 184 |
+
else {"kind": "flow", "shift": cfg.shift}),
|
| 185 |
+
"blob": ({"lambda": cfg.blob_lambda} if cfg.blob else None),
|
| 186 |
+
"dtype": str(dt).replace("torch.", ""),
|
| 187 |
+
"seed": cfg.seed,
|
| 188 |
+
"recipe": {"optimizer": "adam", "lr": cfg.lr, "weight_decay": 0.0,
|
| 189 |
+
"steps": cfg.steps, "batch": cfg.batch_size,
|
| 190 |
+
"cfg_dropout": cfg.cfg_dropout,
|
| 191 |
+
"band_roles": cfg.band_roles},
|
| 192 |
+
}
|
| 193 |
+
# restore the unwrapped trunk before returning (amoe.train contract)
|
| 194 |
+
for name, block in originals:
|
| 195 |
+
b.replace(model, name, block)
|
| 196 |
+
return DiffusionAnchorCheckpoint(adapters, meta)
|
src/amoe/io/checkpoint.py
CHANGED
|
@@ -96,6 +96,128 @@ def load_dispatch(path: str) -> DispatchCheckpoint:
|
|
| 96 |
raise ValueError(f"unrecognized dispatch checkpoint at {path}")
|
| 97 |
|
| 98 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
def import_legacy_keys(path: str, *, seed: int, d: int,
|
| 100 |
emb: int = 64) -> DispatchCheckpoint:
|
| 101 |
"""Reconstruct a full dispatch state from a keys-only legacy file
|
|
|
|
| 96 |
raise ValueError(f"unrecognized dispatch checkpoint at {path}")
|
| 97 |
|
| 98 |
|
| 99 |
+
# ── diffusion anchors (amoe 0.2) ─────────────────────────────────────────
|
| 100 |
+
|
| 101 |
+
DIFF_ANCHOR_FORMAT = "amoe.diffusion.anchor"
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
@dataclass
|
| 105 |
+
class DiffusionAnchorCheckpoint:
|
| 106 |
+
"""One diffusion adapter stack: adapters keyed "{site_index}.{param}".
|
| 107 |
+
meta.adapter.kind is 'relay' | 'multiband3' | load-only legacy kinds
|
| 108 |
+
('mono', 'bank'). Relay kind requires per-site addr.home (drift
|
| 109 |
+
gauge); multiband has no address, so the requirement is kind-aware."""
|
| 110 |
+
adapters: dict[str, torch.Tensor]
|
| 111 |
+
meta: dict[str, Any] = field(default_factory=dict)
|
| 112 |
+
|
| 113 |
+
@property
|
| 114 |
+
def kind(self) -> str:
|
| 115 |
+
return self.meta.get("adapter", {}).get("kind", "unknown")
|
| 116 |
+
|
| 117 |
+
@property
|
| 118 |
+
def n_sites(self) -> int:
|
| 119 |
+
return len({k.split(".", 1)[0] for k in self.adapters})
|
| 120 |
+
|
| 121 |
+
def per_site(self, i: int) -> dict[str, torch.Tensor]:
|
| 122 |
+
pre = f"{i}."
|
| 123 |
+
return {k[len(pre):]: v for k, v in self.adapters.items()
|
| 124 |
+
if k.startswith(pre)}
|
| 125 |
+
|
| 126 |
+
def save(self, path: str) -> str:
|
| 127 |
+
if str(path).endswith(".safetensors"):
|
| 128 |
+
from .safetensors_io import save_anchor_safetensors
|
| 129 |
+
return save_anchor_safetensors(self, path)
|
| 130 |
+
meta = dict(self.meta)
|
| 131 |
+
meta.setdefault("created", _dt.datetime.now(
|
| 132 |
+
_dt.timezone.utc).isoformat())
|
| 133 |
+
meta["content_hash"] = _content_hash(self.adapters)
|
| 134 |
+
torch.save({"format": DIFF_ANCHOR_FORMAT, "version": VERSION,
|
| 135 |
+
"meta": meta, "adapters": self.adapters}, path)
|
| 136 |
+
return meta["content_hash"]
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _sd_kind(sd: dict) -> str:
|
| 140 |
+
if any(k.endswith("addr.codebook") for k in sd):
|
| 141 |
+
return "relay"
|
| 142 |
+
if "down.0.weight" in sd:
|
| 143 |
+
return "multiband3"
|
| 144 |
+
if "down.weight" in sd:
|
| 145 |
+
return "mono"
|
| 146 |
+
if "key_proj" in sd or "keys" in sd:
|
| 147 |
+
return "bank"
|
| 148 |
+
return "unknown"
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def _flatten_stack(stack, *, kind: str, imported_from: str,
|
| 152 |
+
reconstruct_home: bool = False) -> "DiffusionAnchorCheckpoint":
|
| 153 |
+
adapters, recon = {}, False
|
| 154 |
+
for i, sd in enumerate(stack):
|
| 155 |
+
sd = dict(sd)
|
| 156 |
+
if kind == "relay" and "addr.home" not in sd:
|
| 157 |
+
if not reconstruct_home:
|
| 158 |
+
raise ValueError(
|
| 159 |
+
f"site {i} lacks addr.home (drift gauge); pass a source "
|
| 160 |
+
"that carries it or use the fork import path")
|
| 161 |
+
import torch.nn.functional as _F
|
| 162 |
+
sd["addr.home"] = _F.normalize(
|
| 163 |
+
sd["addr.codebook"].detach().float(), dim=-1)
|
| 164 |
+
recon = True
|
| 165 |
+
for k, v in sd.items():
|
| 166 |
+
adapters[f"{i}.{k}"] = v
|
| 167 |
+
meta: dict[str, Any] = {"adapter": {"kind": kind},
|
| 168 |
+
"imported_from": imported_from}
|
| 169 |
+
if recon:
|
| 170 |
+
meta["home_reconstructed"] = True # drift gauge void — never silent
|
| 171 |
+
return DiffusionAnchorCheckpoint(adapters, meta)
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def load_diffusion_anchor(path: str, *, substrate: "dict | None" = None
|
| 175 |
+
) -> DiffusionAnchorCheckpoint:
|
| 176 |
+
"""Load a diffusion anchor: the versioned format, a .safetensors
|
| 177 |
+
companion, or any of the campaign's legacy stack shapes
|
| 178 |
+
({'relays': [...]}, {'mods': [...]}, {'banks'/'monos': [...]},
|
| 179 |
+
diffusion-pipe {'relays': {idx: sd}})."""
|
| 180 |
+
if str(path).endswith(".safetensors"):
|
| 181 |
+
from .safetensors_io import load_anchor_safetensors
|
| 182 |
+
ck = load_anchor_safetensors(path)
|
| 183 |
+
else:
|
| 184 |
+
blob = torch.load(path, map_location="cpu", weights_only=True)
|
| 185 |
+
if blob.get("format") == DIFF_ANCHOR_FORMAT:
|
| 186 |
+
ck = DiffusionAnchorCheckpoint(blob["adapters"],
|
| 187 |
+
blob.get("meta", {}))
|
| 188 |
+
elif "relays" in blob:
|
| 189 |
+
r = blob["relays"]
|
| 190 |
+
if isinstance(r, dict): # diffusion-pipe fork save
|
| 191 |
+
stack = [r[k] for k in sorted(r, key=int)]
|
| 192 |
+
ck = _flatten_stack(stack, kind="relay",
|
| 193 |
+
imported_from="diffusion_pipe",
|
| 194 |
+
reconstruct_home=True)
|
| 195 |
+
else: # exp001/exp006 relay stacks
|
| 196 |
+
ck = _flatten_stack(list(r), kind="relay",
|
| 197 |
+
imported_from="legacy_relays")
|
| 198 |
+
elif "mods" in blob: # exp008/009/011/012/013 stacks
|
| 199 |
+
stack = list(blob["mods"])
|
| 200 |
+
ck = _flatten_stack(stack, kind=_sd_kind(stack[0]),
|
| 201 |
+
imported_from="legacy_mods")
|
| 202 |
+
elif "banks" in blob or "monos" in blob: # exp007 (load-only)
|
| 203 |
+
key = "banks" if "banks" in blob else "monos"
|
| 204 |
+
ck = _flatten_stack(list(blob[key]), kind=_sd_kind(blob[key][0]),
|
| 205 |
+
imported_from=f"legacy_{key}")
|
| 206 |
+
ck.meta["routing_negative"] = (
|
| 207 |
+
"exp007/014/015: comparative dispatch on diffusion is a "
|
| 208 |
+
"2-seed falsified line — load for analysis, not deployment")
|
| 209 |
+
else:
|
| 210 |
+
raise ValueError(f"unrecognized diffusion anchor at {path}")
|
| 211 |
+
if ck.kind == "relay":
|
| 212 |
+
blocks = {k.split(".", 1)[0] for k in ck.adapters}
|
| 213 |
+
for b in blocks:
|
| 214 |
+
if f"{b}.addr.home" not in ck.adapters:
|
| 215 |
+
raise ValueError(f"relay anchor site {b} lacks addr.home")
|
| 216 |
+
if substrate:
|
| 217 |
+
ck.meta.setdefault("substrate", dict(substrate))
|
| 218 |
+
return ck
|
| 219 |
+
|
| 220 |
+
|
| 221 |
def import_legacy_keys(path: str, *, seed: int, d: int,
|
| 222 |
emb: int = 64) -> DispatchCheckpoint:
|
| 223 |
"""Reconstruct a full dispatch state from a keys-only legacy file
|
src/amoe/io/convert.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""amoe-convert — .pt anchor stacks -> .safetensors companions, verified.
|
| 2 |
+
|
| 3 |
+
Converts the versioned format AND every legacy campaign shape
|
| 4 |
+
(load_diffusion_anchor handles the zoo). Each conversion round-trips the
|
| 5 |
+
result and asserts BITWISE tensor equality before reporting success.
|
| 6 |
+
|
| 7 |
+
Colab-safe usage (no argparse required):
|
| 8 |
+
from amoe.io.convert import convert_one, convert_dir
|
| 9 |
+
convert_one("mb3_s0.pt", substrate={"family": "sd15"}, objective="eps")
|
| 10 |
+
convert_dir("ckpts/", substrate={"family": "sd15"})
|
| 11 |
+
|
| 12 |
+
CLI: amoe-convert <path.pt|dir> [--substrate sd15] [--objective eps]
|
| 13 |
+
[--layout amoe|comfy] [--out OUT]
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import sys
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
import torch
|
| 21 |
+
|
| 22 |
+
from .checkpoint import load_diffusion_anchor
|
| 23 |
+
from .safetensors_io import load_anchor_safetensors, save_anchor_safetensors
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def convert_one(path, *, substrate: "dict | None" = None,
|
| 27 |
+
objective: "str | None" = None, layout: str = "amoe",
|
| 28 |
+
out: "str | None" = None) -> str:
|
| 29 |
+
path = Path(path)
|
| 30 |
+
ck = load_diffusion_anchor(str(path), substrate=substrate)
|
| 31 |
+
if objective:
|
| 32 |
+
ck.meta.setdefault("objective", {"kind": objective})
|
| 33 |
+
dst = Path(out) if out else path.with_suffix(".safetensors")
|
| 34 |
+
save_anchor_safetensors(ck, str(dst), key_layout=layout)
|
| 35 |
+
back = load_anchor_safetensors(str(dst))
|
| 36 |
+
assert set(back.adapters) == set(ck.adapters), "key set changed"
|
| 37 |
+
for k in ck.adapters:
|
| 38 |
+
a, b = ck.adapters[k], back.adapters[k]
|
| 39 |
+
assert a.dtype == b.dtype and torch.equal(a, b), \
|
| 40 |
+
f"bitwise mismatch at {k}"
|
| 41 |
+
print(f"converted {path.name} -> {dst.name} "
|
| 42 |
+
f"({len(ck.adapters)} tensors, kind={ck.kind}, verified bitwise)")
|
| 43 |
+
return str(dst)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def convert_dir(root, **kw) -> list[str]:
|
| 47 |
+
out = []
|
| 48 |
+
for p in sorted(Path(root).rglob("*.pt")):
|
| 49 |
+
try:
|
| 50 |
+
out.append(convert_one(p, **kw))
|
| 51 |
+
except Exception as e: # noqa: BLE001 — report and continue
|
| 52 |
+
print(f"SKIP {p.name}: {e}")
|
| 53 |
+
return out
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def main(argv=None):
|
| 57 |
+
args = list(argv if argv is not None else sys.argv[1:])
|
| 58 |
+
if not args:
|
| 59 |
+
print(__doc__)
|
| 60 |
+
return 1
|
| 61 |
+
kw = {}
|
| 62 |
+
target = args.pop(0)
|
| 63 |
+
while args:
|
| 64 |
+
flag = args.pop(0)
|
| 65 |
+
if flag == "--substrate":
|
| 66 |
+
kw["substrate"] = {"family": args.pop(0)}
|
| 67 |
+
elif flag == "--objective":
|
| 68 |
+
kw["objective"] = args.pop(0)
|
| 69 |
+
elif flag == "--layout":
|
| 70 |
+
kw["layout"] = args.pop(0)
|
| 71 |
+
elif flag == "--out":
|
| 72 |
+
kw["out"] = args.pop(0)
|
| 73 |
+
else:
|
| 74 |
+
print(f"unknown flag {flag}")
|
| 75 |
+
return 1
|
| 76 |
+
p = Path(target)
|
| 77 |
+
if p.is_dir():
|
| 78 |
+
kw.pop("out", None)
|
| 79 |
+
convert_dir(p, **kw)
|
| 80 |
+
else:
|
| 81 |
+
convert_one(p, **kw)
|
| 82 |
+
return 0
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
if __name__ == "__main__":
|
| 86 |
+
raise SystemExit(main())
|
src/amoe/io/safetensors_io.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""safetensors serialization for amoe anchors (new in 0.2 — the diffusion
|
| 2 |
+
line needs ComfyUI-consumable artifacts; a documented 0.1 non-goal ends).
|
| 3 |
+
|
| 4 |
+
Two key layouts:
|
| 5 |
+
"amoe" (canonical, portable): blocks.{site_index}.{param_path}
|
| 6 |
+
"comfy" (export): diffusion_model.{site_name}.aleph_relay.{param_path}
|
| 7 |
+
— the cosmos_predict2 save_adapter precedent. Requires
|
| 8 |
+
meta.substrate.site_names. The ComfyUI node consumes the
|
| 9 |
+
canonical layout and asserts the width signature (F2).
|
| 10 |
+
|
| 11 |
+
safetensors metadata is str:str only, so the full meta rides as one JSON
|
| 12 |
+
blob ("amoe_meta") plus greppable duplicates. Content-hash v2 is defined
|
| 13 |
+
over sorted(key)+shape+dtype+little-endian bytes — format-independent
|
| 14 |
+
(the .pt v1 hash depends on torch.save serialization).
|
| 15 |
+
"""
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import hashlib
|
| 19 |
+
import json
|
| 20 |
+
from typing import TYPE_CHECKING
|
| 21 |
+
|
| 22 |
+
import torch
|
| 23 |
+
|
| 24 |
+
if TYPE_CHECKING: # pragma: no cover
|
| 25 |
+
from .checkpoint import DiffusionAnchorCheckpoint
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _require_safetensors():
|
| 29 |
+
try:
|
| 30 |
+
import safetensors.torch as st # noqa: F401
|
| 31 |
+
return st
|
| 32 |
+
except ImportError as e: # pragma: no cover
|
| 33 |
+
raise ImportError(
|
| 34 |
+
"safetensors is required for this path: "
|
| 35 |
+
"pip install amoe-lora[diffusion] (or pip install safetensors)"
|
| 36 |
+
) from e
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def content_hash_v2(tensors: dict[str, torch.Tensor]) -> str:
|
| 40 |
+
h = hashlib.sha256()
|
| 41 |
+
for k in sorted(tensors):
|
| 42 |
+
t = tensors[k].detach().cpu().contiguous()
|
| 43 |
+
h.update(k.encode())
|
| 44 |
+
h.update(str(tuple(t.shape)).encode())
|
| 45 |
+
h.update(str(t.dtype).encode())
|
| 46 |
+
# byte view works for every dtype incl. bf16 (no numpy dtype needed)
|
| 47 |
+
h.update(t.flatten().view(torch.uint8).numpy().tobytes())
|
| 48 |
+
return "sha256v2:" + h.hexdigest()
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def save_anchor_safetensors(ck: "DiffusionAnchorCheckpoint", path: str,
|
| 52 |
+
*, key_layout: str = "amoe") -> str:
|
| 53 |
+
st = _require_safetensors()
|
| 54 |
+
if key_layout == "amoe":
|
| 55 |
+
tensors = {f"blocks.{k}": v.detach().cpu().contiguous()
|
| 56 |
+
for k, v in ck.adapters.items()}
|
| 57 |
+
elif key_layout == "comfy":
|
| 58 |
+
names = ck.meta.get("substrate", {}).get("site_names")
|
| 59 |
+
if not names:
|
| 60 |
+
raise ValueError(
|
| 61 |
+
"comfy layout needs meta.substrate.site_names (the trunk "
|
| 62 |
+
"module paths); save the canonical 'amoe' layout instead")
|
| 63 |
+
tensors = {}
|
| 64 |
+
for k, v in ck.adapters.items():
|
| 65 |
+
i, param = k.split(".", 1)
|
| 66 |
+
tensors[f"diffusion_model.{names[int(i)]}.aleph_relay.{param}"] = \
|
| 67 |
+
v.detach().cpu().contiguous()
|
| 68 |
+
else:
|
| 69 |
+
raise ValueError(f"unknown key_layout '{key_layout}'")
|
| 70 |
+
chash = content_hash_v2({k: v for k, v in ck.adapters.items()})
|
| 71 |
+
meta = dict(ck.meta)
|
| 72 |
+
meta["content_hash_v2"] = chash
|
| 73 |
+
md = {"format": "amoe.diffusion.anchor", "version": "1",
|
| 74 |
+
"key_layout": key_layout, "amoe_meta": json.dumps(meta),
|
| 75 |
+
"adapter_kind": ck.kind,
|
| 76 |
+
"substrate_family": str(meta.get("substrate", {}).get("family", "")),
|
| 77 |
+
"content_hash_v2": chash}
|
| 78 |
+
st.save_file(tensors, path, metadata=md)
|
| 79 |
+
return chash
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def load_anchor_safetensors(path: str) -> "DiffusionAnchorCheckpoint":
|
| 83 |
+
st = _require_safetensors()
|
| 84 |
+
from .checkpoint import DiffusionAnchorCheckpoint
|
| 85 |
+
from safetensors import safe_open
|
| 86 |
+
with safe_open(path, framework="pt", device="cpu") as f:
|
| 87 |
+
md = f.metadata() or {}
|
| 88 |
+
tensors = {k: f.get_tensor(k) for k in f.keys()}
|
| 89 |
+
if md.get("format") != "amoe.diffusion.anchor":
|
| 90 |
+
raise ValueError(f"not an amoe.diffusion.anchor safetensors: {path}")
|
| 91 |
+
meta = json.loads(md.get("amoe_meta", "{}"))
|
| 92 |
+
layout = md.get("key_layout", "amoe")
|
| 93 |
+
if layout == "amoe":
|
| 94 |
+
adapters = {k[len("blocks."):]: v for k, v in tensors.items()}
|
| 95 |
+
else: # comfy layout round-trip
|
| 96 |
+
names = meta.get("substrate", {}).get("site_names", [])
|
| 97 |
+
idx = {n: i for i, n in enumerate(names)}
|
| 98 |
+
adapters = {}
|
| 99 |
+
for k, v in tensors.items():
|
| 100 |
+
body = k[len("diffusion_model."):]
|
| 101 |
+
site, param = body.split(".aleph_relay.")
|
| 102 |
+
adapters[f"{idx[site]}.{param}"] = v
|
| 103 |
+
ck = DiffusionAnchorCheckpoint(adapters, meta)
|
| 104 |
+
want = md.get("content_hash_v2")
|
| 105 |
+
if want:
|
| 106 |
+
got = content_hash_v2(adapters)
|
| 107 |
+
if got != want:
|
| 108 |
+
raise ValueError(f"content hash mismatch in {path}: "
|
| 109 |
+
f"{got} != {want}")
|
| 110 |
+
return ck
|
src/amoe/testing/diffusion_invariants.py
ADDED
|
@@ -0,0 +1,334 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CPU-testable diffusion invariants on tiny stubs (no downloads).
|
| 2 |
+
|
| 3 |
+
The contracts, each tracing to a paid-for receipt:
|
| 4 |
+
toggle law (code-path skip, bit-exact) — every r2 experiment
|
| 5 |
+
P-INIT (zero-init W+B exactly inert) — exp006 bias-leak law
|
| 6 |
+
detach bit-exact via binding probe — the text line's exp011 bar
|
| 7 |
+
band windows sum-to-1, smooth, positional — exp008
|
| 8 |
+
flow x0 recovery exact and linear — exp013 (the conditioning law)
|
| 9 |
+
dtype law (declared, refuses meta) — R0b catch, fork f4f1c46
|
| 10 |
+
checkpoint round trips (.pt/.safetensors/legacy shapes) — burndown record
|
| 11 |
+
blob-on-eps refusal — exp012, 2-seed
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import os
|
| 16 |
+
import tempfile
|
| 17 |
+
|
| 18 |
+
import torch
|
| 19 |
+
import torch.nn as nn
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
# ── tiny stubs ──────────────────────────────────────────────────────────
|
| 23 |
+
|
| 24 |
+
class _TinySite(nn.Module):
|
| 25 |
+
"""Marker block with a width witness; tensor output."""
|
| 26 |
+
|
| 27 |
+
def __init__(self, d):
|
| 28 |
+
super().__init__()
|
| 29 |
+
self.d = d
|
| 30 |
+
self.lin = nn.Linear(d, d)
|
| 31 |
+
|
| 32 |
+
def forward(self, x):
|
| 33 |
+
return x + torch.tanh(self.lin(x))
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class _TinyTupleSite(_TinySite):
|
| 37 |
+
def forward(self, x):
|
| 38 |
+
return (x + torch.tanh(self.lin(x)), None)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class TinyUNetStub(nn.Module):
|
| 42 |
+
"""Scattered heterogeneous-width sites (down 8/8, mid 16, up 8) —
|
| 43 |
+
the SD15 shape in miniature, one tuple-output site for the
|
| 44 |
+
hybrid-safety path."""
|
| 45 |
+
|
| 46 |
+
def __init__(self, tuple_site=False):
|
| 47 |
+
super().__init__()
|
| 48 |
+
torch.manual_seed(11)
|
| 49 |
+
cls_last = _TinyTupleSite if tuple_site else _TinySite
|
| 50 |
+
self.down = nn.ModuleList([_TinySite(8), _TinySite(8)])
|
| 51 |
+
self.mid = _TinySite(16)
|
| 52 |
+
self.up = nn.ModuleList([cls_last(8)])
|
| 53 |
+
self.enc = nn.Linear(8, 16)
|
| 54 |
+
self.dec = nn.Linear(16, 8)
|
| 55 |
+
|
| 56 |
+
def forward(self, x): # x: [B, T, 8]
|
| 57 |
+
for blk in self.down:
|
| 58 |
+
out = blk(x)
|
| 59 |
+
x = out[0] if isinstance(out, tuple) else out
|
| 60 |
+
h = self.enc(x)
|
| 61 |
+
out = self.mid(h)
|
| 62 |
+
h = out[0] if isinstance(out, tuple) else out
|
| 63 |
+
x = self.dec(h)
|
| 64 |
+
out = self.up[0](x)
|
| 65 |
+
return out[0] if isinstance(out, tuple) else out
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
class TinyBinding:
|
| 69 |
+
name = "tiny_diffusion"
|
| 70 |
+
expected_sites = 4
|
| 71 |
+
|
| 72 |
+
def sites(self, model):
|
| 73 |
+
from ..binding.diffusion import DiffusionBinding # noqa: F401
|
| 74 |
+
out = [(n, m, m.d) for n, m in model.named_modules()
|
| 75 |
+
if isinstance(m, _TinySite)]
|
| 76 |
+
assert len(out) == self.expected_sites, len(out)
|
| 77 |
+
return out
|
| 78 |
+
|
| 79 |
+
def replace(self, model, name, new):
|
| 80 |
+
from ..binding.diffusion import walk_replace
|
| 81 |
+
walk_replace(model, name, new)
|
| 82 |
+
|
| 83 |
+
def declared_dtype(self, model):
|
| 84 |
+
from ..binding.diffusion import _declared_dtype
|
| 85 |
+
return _declared_dtype(model)
|
| 86 |
+
|
| 87 |
+
def probe(self, model):
|
| 88 |
+
g = torch.Generator().manual_seed(1400)
|
| 89 |
+
dt = self.declared_dtype(model)
|
| 90 |
+
x = torch.randn(2, 5, 8, generator=g).to(dt)
|
| 91 |
+
with torch.no_grad():
|
| 92 |
+
return model(x).detach().float().cpu()
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
# ── checkpoint builders ─────────────────────────────────────────────────
|
| 96 |
+
|
| 97 |
+
def _relay_ckpt(model, *, trained=True, name="t"):
|
| 98 |
+
from ..diffusion.core.relay import RelayPatch2D
|
| 99 |
+
from ..io.checkpoint import DiffusionAnchorCheckpoint
|
| 100 |
+
b = TinyBinding()
|
| 101 |
+
torch.manual_seed(23)
|
| 102 |
+
adapters = {}
|
| 103 |
+
widths = []
|
| 104 |
+
for i, (_, _, d) in enumerate(b.sites(model)):
|
| 105 |
+
r = RelayPatch2D(d, n_slots=4, K=8, hidden=12)
|
| 106 |
+
if trained:
|
| 107 |
+
nn.init.normal_(r.consume[-1].weight, std=0.05)
|
| 108 |
+
nn.init.normal_(r.consume[-1].bias, std=0.05)
|
| 109 |
+
for k, v in r.state_dict().items():
|
| 110 |
+
adapters[f"{i}.{k}"] = v.clone()
|
| 111 |
+
widths.append(d)
|
| 112 |
+
return DiffusionAnchorCheckpoint(adapters, {
|
| 113 |
+
"name": name, "adapter": {"kind": "relay"},
|
| 114 |
+
"substrate": {"family": "tiny_diffusion", "n_sites": len(widths),
|
| 115 |
+
"site_names": [s[0] for s in b.sites(model)],
|
| 116 |
+
"widths": widths}})
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _mb_ckpt(model, *, trained=True):
|
| 120 |
+
from ..diffusion.core.multiband import MultibandDelta
|
| 121 |
+
from ..io.checkpoint import DiffusionAnchorCheckpoint
|
| 122 |
+
b = TinyBinding()
|
| 123 |
+
torch.manual_seed(29)
|
| 124 |
+
adapters = {}
|
| 125 |
+
for i, (_, _, d) in enumerate(b.sites(model)):
|
| 126 |
+
m = MultibandDelta(d, r=3)
|
| 127 |
+
if trained:
|
| 128 |
+
for up in m.up:
|
| 129 |
+
nn.init.normal_(up.weight, std=0.05)
|
| 130 |
+
for k, v in m.state_dict().items():
|
| 131 |
+
adapters[f"{i}.{k}"] = v.clone()
|
| 132 |
+
return DiffusionAnchorCheckpoint(adapters, {
|
| 133 |
+
"name": "mb", "adapter": {"kind": "multiband3"},
|
| 134 |
+
"substrate": {"family": "tiny_diffusion", "n_sites": 4}})
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
# ── invariants ──────────────────────────────────────────────────────────
|
| 138 |
+
|
| 139 |
+
def assert_band_windows():
|
| 140 |
+
from ..diffusion.core.multiband import band_of, band_weights
|
| 141 |
+
s = torch.linspace(0, 1, 501)
|
| 142 |
+
w = band_weights(s)
|
| 143 |
+
assert (w.sum(-1) - 1).abs().max() < 1e-6, "windows do not sum to 1"
|
| 144 |
+
assert (w >= 0).all() and (w <= 1).all()
|
| 145 |
+
for v, b in ((0.1, 0), (0.5, 1), (0.9, 2)):
|
| 146 |
+
assert band_of(v) == b
|
| 147 |
+
assert int(band_weights(torch.tensor([v])).argmax()) == b
|
| 148 |
+
# max slope of the cosine ramp is pi/(4*XFADE) ~= 13.1 per unit s01;
|
| 149 |
+
# on a 1/500 grid that is ~0.0262 per step — assert the theory bound
|
| 150 |
+
d = (w[1:] - w[:-1]).abs().max()
|
| 151 |
+
assert d < 0.03, f"windows not smooth (max step {d} > theory ~0.0262)"
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def assert_flow_x0_recovery():
|
| 155 |
+
from ..diffusion.train.objectives import flow_pieces
|
| 156 |
+
g = torch.Generator().manual_seed(5)
|
| 157 |
+
lat = torch.randn(6, 4, 8, 8, generator=g)
|
| 158 |
+
s = torch.rand(6, generator=g)
|
| 159 |
+
noise = torch.randn(6, 4, 8, 8, generator=g)
|
| 160 |
+
x_t, v = flow_pieces(lat, s, noise)
|
| 161 |
+
x0 = x_t - s[:, None, None, None] * v
|
| 162 |
+
assert (x0 - lat).abs().max() < 1e-6, "flow x0 recovery not linear/exact"
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def assert_toggle_law_diffusion(tuple_site=False):
|
| 166 |
+
from ..diffusion.runtime.attach import attach
|
| 167 |
+
m = TinyUNetStub(tuple_site=tuple_site)
|
| 168 |
+
b = TinyBinding()
|
| 169 |
+
g = torch.Generator().manual_seed(31)
|
| 170 |
+
x = torch.randn(2, 5, 8, generator=g)
|
| 171 |
+
with torch.no_grad():
|
| 172 |
+
base = m(x).clone()
|
| 173 |
+
|
| 174 |
+
# relay: P-INIT (fresh zero-init anchors are EXACTLY inert)
|
| 175 |
+
h = attach(m, _relay_ckpt(m, trained=False), binding=b)
|
| 176 |
+
with torch.no_grad():
|
| 177 |
+
assert torch.equal(m(x), base), "P-INIT broken (zero-init leak)"
|
| 178 |
+
h.detach(verify=True)
|
| 179 |
+
|
| 180 |
+
# relay: trained anchor moves the output; all_off is bit-exact
|
| 181 |
+
h = attach(m, _relay_ckpt(m, trained=True), binding=b)
|
| 182 |
+
with torch.no_grad():
|
| 183 |
+
on = m(x)
|
| 184 |
+
assert not torch.equal(on, base), "trained relay produced no delta"
|
| 185 |
+
with h.all_off():
|
| 186 |
+
with torch.no_grad():
|
| 187 |
+
assert torch.equal(m(x), base), "toggle law violated (relay)"
|
| 188 |
+
h.detach(verify=True)
|
| 189 |
+
with torch.no_grad():
|
| 190 |
+
assert torch.equal(m(x), base), "model altered after detach"
|
| 191 |
+
|
| 192 |
+
# multiband: windows drive it; all_off + full lesion are bit-exact
|
| 193 |
+
h = attach(m, _mb_ckpt(m, trained=True), binding=b)
|
| 194 |
+
h.set_band_windows(torch.tensor([0.9, 0.1]))
|
| 195 |
+
with torch.no_grad():
|
| 196 |
+
on = m(x)
|
| 197 |
+
assert not torch.equal(on, base), "trained multiband produced no delta"
|
| 198 |
+
with h.all_off():
|
| 199 |
+
with torch.no_grad():
|
| 200 |
+
assert torch.equal(m(x), base), "toggle law violated (multiband)"
|
| 201 |
+
with h.lesion_band(0, 1, 2):
|
| 202 |
+
with torch.no_grad():
|
| 203 |
+
assert torch.equal(m(x), base), "full band lesion not bit-exact"
|
| 204 |
+
h.detach(verify=True)
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def assert_lesion_guard():
|
| 208 |
+
from ..diffusion.runtime.attach import attach
|
| 209 |
+
m = TinyUNetStub()
|
| 210 |
+
h = attach(m, _relay_ckpt(m), binding=TinyBinding())
|
| 211 |
+
try:
|
| 212 |
+
with h.lesion_band(0):
|
| 213 |
+
pass
|
| 214 |
+
raise AssertionError("lesion_band on a relay anchor must TypeError")
|
| 215 |
+
except TypeError:
|
| 216 |
+
pass
|
| 217 |
+
h.detach(verify=True)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def assert_dtype_law():
|
| 221 |
+
from ..binding.diffusion import _declared_dtype
|
| 222 |
+
from ..diffusion.runtime.attach import attach
|
| 223 |
+
m = TinyUNetStub().to(torch.bfloat16)
|
| 224 |
+
b = TinyBinding()
|
| 225 |
+
g = torch.Generator().manual_seed(31)
|
| 226 |
+
x = torch.randn(2, 5, 8, generator=g).to(torch.bfloat16)
|
| 227 |
+
with torch.no_grad():
|
| 228 |
+
base = m(x).clone()
|
| 229 |
+
ck = _relay_ckpt(m, trained=True)
|
| 230 |
+
h = attach(m, ck, binding=b) # dtype=None -> declared
|
| 231 |
+
for mod in h.modules():
|
| 232 |
+
for p in mod.parameters():
|
| 233 |
+
assert p.dtype == torch.bfloat16, "dtype law: adapter != trunk"
|
| 234 |
+
with h.all_off():
|
| 235 |
+
with torch.no_grad():
|
| 236 |
+
assert torch.equal(m(x), base), \
|
| 237 |
+
"toggle law must be dtype-independent (code-path skip)"
|
| 238 |
+
h.detach(verify=True)
|
| 239 |
+
|
| 240 |
+
class _Meta(nn.Module):
|
| 241 |
+
def __init__(self):
|
| 242 |
+
super().__init__()
|
| 243 |
+
self.p = nn.Parameter(torch.empty(1, device="meta"))
|
| 244 |
+
try:
|
| 245 |
+
_declared_dtype(_Meta())
|
| 246 |
+
raise AssertionError("declared_dtype must refuse meta params")
|
| 247 |
+
except RuntimeError:
|
| 248 |
+
pass
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def assert_checkpoint_roundtrip():
|
| 252 |
+
from ..io.checkpoint import load_diffusion_anchor
|
| 253 |
+
from ..io.safetensors_io import (load_anchor_safetensors,
|
| 254 |
+
save_anchor_safetensors)
|
| 255 |
+
m = TinyUNetStub()
|
| 256 |
+
ck = _relay_ckpt(m, trained=True)
|
| 257 |
+
tmp = tempfile.mkdtemp()
|
| 258 |
+
|
| 259 |
+
# .pt round trip
|
| 260 |
+
p = os.path.join(tmp, "a.pt")
|
| 261 |
+
ck.save(p)
|
| 262 |
+
back = load_diffusion_anchor(p)
|
| 263 |
+
assert back.kind == "relay" and set(back.adapters) == set(ck.adapters)
|
| 264 |
+
for k in ck.adapters:
|
| 265 |
+
assert torch.equal(back.adapters[k], ck.adapters[k])
|
| 266 |
+
|
| 267 |
+
# safetensors round trip, canonical + comfy layouts, hash verified
|
| 268 |
+
for layout in ("amoe", "comfy"):
|
| 269 |
+
sp = os.path.join(tmp, f"a_{layout}.safetensors")
|
| 270 |
+
save_anchor_safetensors(ck, sp, key_layout=layout)
|
| 271 |
+
b2 = load_anchor_safetensors(sp)
|
| 272 |
+
assert set(b2.adapters) == set(ck.adapters), layout
|
| 273 |
+
for k in ck.adapters:
|
| 274 |
+
assert torch.equal(b2.adapters[k], ck.adapters[k]), (layout, k)
|
| 275 |
+
|
| 276 |
+
# legacy shapes: relays list / mods list / fork dict (home rebuilt)
|
| 277 |
+
per = [ck.per_site(i) for i in range(4)]
|
| 278 |
+
lp = os.path.join(tmp, "legacy_relays.pt")
|
| 279 |
+
torch.save({"relays": per}, lp)
|
| 280 |
+
assert load_diffusion_anchor(lp).kind == "relay"
|
| 281 |
+
|
| 282 |
+
mb = _mb_ckpt(m)
|
| 283 |
+
mp = os.path.join(tmp, "legacy_mods.pt")
|
| 284 |
+
torch.save({"mods": [mb.per_site(i) for i in range(4)]}, mp)
|
| 285 |
+
assert load_diffusion_anchor(mp).kind == "multiband3"
|
| 286 |
+
|
| 287 |
+
fork = {str(i): {k: v for k, v in per[i].items() if k != "addr.home"}
|
| 288 |
+
for i in range(4)}
|
| 289 |
+
fp = os.path.join(tmp, "fork.pt")
|
| 290 |
+
torch.save({"relays": fork}, fp)
|
| 291 |
+
fk = load_diffusion_anchor(fp)
|
| 292 |
+
assert fk.meta.get("home_reconstructed") is True, \
|
| 293 |
+
"fork import must STAMP the reconstructed home"
|
| 294 |
+
assert "0.addr.home" in fk.adapters
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def assert_blob_on_eps_refusal():
|
| 298 |
+
from ..diffusion.train.config import DiffusionTrainConfig
|
| 299 |
+
from ..diffusion.train.trainer import train
|
| 300 |
+
cfg = DiffusionTrainConfig(objective="eps", blob=True)
|
| 301 |
+
try:
|
| 302 |
+
train(None, {}, cfg)
|
| 303 |
+
raise AssertionError("blob-on-eps must refuse (conditioning law)")
|
| 304 |
+
except ValueError as e:
|
| 305 |
+
assert "conditioning law" in str(e)
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def assert_align_negative():
|
| 309 |
+
from ..diffusion.train.aligner import align
|
| 310 |
+
try:
|
| 311 |
+
align()
|
| 312 |
+
raise AssertionError("align must raise with the record")
|
| 313 |
+
except NotImplementedError as e:
|
| 314 |
+
assert "falsified" in str(e)
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
def run_all() -> None:
|
| 318 |
+
assert_band_windows()
|
| 319 |
+
assert_flow_x0_recovery()
|
| 320 |
+
assert_toggle_law_diffusion(tuple_site=False)
|
| 321 |
+
assert_toggle_law_diffusion(tuple_site=True)
|
| 322 |
+
assert_lesion_guard()
|
| 323 |
+
assert_dtype_law()
|
| 324 |
+
assert_checkpoint_roundtrip()
|
| 325 |
+
assert_blob_on_eps_refusal()
|
| 326 |
+
assert_align_negative()
|
| 327 |
+
print("amoe.diffusion invariants: band windows, exact flow x0, toggle "
|
| 328 |
+
"law (relay+multiband, tensor+tuple), P-INIT, bit-exact detach, "
|
| 329 |
+
"dtype law, checkpoint round trips (.pt/.safetensors/legacy/"
|
| 330 |
+
"fork), blob-on-eps refusal, align negative — ALL GREEN")
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
if __name__ == "__main__":
|
| 334 |
+
run_all()
|
tests/test_invariants.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
| 1 |
import sys; sys.path.insert(0, "src")
|
| 2 |
from amoe.testing.invariants import run_all
|
| 3 |
run_all()
|
|
|
|
|
|
|
|
|
| 1 |
import sys; sys.path.insert(0, "src")
|
| 2 |
from amoe.testing.invariants import run_all
|
| 3 |
run_all()
|
| 4 |
+
from amoe.testing.diffusion_invariants import run_all as run_diffusion
|
| 5 |
+
run_diffusion()
|