File size: 19,159 Bytes
7d718e7
6e05fb0
 
 
 
7d718e7
563a5ec
 
6e05fb0
 
 
 
 
 
26e5956
 
 
6e05fb0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ad0b7cd
6e05fb0
 
 
 
 
 
 
 
 
 
 
 
 
5027fd8
7d718e7
 
 
563a5ec
 
 
6e05fb0
 
 
 
 
 
7d718e7
 
 
6e05fb0
 
 
 
7d718e7
6e05fb0
 
 
7d718e7
 
 
6e05fb0
 
7d718e7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6e05fb0
 
 
 
 
7d718e7
 
 
 
 
 
6e05fb0
7d718e7
6e05fb0
 
 
7d718e7
6e05fb0
 
 
7d718e7
6e05fb0
ad0b7cd
 
 
6e05fb0
ad0b7cd
 
 
 
7d718e7
6e05fb0
 
ad0b7cd
 
 
 
 
 
 
 
 
6e05fb0
7d718e7
6e05fb0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ad0b7cd
7d718e7
 
ad0b7cd
6e05fb0
ad0b7cd
6e05fb0
 
 
 
ad0b7cd
6e05fb0
 
ad0b7cd
6e05fb0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
563a5ec
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70a9b4a
6e05fb0
563a5ec
6e05fb0
 
 
ad0b7cd
 
6e05fb0
 
 
563a5ec
6e05fb0
 
563a5ec
 
6e05fb0
 
563a5ec
6e05fb0
 
 
563a5ec
 
 
 
 
 
 
 
 
 
 
6e05fb0
 
 
563a5ec
 
7d718e7
563a5ec
7d718e7
 
563a5ec
 
 
 
 
 
7d718e7
 
 
 
563a5ec
7d718e7
 
 
563a5ec
5027fd8
7d718e7
563a5ec
7d718e7
6e05fb0
 
 
 
7d718e7
 
6e05fb0
 
 
 
 
 
 
 
 
 
 
7d718e7
 
6e05fb0
 
 
 
7d718e7
 
6e05fb0
 
7d718e7
 
 
 
 
 
 
 
6e05fb0
7d718e7
6e05fb0
7d718e7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6e05fb0
 
 
 
5027fd8
 
7d718e7
6e05fb0
7d718e7
 
6e05fb0
7d718e7
 
 
 
563a5ec
 
 
 
 
 
5027fd8
 
6e05fb0
 
 
 
 
 
7d718e7
6e05fb0
 
 
 
 
 
 
 
 
 
 
 
 
563a5ec
6e05fb0
 
 
 
 
7d718e7
 
6e05fb0
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
"""Agatha diffusion playground: any two checkpoints of the training runs side by side, from the same noise.

Weights: AbstractPhil/geofractal-agatha-diffusion-training, folder coco256/ (non-commercial: trained on COCO 2014 photos).
Code: github.com/AbstractEyes/geofractal, src/geofractal/router/prefab/agatha_v2, installed at the pinned commit below.
Sampling is the library's own sample_latents (the helper the training's picture sheets use), so a picture here matches a
sheet made at the same checkpoint, caption, seed, guidance and step count.
Tower switches (Agatha only): an unticked tower's force is set to zero inside the velocity for that side's pictures; the
trunk, the oscillator, the band weights and the other towers stay as trained, and the model is restored afterwards.
"""
import random
import re
import subprocess
import sys

import spaces  # first, before anything imports torch: ZeroGPU patches torch at this import

GEOFRACTAL ="git+https://github.com/AbstractEyes/geofractal@46fabb02ec851e4a4fb03b3ada30185a26efe2f7"
GEOVOCAB = "git+https://github.com/AbstractEyes/lattice_vocabulary@2982a5c2f9dcd71a496468086d45083513deecf4"
WIDE_COMPILER = "git+https://github.com/AbstractEyes/pytorch-parallel-compiler@70bcba4f2cdddb26909df59856eb6afead753826"


def _install_library():
    """The library at its pinned commit, without dependencies (they are in requirements.txt)."""
    try:
        import geofractal.router.prefab.agatha_v2.hooks  # noqa: F401
        return
    except Exception:  # noqa: BLE001
        pass
    for spec in (GEOVOCAB, WIDE_COMPILER, GEOFRACTAL):
        subprocess.run([sys.executable, "-m", "pip", "install", "-q", "--no-deps", spec], check=True)


_install_library()

import gradio as gr  # noqa: E402
import numpy as np  # noqa: E402
import torch  # noqa: E402
from diffusers import AutoencoderKLFlux2  # noqa: E402
from huggingface_hub import HfApi, hf_hub_download, snapshot_download  # noqa: E402
from PIL import Image  # noqa: E402
from transformers import AutoTokenizer, T5EncoderModel  # noqa: E402

from geofractal.router.prefab.agatha_v2.hooks import sample_latents  # noqa: E402
from geofractal.router.prefab.agatha_v2.trainer import load_checkpoint  # noqa: E402

REPO = "AbstractPhil/geofractal-agatha-diffusion-training"
# The encoders at the revisions the training data was encoded with.
VAE_REPO, VAE_SUBFOLDER = "black-forest-labs/FLUX.2-klein-4B", "vae"
VAE_REVISION = "e7b7dc27f91deacad38e78976d1f2b499d76a294"
TEXT_ENCODER, TEXT_REVISION, TEXT_MAX = "google-t5/t5-base", "a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1", 128
LATENT_SHAPE = (32, 32, 32)
ARMS = ("agatha", "plain")
NAMES = {"agatha": "Agatha", "plain": "Lia"}  # Lia = the control, a plain transformer; her runs keep "plain" in their names
NONE = "None"
FILE_RE = re.compile(r"^coco256/(agatha_v2_coco256_(agatha|plain)_s(\d+)(_rep)?)/\1_step(\d+)_(raw|ema)\.pt$")
KEEP_LOADED = 8                 # checkpoints kept in memory per process (each about 170 MB)
# Agatha's six towers in the model's own order (geometry towers, then convolution towers); a checkpoint's config decides
# which index each name has when a switch is applied.
TOWERS = ("cantor", "beatrix", "simplex", "helix", "wide_resnet", "frequency")
EXAMPLES = ["A man riding a bicycle down a city street.", "A herd of elephants walking through tall grass.",
            "A baseball player swinging a bat at a ball.", "A plate of food with broccoli and rice.",
            "A red double decker bus driving down a street.", "Two dogs playing in the snow.",
            "A kitchen with wooden cabinets and a white stove.", "A person riding a wave on a surfboard."]


# ----------------------------------------------------------------------------- the checkpoint list
def list_checkpoints():
    """{(arm, run, step, weights): path} for every milestone file in the weights repository."""
    out = {}
    for path in HfApi().list_repo_files(REPO):
        m = FILE_RE.match(path)
        if m:
            out[(m.group(2), m.group(1), int(m.group(5)), m.group(6))] = path
    return out


def label_of(key):
    arm, run, step, weights = key
    return "%s, %s, step %s, %s" % (NAMES[arm], run_words(run), format(step, ","), weights)


def ordered(found):
    """Checkpoints by model, then run, newest step first, raw before averaged."""
    return sorted(found, key=lambda k: (ARMS.index(k[0]), k[1], -k[2], k[3] != "raw"))


def catalog(found):
    """({label: key}, [labels in order])."""
    keys = ordered(found)
    return {label_of(k): k for k in keys}, [label_of(k) for k in keys]


def defaults(found):
    """The newest raw checkpoint of each model (seed 0 first); None for a model with nothing yet."""
    out = []
    for arm in ARMS:
        keys = [k for k in ordered(found) if k[0] == arm and k[3] == "raw"]
        out.append(label_of(keys[0]) if keys else NONE)
    return out


def run_words(run):
    """'agatha_v2_coco256_agatha_s0_rep' -> 'seed 0 (repeat)'."""
    seed = re.search(r"_s(\d+)", run).group(1)
    return "seed %s%s" % (seed, " (repeat)" if run.endswith("_rep") else "")


def availability(found):
    parts = []
    for arm in ARMS:
        steps = {}
        for k in found:
            if k[0] == arm:
                steps.setdefault(k[1], set()).add(k[2])
        text = "; ".join("%s at %s" % (run_words(run), ", ".join(format(s, ",") for s in sorted(st)))
                         for run, st in sorted(steps.items())) or "none yet"
        parts.append("**%s:** %s" % (NAMES[arm], text))
    return "Checkpoints in the weights repository: " + " | ".join(parts)


try:
    FOUND = list_checkpoints()
except Exception as e:  # noqa: BLE001
    print("could not list the weights repository at start-up:", repr(e)[:200], flush=True)
    FOUND = {}
LABELS, ORDER = catalog(FOUND)

# ----------------------------------------------------------------------------- encoders
# Start-up only downloads files and builds the tokenizer. Every model is built inside the GPU function, in the process that
# runs it: nothing at import asks torch about a GPU (the start-up process has none, and the GPU worker inherits what it asked).
TOKENIZER = AutoTokenizer.from_pretrained(TEXT_ENCODER, revision=TEXT_REVISION)
snapshot_download(TEXT_ENCODER, revision=TEXT_REVISION, allow_patterns=["config.json", "model.safetensors"])
snapshot_download(VAE_REPO, revision=VAE_REVISION,
                  allow_patterns=["vae/config.json", "vae/diffusion_pytorch_model.safetensors"])
_ENCODERS = {}                  # "t5" and "vae", built once per process
_LOADED = {}                    # (arm, run, step, weights) -> (model, latent mean, latent std), this process only


def encoders(device):
    """T5-base and the FLUX.2 autoencoder (from the files downloaded at start-up), on the device."""
    if not _ENCODERS:
        _ENCODERS["t5"] = T5EncoderModel.from_pretrained(TEXT_ENCODER, revision=TEXT_REVISION).eval()
        _ENCODERS["vae"] = AutoencoderKLFlux2.from_pretrained(VAE_REPO, subfolder=VAE_SUBFOLDER, revision=VAE_REVISION,
                                                              torch_dtype=torch.bfloat16).eval()
    return _ENCODERS["t5"].to(device), _ENCODERS["vae"].to(device)


def fetch(key):
    """The checkpoint file on local disk (the Hub's cache). Called outside the GPU function."""
    return hf_hub_download(REPO, FOUND[key])


def load_arm(key, path):
    """The model and its latent statistics (written into every milestone file by the training run)."""
    if key not in _LOADED:
        payload = torch.load(path, map_location="cpu", weights_only=True)
        model, _, _ = load_checkpoint(path, map_location="cpu")
        extra = payload.get("extra", {})
        mean = torch.tensor(extra["latent_mean"], dtype=torch.float32).view(1, -1, 1, 1)
        std = torch.tensor(extra["latent_std"], dtype=torch.float32).view(1, -1, 1, 1)
        while len(_LOADED) >= KEEP_LOADED:
            _LOADED.pop(next(iter(_LOADED)))
        _LOADED[key] = (model.eval(), mean, std)
    return _LOADED[key]


try:                            # the default pair's files, downloaded at start-up so the first picture does not wait
    for _label in defaults(FOUND):
        if _label != NONE:
            fetch(LABELS[_label])
except Exception as e:  # noqa: BLE001
    print("prefetch skipped:", repr(e)[:200], flush=True)


# ----------------------------------------------------------------------------- pictures
@torch.no_grad()
def encode_prompt(prompt, t5, device):
    """T5-base states as in training: the true length from the attention mask, rounded through float16."""
    t = TOKENIZER([prompt], padding=True, truncation=True, max_length=TEXT_MAX, return_tensors="pt")
    states = t5(input_ids=t["input_ids"].to(device), attention_mask=t["attention_mask"].to(device)).last_hidden_state
    n = int(t["attention_mask"].sum())
    return states[:, :n].half().float().cpu(), torch.ones(1, n, dtype=torch.bool)


def grid(images):
    """[n, 3, h, w] in [0, 1] -> one picture, two columns when n > 2."""
    n, _, h, w = images.shape
    cols = 2 if n > 2 else n
    rows = (n + cols - 1) // cols
    canvas = torch.ones(3, rows * h, cols * w)
    for i in range(n):
        r, c = divmod(i, cols)
        canvas[:, r * h:(r + 1) * h, c * w:(c + 1) * w] = images[i]
    arr = (canvas.permute(1, 2, 0).clamp(0, 1).numpy() * 255.0).round().astype(np.uint8)
    return Image.fromarray(arr)


class TowersOff:
    """Inside the model's velocity, the named towers' forces are zero; the model is restored on exit (also on an error)."""

    def __init__(self, model, names_off):
        kinds = list(getattr(model.config, "geometry_towers", ())) + list(getattr(model.config, "conv_towers", ()))
        off = set(names_off or ())
        self.model, self.drop = model, [i for i, kind in enumerate(kinds) if kind in off]

    def __enter__(self):
        if self.drop:
            original, drop = self.model.tower_forces, list(self.drop)

            def tower_forces(h, c, text_mask, e):
                forces = original(h, c, text_mask, e)                     # [K, B, N, P]
                keep = torch.ones(forces.shape[0], device=forces.device, dtype=forces.dtype)
                keep[drop] = 0
                return forces * keep.view(-1, *([1] * (forces.dim() - 1)))
            self.model.tower_forces = tower_forces                        # shadows the method for this model only
        return self.model

    def __exit__(self, *exc):
        if self.drop:
            del self.model.tower_forces
        return False


@spaces.GPU(duration=45, size="large")
def generate(prompt, jobs, seed, n, guidance, steps):
    """jobs: [(side name, key, local path, towers off)]. Every side starts from the same noises (seeds seed .. seed + n - 1)."""
    device = "cuda" if torch.cuda.is_available() else "cpu"
    out = {}
    with torch.no_grad():
        t5, vae = encoders(device)
        text, mask = encode_prompt(prompt, t5, device)
        text, mask = text.repeat(n, 1, 1), mask.repeat(n, 1)
        noise = torch.stack([torch.randn(LATENT_SHAPE, generator=torch.Generator().manual_seed(seed + k))
                             for k in range(n)])
        for name, key, path, off in jobs:
            model, mean, std = load_arm(key, path)
            model.to(device)
            with TowersOff(model, off):
                z = sample_latents(model, text, mask, noise, steps, guidance, device, use_amp=True, batch_size=8)
            z_raw = z * std + mean
            x = vae.decode(z_raw.to(device, torch.bfloat16)).sample.float()
            out[name] = grid(((x + 1.0) / 2.0).clamp(0, 1).cpu())
    return out


def side(label, towers_on):
    """(side name, checkpoint key, towers off). Switches count for Agatha only; the name carries the towers that are off."""
    key = LABELS[label]
    on = TOWERS if towers_on is None else [t for t in TOWERS if t in towers_on]
    off = tuple(t for t in TOWERS if t not in on) if key[0] == "agatha" else ()
    if not off:
        return label, key, off
    return "%s, %s" % (label, "all towers off" if len(off) == len(TOWERS) else "towers off: " + ", ".join(off)), key, off


def run(prompt, a, b, seed, n, guidance, steps, blind, tally, towers_a=None, towers_b=None):
    prompt = (prompt or "").strip()
    if not prompt:
        raise gr.Error("Write a caption first.")
    picked = [(x, t) for x, t in ((a, towers_a), (b, towers_b)) if x and x != NONE]
    if not picked:
        raise gr.Error("Choose at least one checkpoint.")
    stale = [x for x, _ in picked if x not in LABELS]
    if stale:
        raise gr.Error("Not in the list any more: %s. Press Refresh." % ", ".join(stale))
    sides = [side(x, t) for x, t in picked]
    jobs = {name: (key, off) for name, key, off in sides}
    images = generate(prompt, [(name, key, fetch(key), list(off)) for name, (key, off) in jobs.items()],
                      int(seed), int(n), float(guidance), int(steps))
    ignored = any(LABELS[x][0] != "agatha" and t is not None and len(t) < len(TOWERS) for x, t in picked)
    chosen = [name for name, _, _ in sides]
    if len(chosen) == 1:
        order = [chosen[0], None]
        note = "One checkpoint chosen: shown on the left."
    else:
        order = list(chosen)
        if blind:
            random.shuffle(order)
        note = "Which side is better? Pick, then reveal." if blind else "Shown by name (blind mode is off)."
    if ignored:
        note += " Lia, the control, has no towers: her switches change nothing."
    show = [("Left", "Right")[i] if (blind and len(chosen) == 2) else (x or "No checkpoint") for i, x in enumerate(order)]
    pictures = [images[x] if x else None for x in order]
    return (gr.update(value=pictures[0], label=show[0]), gr.update(value=pictures[1], label=show[1]),
            order, note, tally)


def pick(side, order, tally):
    if not order or None in order:
        return "Generate a pair of two checkpoints first.", tally
    tally = dict(tally or {})
    if side == "tie":
        tally["tie"] = tally.get("tie", 0) + 1
    else:
        winner = order[0] if side == "left" else order[1]
        tally[winner] = tally.get(winner, 0) + 1
    return "Counted. " + counts(tally), tally


def counts(tally):
    tally = tally or {}
    wins = ["%s: %d" % (k, v) for k, v in sorted(tally.items(), key=lambda kv: -kv[1]) if k != "tie"]
    return "Picks this session: %s | ties: %d." % ("; ".join(wins) or "none yet", tally.get("tie", 0))


def reveal(order, tally):
    if not order:
        return "Generate first."
    return "Left = %s, right = %s. %s" % (order[0], order[1] or "no checkpoint", counts(tally))


def _relist():
    global FOUND, LABELS, ORDER
    FOUND = list_checkpoints()
    LABELS, ORDER = catalog(FOUND)


def refresh(a, b):
    """The Refresh button: re-read the list and keep each choice that is still in it."""
    try:
        _relist()
    except Exception as e:  # noqa: BLE001
        return gr.update(), gr.update(), "Could not read the weights repository (%s); try again." % type(e).__name__
    options = [NONE] + ORDER
    d = defaults(FOUND)
    a = a if a in options else d[0]
    b = b if b in options else d[1]
    return gr.update(choices=options, value=a), gr.update(choices=options, value=b), availability(FOUND)


def load_page():
    """A page load: the fresh list and the default pair (each model's newest checkpoint, None where it has none)."""
    try:
        _relist()
    except Exception:  # noqa: BLE001
        pass
    d = defaults(FOUND)
    options = [NONE] + ORDER
    return gr.update(choices=options, value=d[0]), gr.update(choices=options, value=d[1]), availability(FOUND)


# ----------------------------------------------------------------------------- the page
with gr.Blocks(title="Agatha diffusion playground") as demo:
    gr.Markdown("# Agatha diffusion playground\nAgatha and Lia, her control (a plain transformer of the same size; about "
                "42M parameters each), trained the same way on COCO 2014 at 256 px. Pick any two checkpoints (either model, any step, "
                "raw or averaged weights) or one with None on the other side; both sides start from the same noise. "
                "Non-commercial use only (trained on COCO photos).")
    d0 = defaults(FOUND)
    prompt = gr.Textbox(label="Caption", value=EXAMPLES[0])
    with gr.Row():
        ckpt_a = gr.Dropdown([NONE] + ORDER, value=d0[0], label="Checkpoint A")
        ckpt_b = gr.Dropdown([NONE] + ORDER, value=d0[1], label="Checkpoint B")
        refresh_btn = gr.Button("Refresh list", scale=0, min_width=140)
    gr.Markdown("Early in training the raw weights are sharper; the averaged (EMA) weights catch up later.")
    with gr.Row():
        towers_a = gr.CheckboxGroup(list(TOWERS), value=list(TOWERS), label="Agatha's towers on, checkpoint A")
        towers_b = gr.CheckboxGroup(list(TOWERS), value=list(TOWERS), label="Agatha's towers on, checkpoint B")
    gr.Markdown("Untick a tower to switch its push off for that side; the trunk and the other towers stay as trained. "
                "cantor, beatrix, simplex and helix are attention towers, each with its own rotary position code; "
                "wide_resnet and frequency are convolution towers (frequency sees a high-passed input). The switches act on "
                "Agatha checkpoints only (Lia, the plain "
                "transformer, has no towers). Put the same checkpoint on both sides to compare tower settings.")
    with gr.Row():
        seed = gr.Number(value=0, precision=0, label="Seed")
        n = gr.Slider(1, 4, value=4, step=1, label="Pictures per side")
        guidance = gr.Slider(1.0, 7.0, value=3.0, step=0.5, label="Guidance")
        steps = gr.Slider(10, 50, value=25, step=1, label="Sampling steps")
        blind = gr.Checkbox(value=True, label="Blind (hide which side is which)")
    go = gr.Button("Generate", variant="primary")
    gr.Examples(EXAMPLES, inputs=prompt)
    with gr.Row():
        left = gr.Image(label="Left", type="pil", height=420)
        right = gr.Image(label="Right", type="pil", height=420)
    with gr.Row():
        b_left = gr.Button("Left is better")
        b_tie = gr.Button("Tie")
        b_right = gr.Button("Right is better")
        b_reveal = gr.Button("Reveal")
    status = gr.Markdown()
    found_md = gr.Markdown(availability(FOUND))
    order_state = gr.State([])
    tally_state = gr.State({})
    go.click(run, [prompt, ckpt_a, ckpt_b, seed, n, guidance, steps, blind, tally_state, towers_a, towers_b],
             [left, right, order_state, status, tally_state])
    b_left.click(lambda o, t: pick("left", o, t), [order_state, tally_state], [status, tally_state])
    b_right.click(lambda o, t: pick("right", o, t), [order_state, tally_state], [status, tally_state])
    b_tie.click(lambda o, t: pick("tie", o, t), [order_state, tally_state], [status, tally_state])
    b_reveal.click(reveal, [order_state, tally_state], status)
    refresh_btn.click(refresh, [ckpt_a, ckpt_b], [ckpt_a, ckpt_b, found_md])
    demo.load(load_page, None, [ckpt_a, ckpt_b, found_md])

if __name__ == "__main__":
    demo.launch()