"""Agatha diffusion playground: any two checkpoints of the training runs side by side, from the same noise. Weights: AbstractPhil/geofractal-agatha-diffusion-training, folder coco256/ (non-commercial: trained on COCO 2014 photos). Code: github.com/AbstractEyes/geofractal, src/geofractal/router/prefab/agatha_v2, installed at the pinned commit below. Sampling is the library's own sample_latents (the helper the training's picture sheets use), so a picture here matches a sheet made at the same checkpoint, caption, seed, guidance and step count. Tower switches (Agatha only): an unticked tower's force is set to zero inside the velocity for that side's pictures; the trunk, the oscillator, the band weights and the other towers stay as trained, and the model is restored afterwards. """ import random import re import subprocess import sys import spaces # first, before anything imports torch: ZeroGPU patches torch at this import GEOFRACTAL ="git+https://github.com/AbstractEyes/geofractal@46fabb02ec851e4a4fb03b3ada30185a26efe2f7" GEOVOCAB = "git+https://github.com/AbstractEyes/lattice_vocabulary@2982a5c2f9dcd71a496468086d45083513deecf4" WIDE_COMPILER = "git+https://github.com/AbstractEyes/pytorch-parallel-compiler@70bcba4f2cdddb26909df59856eb6afead753826" def _install_library(): """The library at its pinned commit, without dependencies (they are in requirements.txt).""" try: import geofractal.router.prefab.agatha_v2.hooks # noqa: F401 return except Exception: # noqa: BLE001 pass for spec in (GEOVOCAB, WIDE_COMPILER, GEOFRACTAL): subprocess.run([sys.executable, "-m", "pip", "install", "-q", "--no-deps", spec], check=True) _install_library() import gradio as gr # noqa: E402 import numpy as np # noqa: E402 import torch # noqa: E402 from diffusers import AutoencoderKLFlux2 # noqa: E402 from huggingface_hub import HfApi, hf_hub_download, snapshot_download # noqa: E402 from PIL import Image # noqa: E402 from transformers import AutoTokenizer, T5EncoderModel # noqa: E402 from geofractal.router.prefab.agatha_v2.hooks import sample_latents # noqa: E402 from geofractal.router.prefab.agatha_v2.trainer import load_checkpoint # noqa: E402 REPO = "AbstractPhil/geofractal-agatha-diffusion-training" # The encoders at the revisions the training data was encoded with. VAE_REPO, VAE_SUBFOLDER = "black-forest-labs/FLUX.2-klein-4B", "vae" VAE_REVISION = "e7b7dc27f91deacad38e78976d1f2b499d76a294" TEXT_ENCODER, TEXT_REVISION, TEXT_MAX = "google-t5/t5-base", "a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1", 128 LATENT_SHAPE = (32, 32, 32) ARMS = ("agatha", "plain") NAMES = {"agatha": "Agatha", "plain": "Lia"} # Lia = the control, a plain transformer; her runs keep "plain" in their names NONE = "None" FILE_RE = re.compile(r"^coco256/(agatha_v2_coco256_(agatha|plain)_s(\d+)(_rep)?)/\1_step(\d+)_(raw|ema)\.pt$") KEEP_LOADED = 8 # checkpoints kept in memory per process (each about 170 MB) # Agatha's six towers in the model's own order (geometry towers, then convolution towers); a checkpoint's config decides # which index each name has when a switch is applied. TOWERS = ("cantor", "beatrix", "simplex", "helix", "wide_resnet", "frequency") EXAMPLES = ["A man riding a bicycle down a city street.", "A herd of elephants walking through tall grass.", "A baseball player swinging a bat at a ball.", "A plate of food with broccoli and rice.", "A red double decker bus driving down a street.", "Two dogs playing in the snow.", "A kitchen with wooden cabinets and a white stove.", "A person riding a wave on a surfboard."] # ----------------------------------------------------------------------------- the checkpoint list def list_checkpoints(): """{(arm, run, step, weights): path} for every milestone file in the weights repository.""" out = {} for path in HfApi().list_repo_files(REPO): m = FILE_RE.match(path) if m: out[(m.group(2), m.group(1), int(m.group(5)), m.group(6))] = path return out def label_of(key): arm, run, step, weights = key return "%s, %s, step %s, %s" % (NAMES[arm], run_words(run), format(step, ","), weights) def ordered(found): """Checkpoints by model, then run, newest step first, raw before averaged.""" return sorted(found, key=lambda k: (ARMS.index(k[0]), k[1], -k[2], k[3] != "raw")) def catalog(found): """({label: key}, [labels in order]).""" keys = ordered(found) return {label_of(k): k for k in keys}, [label_of(k) for k in keys] def defaults(found): """The newest raw checkpoint of each model (seed 0 first); None for a model with nothing yet.""" out = [] for arm in ARMS: keys = [k for k in ordered(found) if k[0] == arm and k[3] == "raw"] out.append(label_of(keys[0]) if keys else NONE) return out def run_words(run): """'agatha_v2_coco256_agatha_s0_rep' -> 'seed 0 (repeat)'.""" seed = re.search(r"_s(\d+)", run).group(1) return "seed %s%s" % (seed, " (repeat)" if run.endswith("_rep") else "") def availability(found): parts = [] for arm in ARMS: steps = {} for k in found: if k[0] == arm: steps.setdefault(k[1], set()).add(k[2]) text = "; ".join("%s at %s" % (run_words(run), ", ".join(format(s, ",") for s in sorted(st))) for run, st in sorted(steps.items())) or "none yet" parts.append("**%s:** %s" % (NAMES[arm], text)) return "Checkpoints in the weights repository: " + " | ".join(parts) try: FOUND = list_checkpoints() except Exception as e: # noqa: BLE001 print("could not list the weights repository at start-up:", repr(e)[:200], flush=True) FOUND = {} LABELS, ORDER = catalog(FOUND) # ----------------------------------------------------------------------------- encoders # Start-up only downloads files and builds the tokenizer. Every model is built inside the GPU function, in the process that # runs it: nothing at import asks torch about a GPU (the start-up process has none, and the GPU worker inherits what it asked). TOKENIZER = AutoTokenizer.from_pretrained(TEXT_ENCODER, revision=TEXT_REVISION) snapshot_download(TEXT_ENCODER, revision=TEXT_REVISION, allow_patterns=["config.json", "model.safetensors"]) snapshot_download(VAE_REPO, revision=VAE_REVISION, allow_patterns=["vae/config.json", "vae/diffusion_pytorch_model.safetensors"]) _ENCODERS = {} # "t5" and "vae", built once per process _LOADED = {} # (arm, run, step, weights) -> (model, latent mean, latent std), this process only def encoders(device): """T5-base and the FLUX.2 autoencoder (from the files downloaded at start-up), on the device.""" if not _ENCODERS: _ENCODERS["t5"] = T5EncoderModel.from_pretrained(TEXT_ENCODER, revision=TEXT_REVISION).eval() _ENCODERS["vae"] = AutoencoderKLFlux2.from_pretrained(VAE_REPO, subfolder=VAE_SUBFOLDER, revision=VAE_REVISION, torch_dtype=torch.bfloat16).eval() return _ENCODERS["t5"].to(device), _ENCODERS["vae"].to(device) def fetch(key): """The checkpoint file on local disk (the Hub's cache). Called outside the GPU function.""" return hf_hub_download(REPO, FOUND[key]) def load_arm(key, path): """The model and its latent statistics (written into every milestone file by the training run).""" if key not in _LOADED: payload = torch.load(path, map_location="cpu", weights_only=True) model, _, _ = load_checkpoint(path, map_location="cpu") extra = payload.get("extra", {}) mean = torch.tensor(extra["latent_mean"], dtype=torch.float32).view(1, -1, 1, 1) std = torch.tensor(extra["latent_std"], dtype=torch.float32).view(1, -1, 1, 1) while len(_LOADED) >= KEEP_LOADED: _LOADED.pop(next(iter(_LOADED))) _LOADED[key] = (model.eval(), mean, std) return _LOADED[key] try: # the default pair's files, downloaded at start-up so the first picture does not wait for _label in defaults(FOUND): if _label != NONE: fetch(LABELS[_label]) except Exception as e: # noqa: BLE001 print("prefetch skipped:", repr(e)[:200], flush=True) # ----------------------------------------------------------------------------- pictures @torch.no_grad() def encode_prompt(prompt, t5, device): """T5-base states as in training: the true length from the attention mask, rounded through float16.""" t = TOKENIZER([prompt], padding=True, truncation=True, max_length=TEXT_MAX, return_tensors="pt") states = t5(input_ids=t["input_ids"].to(device), attention_mask=t["attention_mask"].to(device)).last_hidden_state n = int(t["attention_mask"].sum()) return states[:, :n].half().float().cpu(), torch.ones(1, n, dtype=torch.bool) def grid(images): """[n, 3, h, w] in [0, 1] -> one picture, two columns when n > 2.""" n, _, h, w = images.shape cols = 2 if n > 2 else n rows = (n + cols - 1) // cols canvas = torch.ones(3, rows * h, cols * w) for i in range(n): r, c = divmod(i, cols) canvas[:, r * h:(r + 1) * h, c * w:(c + 1) * w] = images[i] arr = (canvas.permute(1, 2, 0).clamp(0, 1).numpy() * 255.0).round().astype(np.uint8) return Image.fromarray(arr) class TowersOff: """Inside the model's velocity, the named towers' forces are zero; the model is restored on exit (also on an error).""" def __init__(self, model, names_off): kinds = list(getattr(model.config, "geometry_towers", ())) + list(getattr(model.config, "conv_towers", ())) off = set(names_off or ()) self.model, self.drop = model, [i for i, kind in enumerate(kinds) if kind in off] def __enter__(self): if self.drop: original, drop = self.model.tower_forces, list(self.drop) def tower_forces(h, c, text_mask, e): forces = original(h, c, text_mask, e) # [K, B, N, P] keep = torch.ones(forces.shape[0], device=forces.device, dtype=forces.dtype) keep[drop] = 0 return forces * keep.view(-1, *([1] * (forces.dim() - 1))) self.model.tower_forces = tower_forces # shadows the method for this model only return self.model def __exit__(self, *exc): if self.drop: del self.model.tower_forces return False @spaces.GPU(duration=45, size="large") def generate(prompt, jobs, seed, n, guidance, steps): """jobs: [(side name, key, local path, towers off)]. Every side starts from the same noises (seeds seed .. seed + n - 1).""" device = "cuda" if torch.cuda.is_available() else "cpu" out = {} with torch.no_grad(): t5, vae = encoders(device) text, mask = encode_prompt(prompt, t5, device) text, mask = text.repeat(n, 1, 1), mask.repeat(n, 1) noise = torch.stack([torch.randn(LATENT_SHAPE, generator=torch.Generator().manual_seed(seed + k)) for k in range(n)]) for name, key, path, off in jobs: model, mean, std = load_arm(key, path) model.to(device) with TowersOff(model, off): z = sample_latents(model, text, mask, noise, steps, guidance, device, use_amp=True, batch_size=8) z_raw = z * std + mean x = vae.decode(z_raw.to(device, torch.bfloat16)).sample.float() out[name] = grid(((x + 1.0) / 2.0).clamp(0, 1).cpu()) return out def side(label, towers_on): """(side name, checkpoint key, towers off). Switches count for Agatha only; the name carries the towers that are off.""" key = LABELS[label] on = TOWERS if towers_on is None else [t for t in TOWERS if t in towers_on] off = tuple(t for t in TOWERS if t not in on) if key[0] == "agatha" else () if not off: return label, key, off return "%s, %s" % (label, "all towers off" if len(off) == len(TOWERS) else "towers off: " + ", ".join(off)), key, off def run(prompt, a, b, seed, n, guidance, steps, blind, tally, towers_a=None, towers_b=None): prompt = (prompt or "").strip() if not prompt: raise gr.Error("Write a caption first.") picked = [(x, t) for x, t in ((a, towers_a), (b, towers_b)) if x and x != NONE] if not picked: raise gr.Error("Choose at least one checkpoint.") stale = [x for x, _ in picked if x not in LABELS] if stale: raise gr.Error("Not in the list any more: %s. Press Refresh." % ", ".join(stale)) sides = [side(x, t) for x, t in picked] jobs = {name: (key, off) for name, key, off in sides} images = generate(prompt, [(name, key, fetch(key), list(off)) for name, (key, off) in jobs.items()], int(seed), int(n), float(guidance), int(steps)) ignored = any(LABELS[x][0] != "agatha" and t is not None and len(t) < len(TOWERS) for x, t in picked) chosen = [name for name, _, _ in sides] if len(chosen) == 1: order = [chosen[0], None] note = "One checkpoint chosen: shown on the left." else: order = list(chosen) if blind: random.shuffle(order) note = "Which side is better? Pick, then reveal." if blind else "Shown by name (blind mode is off)." if ignored: note += " Lia, the control, has no towers: her switches change nothing." show = [("Left", "Right")[i] if (blind and len(chosen) == 2) else (x or "No checkpoint") for i, x in enumerate(order)] pictures = [images[x] if x else None for x in order] return (gr.update(value=pictures[0], label=show[0]), gr.update(value=pictures[1], label=show[1]), order, note, tally) def pick(side, order, tally): if not order or None in order: return "Generate a pair of two checkpoints first.", tally tally = dict(tally or {}) if side == "tie": tally["tie"] = tally.get("tie", 0) + 1 else: winner = order[0] if side == "left" else order[1] tally[winner] = tally.get(winner, 0) + 1 return "Counted. " + counts(tally), tally def counts(tally): tally = tally or {} wins = ["%s: %d" % (k, v) for k, v in sorted(tally.items(), key=lambda kv: -kv[1]) if k != "tie"] return "Picks this session: %s | ties: %d." % ("; ".join(wins) or "none yet", tally.get("tie", 0)) def reveal(order, tally): if not order: return "Generate first." return "Left = %s, right = %s. %s" % (order[0], order[1] or "no checkpoint", counts(tally)) def _relist(): global FOUND, LABELS, ORDER FOUND = list_checkpoints() LABELS, ORDER = catalog(FOUND) def refresh(a, b): """The Refresh button: re-read the list and keep each choice that is still in it.""" try: _relist() except Exception as e: # noqa: BLE001 return gr.update(), gr.update(), "Could not read the weights repository (%s); try again." % type(e).__name__ options = [NONE] + ORDER d = defaults(FOUND) a = a if a in options else d[0] b = b if b in options else d[1] return gr.update(choices=options, value=a), gr.update(choices=options, value=b), availability(FOUND) def load_page(): """A page load: the fresh list and the default pair (each model's newest checkpoint, None where it has none).""" try: _relist() except Exception: # noqa: BLE001 pass d = defaults(FOUND) options = [NONE] + ORDER return gr.update(choices=options, value=d[0]), gr.update(choices=options, value=d[1]), availability(FOUND) # ----------------------------------------------------------------------------- the page with gr.Blocks(title="Agatha diffusion playground") as demo: gr.Markdown("# Agatha diffusion playground\nAgatha and Lia, her control (a plain transformer of the same size; about " "42M parameters each), trained the same way on COCO 2014 at 256 px. Pick any two checkpoints (either model, any step, " "raw or averaged weights) or one with None on the other side; both sides start from the same noise. " "Non-commercial use only (trained on COCO photos).") d0 = defaults(FOUND) prompt = gr.Textbox(label="Caption", value=EXAMPLES[0]) with gr.Row(): ckpt_a = gr.Dropdown([NONE] + ORDER, value=d0[0], label="Checkpoint A") ckpt_b = gr.Dropdown([NONE] + ORDER, value=d0[1], label="Checkpoint B") refresh_btn = gr.Button("Refresh list", scale=0, min_width=140) gr.Markdown("Early in training the raw weights are sharper; the averaged (EMA) weights catch up later.") with gr.Row(): towers_a = gr.CheckboxGroup(list(TOWERS), value=list(TOWERS), label="Agatha's towers on, checkpoint A") towers_b = gr.CheckboxGroup(list(TOWERS), value=list(TOWERS), label="Agatha's towers on, checkpoint B") gr.Markdown("Untick a tower to switch its push off for that side; the trunk and the other towers stay as trained. " "cantor, beatrix, simplex and helix are attention towers, each with its own rotary position code; " "wide_resnet and frequency are convolution towers (frequency sees a high-passed input). The switches act on " "Agatha checkpoints only (Lia, the plain " "transformer, has no towers). Put the same checkpoint on both sides to compare tower settings.") with gr.Row(): seed = gr.Number(value=0, precision=0, label="Seed") n = gr.Slider(1, 4, value=4, step=1, label="Pictures per side") guidance = gr.Slider(1.0, 7.0, value=3.0, step=0.5, label="Guidance") steps = gr.Slider(10, 50, value=25, step=1, label="Sampling steps") blind = gr.Checkbox(value=True, label="Blind (hide which side is which)") go = gr.Button("Generate", variant="primary") gr.Examples(EXAMPLES, inputs=prompt) with gr.Row(): left = gr.Image(label="Left", type="pil", height=420) right = gr.Image(label="Right", type="pil", height=420) with gr.Row(): b_left = gr.Button("Left is better") b_tie = gr.Button("Tie") b_right = gr.Button("Right is better") b_reveal = gr.Button("Reveal") status = gr.Markdown() found_md = gr.Markdown(availability(FOUND)) order_state = gr.State([]) tally_state = gr.State({}) go.click(run, [prompt, ckpt_a, ckpt_b, seed, n, guidance, steps, blind, tally_state, towers_a, towers_b], [left, right, order_state, status, tally_state]) b_left.click(lambda o, t: pick("left", o, t), [order_state, tally_state], [status, tally_state]) b_right.click(lambda o, t: pick("right", o, t), [order_state, tally_state], [status, tally_state]) b_tie.click(lambda o, t: pick("tie", o, t), [order_state, tally_state], [status, tally_state]) b_reveal.click(reveal, [order_state, tally_state], status) refresh_btn.click(refresh, [ckpt_a, ckpt_b], [ckpt_a, ckpt_b, found_md]) demo.load(load_page, None, [ckpt_a, ckpt_b, found_md]) if __name__ == "__main__": demo.launch()