Spaces:
Running on Zero
Running on Zero
The control is named Lia: labels, notes and card; her design and name sources on the card
5027fd8 verified Download app.py from AbstractPhil/agatha-diffusion-playground: direct link, hf CLI and curl.
- Browser
- Download file 19.2 kB
-
https://huggingface.co/spaces/AbstractPhil/agatha-diffusion-playground/resolve/main/app.py
- Command line
-
hf download hf://spaces/AbstractPhil/agatha-diffusion-playground/app.py
-
curl -L -o app.py https://huggingface.co/spaces/AbstractPhil/agatha-diffusion-playground/resolve/main/app.py
19.2 kB
| """Agatha diffusion playground: any two checkpoints of the training runs side by side, from the same noise. | |
| Weights: AbstractPhil/geofractal-agatha-diffusion-training, folder coco256/ (non-commercial: trained on COCO 2014 photos). | |
| Code: github.com/AbstractEyes/geofractal, src/geofractal/router/prefab/agatha_v2, installed at the pinned commit below. | |
| Sampling is the library's own sample_latents (the helper the training's picture sheets use), so a picture here matches a | |
| sheet made at the same checkpoint, caption, seed, guidance and step count. | |
| Tower switches (Agatha only): an unticked tower's force is set to zero inside the velocity for that side's pictures; the | |
| trunk, the oscillator, the band weights and the other towers stay as trained, and the model is restored afterwards. | |
| """ | |
| import random | |
| import re | |
| import subprocess | |
| import sys | |
| import spaces # first, before anything imports torch: ZeroGPU patches torch at this import | |
| GEOFRACTAL ="git+https://github.com/AbstractEyes/geofractal@46fabb02ec851e4a4fb03b3ada30185a26efe2f7" | |
| GEOVOCAB = "git+https://github.com/AbstractEyes/lattice_vocabulary@2982a5c2f9dcd71a496468086d45083513deecf4" | |
| WIDE_COMPILER = "git+https://github.com/AbstractEyes/pytorch-parallel-compiler@70bcba4f2cdddb26909df59856eb6afead753826" | |
| def _install_library(): | |
| """The library at its pinned commit, without dependencies (they are in requirements.txt).""" | |
| try: | |
| import geofractal.router.prefab.agatha_v2.hooks # noqa: F401 | |
| return | |
| except Exception: # noqa: BLE001 | |
| pass | |
| for spec in (GEOVOCAB, WIDE_COMPILER, GEOFRACTAL): | |
| subprocess.run([sys.executable, "-m", "pip", "install", "-q", "--no-deps", spec], check=True) | |
| _install_library() | |
| import gradio as gr # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| import torch # noqa: E402 | |
| from diffusers import AutoencoderKLFlux2 # noqa: E402 | |
| from huggingface_hub import HfApi, hf_hub_download, snapshot_download # noqa: E402 | |
| from PIL import Image # noqa: E402 | |
| from transformers import AutoTokenizer, T5EncoderModel # noqa: E402 | |
| from geofractal.router.prefab.agatha_v2.hooks import sample_latents # noqa: E402 | |
| from geofractal.router.prefab.agatha_v2.trainer import load_checkpoint # noqa: E402 | |
| REPO = "AbstractPhil/geofractal-agatha-diffusion-training" | |
| # The encoders at the revisions the training data was encoded with. | |
| VAE_REPO, VAE_SUBFOLDER = "black-forest-labs/FLUX.2-klein-4B", "vae" | |
| VAE_REVISION = "e7b7dc27f91deacad38e78976d1f2b499d76a294" | |
| TEXT_ENCODER, TEXT_REVISION, TEXT_MAX = "google-t5/t5-base", "a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1", 128 | |
| LATENT_SHAPE = (32, 32, 32) | |
| ARMS = ("agatha", "plain") | |
| NAMES = {"agatha": "Agatha", "plain": "Lia"} # Lia = the control, a plain transformer; her runs keep "plain" in their names | |
| NONE = "None" | |
| FILE_RE = re.compile(r"^coco256/(agatha_v2_coco256_(agatha|plain)_s(\d+)(_rep)?)/\1_step(\d+)_(raw|ema)\.pt$") | |
| KEEP_LOADED = 8 # checkpoints kept in memory per process (each about 170 MB) | |
| # Agatha's six towers in the model's own order (geometry towers, then convolution towers); a checkpoint's config decides | |
| # which index each name has when a switch is applied. | |
| TOWERS = ("cantor", "beatrix", "simplex", "helix", "wide_resnet", "frequency") | |
| EXAMPLES = ["A man riding a bicycle down a city street.", "A herd of elephants walking through tall grass.", | |
| "A baseball player swinging a bat at a ball.", "A plate of food with broccoli and rice.", | |
| "A red double decker bus driving down a street.", "Two dogs playing in the snow.", | |
| "A kitchen with wooden cabinets and a white stove.", "A person riding a wave on a surfboard."] | |
| # ----------------------------------------------------------------------------- the checkpoint list | |
| def list_checkpoints(): | |
| """{(arm, run, step, weights): path} for every milestone file in the weights repository.""" | |
| out = {} | |
| for path in HfApi().list_repo_files(REPO): | |
| m = FILE_RE.match(path) | |
| if m: | |
| out[(m.group(2), m.group(1), int(m.group(5)), m.group(6))] = path | |
| return out | |
| def label_of(key): | |
| arm, run, step, weights = key | |
| return "%s, %s, step %s, %s" % (NAMES[arm], run_words(run), format(step, ","), weights) | |
| def ordered(found): | |
| """Checkpoints by model, then run, newest step first, raw before averaged.""" | |
| return sorted(found, key=lambda k: (ARMS.index(k[0]), k[1], -k[2], k[3] != "raw")) | |
| def catalog(found): | |
| """({label: key}, [labels in order]).""" | |
| keys = ordered(found) | |
| return {label_of(k): k for k in keys}, [label_of(k) for k in keys] | |
| def defaults(found): | |
| """The newest raw checkpoint of each model (seed 0 first); None for a model with nothing yet.""" | |
| out = [] | |
| for arm in ARMS: | |
| keys = [k for k in ordered(found) if k[0] == arm and k[3] == "raw"] | |
| out.append(label_of(keys[0]) if keys else NONE) | |
| return out | |
| def run_words(run): | |
| """'agatha_v2_coco256_agatha_s0_rep' -> 'seed 0 (repeat)'.""" | |
| seed = re.search(r"_s(\d+)", run).group(1) | |
| return "seed %s%s" % (seed, " (repeat)" if run.endswith("_rep") else "") | |
| def availability(found): | |
| parts = [] | |
| for arm in ARMS: | |
| steps = {} | |
| for k in found: | |
| if k[0] == arm: | |
| steps.setdefault(k[1], set()).add(k[2]) | |
| text = "; ".join("%s at %s" % (run_words(run), ", ".join(format(s, ",") for s in sorted(st))) | |
| for run, st in sorted(steps.items())) or "none yet" | |
| parts.append("**%s:** %s" % (NAMES[arm], text)) | |
| return "Checkpoints in the weights repository: " + " | ".join(parts) | |
| try: | |
| FOUND = list_checkpoints() | |
| except Exception as e: # noqa: BLE001 | |
| print("could not list the weights repository at start-up:", repr(e)[:200], flush=True) | |
| FOUND = {} | |
| LABELS, ORDER = catalog(FOUND) | |
| # ----------------------------------------------------------------------------- encoders | |
| # Start-up only downloads files and builds the tokenizer. Every model is built inside the GPU function, in the process that | |
| # runs it: nothing at import asks torch about a GPU (the start-up process has none, and the GPU worker inherits what it asked). | |
| TOKENIZER = AutoTokenizer.from_pretrained(TEXT_ENCODER, revision=TEXT_REVISION) | |
| snapshot_download(TEXT_ENCODER, revision=TEXT_REVISION, allow_patterns=["config.json", "model.safetensors"]) | |
| snapshot_download(VAE_REPO, revision=VAE_REVISION, | |
| allow_patterns=["vae/config.json", "vae/diffusion_pytorch_model.safetensors"]) | |
| _ENCODERS = {} # "t5" and "vae", built once per process | |
| _LOADED = {} # (arm, run, step, weights) -> (model, latent mean, latent std), this process only | |
| def encoders(device): | |
| """T5-base and the FLUX.2 autoencoder (from the files downloaded at start-up), on the device.""" | |
| if not _ENCODERS: | |
| _ENCODERS["t5"] = T5EncoderModel.from_pretrained(TEXT_ENCODER, revision=TEXT_REVISION).eval() | |
| _ENCODERS["vae"] = AutoencoderKLFlux2.from_pretrained(VAE_REPO, subfolder=VAE_SUBFOLDER, revision=VAE_REVISION, | |
| torch_dtype=torch.bfloat16).eval() | |
| return _ENCODERS["t5"].to(device), _ENCODERS["vae"].to(device) | |
| def fetch(key): | |
| """The checkpoint file on local disk (the Hub's cache). Called outside the GPU function.""" | |
| return hf_hub_download(REPO, FOUND[key]) | |
| def load_arm(key, path): | |
| """The model and its latent statistics (written into every milestone file by the training run).""" | |
| if key not in _LOADED: | |
| payload = torch.load(path, map_location="cpu", weights_only=True) | |
| model, _, _ = load_checkpoint(path, map_location="cpu") | |
| extra = payload.get("extra", {}) | |
| mean = torch.tensor(extra["latent_mean"], dtype=torch.float32).view(1, -1, 1, 1) | |
| std = torch.tensor(extra["latent_std"], dtype=torch.float32).view(1, -1, 1, 1) | |
| while len(_LOADED) >= KEEP_LOADED: | |
| _LOADED.pop(next(iter(_LOADED))) | |
| _LOADED[key] = (model.eval(), mean, std) | |
| return _LOADED[key] | |
| try: # the default pair's files, downloaded at start-up so the first picture does not wait | |
| for _label in defaults(FOUND): | |
| if _label != NONE: | |
| fetch(LABELS[_label]) | |
| except Exception as e: # noqa: BLE001 | |
| print("prefetch skipped:", repr(e)[:200], flush=True) | |
| # ----------------------------------------------------------------------------- pictures | |
| def encode_prompt(prompt, t5, device): | |
| """T5-base states as in training: the true length from the attention mask, rounded through float16.""" | |
| t = TOKENIZER([prompt], padding=True, truncation=True, max_length=TEXT_MAX, return_tensors="pt") | |
| states = t5(input_ids=t["input_ids"].to(device), attention_mask=t["attention_mask"].to(device)).last_hidden_state | |
| n = int(t["attention_mask"].sum()) | |
| return states[:, :n].half().float().cpu(), torch.ones(1, n, dtype=torch.bool) | |
| def grid(images): | |
| """[n, 3, h, w] in [0, 1] -> one picture, two columns when n > 2.""" | |
| n, _, h, w = images.shape | |
| cols = 2 if n > 2 else n | |
| rows = (n + cols - 1) // cols | |
| canvas = torch.ones(3, rows * h, cols * w) | |
| for i in range(n): | |
| r, c = divmod(i, cols) | |
| canvas[:, r * h:(r + 1) * h, c * w:(c + 1) * w] = images[i] | |
| arr = (canvas.permute(1, 2, 0).clamp(0, 1).numpy() * 255.0).round().astype(np.uint8) | |
| return Image.fromarray(arr) | |
| class TowersOff: | |
| """Inside the model's velocity, the named towers' forces are zero; the model is restored on exit (also on an error).""" | |
| def __init__(self, model, names_off): | |
| kinds = list(getattr(model.config, "geometry_towers", ())) + list(getattr(model.config, "conv_towers", ())) | |
| off = set(names_off or ()) | |
| self.model, self.drop = model, [i for i, kind in enumerate(kinds) if kind in off] | |
| def __enter__(self): | |
| if self.drop: | |
| original, drop = self.model.tower_forces, list(self.drop) | |
| def tower_forces(h, c, text_mask, e): | |
| forces = original(h, c, text_mask, e) # [K, B, N, P] | |
| keep = torch.ones(forces.shape[0], device=forces.device, dtype=forces.dtype) | |
| keep[drop] = 0 | |
| return forces * keep.view(-1, *([1] * (forces.dim() - 1))) | |
| self.model.tower_forces = tower_forces # shadows the method for this model only | |
| return self.model | |
| def __exit__(self, *exc): | |
| if self.drop: | |
| del self.model.tower_forces | |
| return False | |
| def generate(prompt, jobs, seed, n, guidance, steps): | |
| """jobs: [(side name, key, local path, towers off)]. Every side starts from the same noises (seeds seed .. seed + n - 1).""" | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| out = {} | |
| with torch.no_grad(): | |
| t5, vae = encoders(device) | |
| text, mask = encode_prompt(prompt, t5, device) | |
| text, mask = text.repeat(n, 1, 1), mask.repeat(n, 1) | |
| noise = torch.stack([torch.randn(LATENT_SHAPE, generator=torch.Generator().manual_seed(seed + k)) | |
| for k in range(n)]) | |
| for name, key, path, off in jobs: | |
| model, mean, std = load_arm(key, path) | |
| model.to(device) | |
| with TowersOff(model, off): | |
| z = sample_latents(model, text, mask, noise, steps, guidance, device, use_amp=True, batch_size=8) | |
| z_raw = z * std + mean | |
| x = vae.decode(z_raw.to(device, torch.bfloat16)).sample.float() | |
| out[name] = grid(((x + 1.0) / 2.0).clamp(0, 1).cpu()) | |
| return out | |
| def side(label, towers_on): | |
| """(side name, checkpoint key, towers off). Switches count for Agatha only; the name carries the towers that are off.""" | |
| key = LABELS[label] | |
| on = TOWERS if towers_on is None else [t for t in TOWERS if t in towers_on] | |
| off = tuple(t for t in TOWERS if t not in on) if key[0] == "agatha" else () | |
| if not off: | |
| return label, key, off | |
| return "%s, %s" % (label, "all towers off" if len(off) == len(TOWERS) else "towers off: " + ", ".join(off)), key, off | |
| def run(prompt, a, b, seed, n, guidance, steps, blind, tally, towers_a=None, towers_b=None): | |
| prompt = (prompt or "").strip() | |
| if not prompt: | |
| raise gr.Error("Write a caption first.") | |
| picked = [(x, t) for x, t in ((a, towers_a), (b, towers_b)) if x and x != NONE] | |
| if not picked: | |
| raise gr.Error("Choose at least one checkpoint.") | |
| stale = [x for x, _ in picked if x not in LABELS] | |
| if stale: | |
| raise gr.Error("Not in the list any more: %s. Press Refresh." % ", ".join(stale)) | |
| sides = [side(x, t) for x, t in picked] | |
| jobs = {name: (key, off) for name, key, off in sides} | |
| images = generate(prompt, [(name, key, fetch(key), list(off)) for name, (key, off) in jobs.items()], | |
| int(seed), int(n), float(guidance), int(steps)) | |
| ignored = any(LABELS[x][0] != "agatha" and t is not None and len(t) < len(TOWERS) for x, t in picked) | |
| chosen = [name for name, _, _ in sides] | |
| if len(chosen) == 1: | |
| order = [chosen[0], None] | |
| note = "One checkpoint chosen: shown on the left." | |
| else: | |
| order = list(chosen) | |
| if blind: | |
| random.shuffle(order) | |
| note = "Which side is better? Pick, then reveal." if blind else "Shown by name (blind mode is off)." | |
| if ignored: | |
| note += " Lia, the control, has no towers: her switches change nothing." | |
| show = [("Left", "Right")[i] if (blind and len(chosen) == 2) else (x or "No checkpoint") for i, x in enumerate(order)] | |
| pictures = [images[x] if x else None for x in order] | |
| return (gr.update(value=pictures[0], label=show[0]), gr.update(value=pictures[1], label=show[1]), | |
| order, note, tally) | |
| def pick(side, order, tally): | |
| if not order or None in order: | |
| return "Generate a pair of two checkpoints first.", tally | |
| tally = dict(tally or {}) | |
| if side == "tie": | |
| tally["tie"] = tally.get("tie", 0) + 1 | |
| else: | |
| winner = order[0] if side == "left" else order[1] | |
| tally[winner] = tally.get(winner, 0) + 1 | |
| return "Counted. " + counts(tally), tally | |
| def counts(tally): | |
| tally = tally or {} | |
| wins = ["%s: %d" % (k, v) for k, v in sorted(tally.items(), key=lambda kv: -kv[1]) if k != "tie"] | |
| return "Picks this session: %s | ties: %d." % ("; ".join(wins) or "none yet", tally.get("tie", 0)) | |
| def reveal(order, tally): | |
| if not order: | |
| return "Generate first." | |
| return "Left = %s, right = %s. %s" % (order[0], order[1] or "no checkpoint", counts(tally)) | |
| def _relist(): | |
| global FOUND, LABELS, ORDER | |
| FOUND = list_checkpoints() | |
| LABELS, ORDER = catalog(FOUND) | |
| def refresh(a, b): | |
| """The Refresh button: re-read the list and keep each choice that is still in it.""" | |
| try: | |
| _relist() | |
| except Exception as e: # noqa: BLE001 | |
| return gr.update(), gr.update(), "Could not read the weights repository (%s); try again." % type(e).__name__ | |
| options = [NONE] + ORDER | |
| d = defaults(FOUND) | |
| a = a if a in options else d[0] | |
| b = b if b in options else d[1] | |
| return gr.update(choices=options, value=a), gr.update(choices=options, value=b), availability(FOUND) | |
| def load_page(): | |
| """A page load: the fresh list and the default pair (each model's newest checkpoint, None where it has none).""" | |
| try: | |
| _relist() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| d = defaults(FOUND) | |
| options = [NONE] + ORDER | |
| return gr.update(choices=options, value=d[0]), gr.update(choices=options, value=d[1]), availability(FOUND) | |
| # ----------------------------------------------------------------------------- the page | |
| with gr.Blocks(title="Agatha diffusion playground") as demo: | |
| gr.Markdown("# Agatha diffusion playground\nAgatha and Lia, her control (a plain transformer of the same size; about " | |
| "42M parameters each), trained the same way on COCO 2014 at 256 px. Pick any two checkpoints (either model, any step, " | |
| "raw or averaged weights) or one with None on the other side; both sides start from the same noise. " | |
| "Non-commercial use only (trained on COCO photos).") | |
| d0 = defaults(FOUND) | |
| prompt = gr.Textbox(label="Caption", value=EXAMPLES[0]) | |
| with gr.Row(): | |
| ckpt_a = gr.Dropdown([NONE] + ORDER, value=d0[0], label="Checkpoint A") | |
| ckpt_b = gr.Dropdown([NONE] + ORDER, value=d0[1], label="Checkpoint B") | |
| refresh_btn = gr.Button("Refresh list", scale=0, min_width=140) | |
| gr.Markdown("Early in training the raw weights are sharper; the averaged (EMA) weights catch up later.") | |
| with gr.Row(): | |
| towers_a = gr.CheckboxGroup(list(TOWERS), value=list(TOWERS), label="Agatha's towers on, checkpoint A") | |
| towers_b = gr.CheckboxGroup(list(TOWERS), value=list(TOWERS), label="Agatha's towers on, checkpoint B") | |
| gr.Markdown("Untick a tower to switch its push off for that side; the trunk and the other towers stay as trained. " | |
| "cantor, beatrix, simplex and helix are attention towers, each with its own rotary position code; " | |
| "wide_resnet and frequency are convolution towers (frequency sees a high-passed input). The switches act on " | |
| "Agatha checkpoints only (Lia, the plain " | |
| "transformer, has no towers). Put the same checkpoint on both sides to compare tower settings.") | |
| with gr.Row(): | |
| seed = gr.Number(value=0, precision=0, label="Seed") | |
| n = gr.Slider(1, 4, value=4, step=1, label="Pictures per side") | |
| guidance = gr.Slider(1.0, 7.0, value=3.0, step=0.5, label="Guidance") | |
| steps = gr.Slider(10, 50, value=25, step=1, label="Sampling steps") | |
| blind = gr.Checkbox(value=True, label="Blind (hide which side is which)") | |
| go = gr.Button("Generate", variant="primary") | |
| gr.Examples(EXAMPLES, inputs=prompt) | |
| with gr.Row(): | |
| left = gr.Image(label="Left", type="pil", height=420) | |
| right = gr.Image(label="Right", type="pil", height=420) | |
| with gr.Row(): | |
| b_left = gr.Button("Left is better") | |
| b_tie = gr.Button("Tie") | |
| b_right = gr.Button("Right is better") | |
| b_reveal = gr.Button("Reveal") | |
| status = gr.Markdown() | |
| found_md = gr.Markdown(availability(FOUND)) | |
| order_state = gr.State([]) | |
| tally_state = gr.State({}) | |
| go.click(run, [prompt, ckpt_a, ckpt_b, seed, n, guidance, steps, blind, tally_state, towers_a, towers_b], | |
| [left, right, order_state, status, tally_state]) | |
| b_left.click(lambda o, t: pick("left", o, t), [order_state, tally_state], [status, tally_state]) | |
| b_right.click(lambda o, t: pick("right", o, t), [order_state, tally_state], [status, tally_state]) | |
| b_tie.click(lambda o, t: pick("tie", o, t), [order_state, tally_state], [status, tally_state]) | |
| b_reveal.click(reveal, [order_state, tally_state], status) | |
| refresh_btn.click(refresh, [ckpt_a, ckpt_b], [ckpt_a, ckpt_b, found_md]) | |
| demo.load(load_page, None, [ckpt_a, ckpt_b, found_md]) | |
| if __name__ == "__main__": | |
| demo.launch() | |