Leeps commited on
Commit
9084ca7
·
1 Parent(s): b1ced3d

Simplify prompt blending workshop UI

Browse files
Files changed (2) hide show
  1. README.md +4 -5
  2. app.py +123 -366
README.md CHANGED
@@ -19,12 +19,11 @@ preload_from_hub:
19
 
20
  A ZeroGPU-ready Gradio app for learning how Stable Diffusion works inside Diffusers.
21
 
22
- Instead of only calling `pipe(prompt)`, the app exposes a small custom denoising loop:
23
 
24
- - prompt embeddings can be used directly, averaged, or combined with vector arithmetic
25
- - three prompt embeddings can be explored as a clickable triangle between concepts
26
- - initial latent noise can come from one seed or a blend of two seeds
27
- - classifier-free guidance can use standard CFG or a student-edited equation
28
  - intermediate latent snapshots show how the image emerges across denoising steps
29
 
30
  The core idea to teach: Stable Diffusion starts from noise in VAE latent space. The prompt does not literally become half of the pixels. The prompt changes the UNet's predicted noise at every denoising step, most commonly through classifier-free guidance:
 
19
 
20
  A ZeroGPU-ready Gradio app for learning how Stable Diffusion works inside Diffusers.
21
 
22
+ Instead of only calling `pipe(prompt)`, the app exposes a beginner-friendly custom denoising loop:
23
 
24
+ - three prompt embeddings can be blended with simple strength sliders
25
+ - the live equation shows the exact weighted embedding blend being used
26
+ - a few Diffusers levers are exposed: seed, denoising steps, prompt guidance, and optional noise mixing
 
27
  - intermediate latent snapshots show how the image emerges across denoising steps
28
 
29
  The core idea to teach: Stable Diffusion starts from noise in VAE latent space. The prompt does not literally become half of the pixels. The prompt changes the UNet's predicted noise at every denoising step, most commonly through classifier-free guidance:
app.py CHANGED
@@ -22,45 +22,43 @@ import numpy as np
22
  import torch
23
  from PIL import Image, ImageDraw
24
 
25
- from diffusers import DDIMScheduler, DPMSolverMultistepScheduler, StableDiffusionPipeline
26
 
27
 
28
  APP_TITLE = "Stable Diffusion Equation Playground"
29
  DEFAULT_MODEL = "stable-diffusion-v1-5/stable-diffusion-v1-5"
30
- DEFAULT_PROMPT_A = "a small glass greenhouse filled with glowing ferns, watercolor"
31
- DEFAULT_PROMPT_B = "a futuristic train station at sunrise, cinematic lighting"
32
- DEFAULT_PROMPT_C = "low quality, blurry, distorted"
 
33
  MAX_SEED = 2_147_483_647
34
 
35
- PROMPT_MATH_CODE = """# Diffusers usually does this inside pipe(prompt).
36
- # Here we expose the text embeddings so we can edit them directly.
37
- prompt_a, negative = encode_prompt(prompt_a, negative_prompt)
38
- prompt_b, _ = encode_prompt(prompt_b, negative_prompt)
39
- prompt_c, _ = encode_prompt(prompt_c, negative_prompt)
40
-
41
- if mode == "average":
42
- prompt_embeds = (1 - mix) * prompt_a + mix * prompt_b
43
- elif mode == "triangle":
44
- total = weight_a + weight_b + weight_c
45
- prompt_embeds = (
46
- (weight_a / total) * prompt_a
47
- + (weight_b / total) * prompt_b
48
- + (weight_c / total) * prompt_c
49
- )
50
- elif mode == "analogy":
51
- prompt_embeds = prompt_a + strength * (prompt_b - prompt_c)
52
  else:
53
- prompt_embeds = prompt_a
 
 
 
 
54
  """
55
 
56
  LATENT_MATH_CODE = """# Stable Diffusion does not start from pixels.
57
  # It starts from noisy latents in the VAE's compressed image space.
58
  noise_a = torch.randn(latent_shape, generator=seed_a)
59
  noise_b = torch.randn(latent_shape, generator=seed_b)
60
- latents = (1 - noise_mix) * noise_a + noise_mix * noise_b
61
 
62
- if renormalize_noise:
63
- latents = (latents - latents.mean()) / latents.std()
 
64
 
65
  latents = latents * scheduler.init_noise_sigma
66
  """
@@ -72,9 +70,7 @@ delta = noise_prompt - noise_negative
72
 
73
  # Standard CFG is:
74
  # guided = noise_negative + guidance_scale * delta
75
- guided = negative_coeff * noise_negative
76
- guided = guided + prompt_coeff * noise_prompt
77
- guided = guided + delta_coeff * delta
78
  """
79
 
80
 
@@ -127,130 +123,43 @@ def blank_image(message="Run generation to make an image."):
127
  return image
128
 
129
 
130
- TRIANGLE_SIZE = 360
131
- TRIANGLE_A = (180, 32)
132
- TRIANGLE_B = (42, 300)
133
- TRIANGLE_C = (318, 300)
134
-
135
-
136
- def normalized_triangle_weights(weight_a, weight_b, weight_c):
137
  weights = [max(0.0, float(weight_a)), max(0.0, float(weight_b)), max(0.0, float(weight_c))]
138
  total = sum(weights)
139
  if total <= 0:
140
- return 1.0, 0.0, 0.0
141
  return tuple(weight / total for weight in weights)
142
 
143
 
144
- def weighted_triangle_point(weight_a, weight_b, weight_c):
145
- weight_a, weight_b, weight_c = normalized_triangle_weights(weight_a, weight_b, weight_c)
146
- x = weight_a * TRIANGLE_A[0] + weight_b * TRIANGLE_B[0] + weight_c * TRIANGLE_C[0]
147
- y = weight_a * TRIANGLE_A[1] + weight_b * TRIANGLE_B[1] + weight_c * TRIANGLE_C[1]
148
- return int(round(x)), int(round(y))
149
-
150
-
151
- def short_corner_label(text, fallback):
152
  text = " ".join(str(text or fallback).split())
153
- return text[:24] + ("..." if len(text) > 24 else "")
154
-
155
-
156
- def make_triangle_picker(prompt_a, prompt_b, prompt_c, weight_a=1 / 3, weight_b=1 / 3, weight_c=1 / 3):
157
- image = Image.new("RGB", (TRIANGLE_SIZE, TRIANGLE_SIZE), (248, 250, 252))
158
- draw = ImageDraw.Draw(image)
159
- vertices = [TRIANGLE_A, TRIANGLE_B, TRIANGLE_C]
160
-
161
- draw.polygon(vertices, fill=(235, 244, 255), outline=(43, 74, 111))
162
- for i in range(1, 7):
163
- t = i / 7
164
- left = (
165
- int((1 - t) * TRIANGLE_A[0] + t * TRIANGLE_B[0]),
166
- int((1 - t) * TRIANGLE_A[1] + t * TRIANGLE_B[1]),
167
- )
168
- right = (
169
- int((1 - t) * TRIANGLE_A[0] + t * TRIANGLE_C[0]),
170
- int((1 - t) * TRIANGLE_A[1] + t * TRIANGLE_C[1]),
171
- )
172
- draw.line((left, right), fill=(190, 207, 225), width=1)
173
-
174
- bottom = (
175
- int((1 - t) * TRIANGLE_B[0] + t * TRIANGLE_C[0]),
176
- int((1 - t) * TRIANGLE_B[1] + t * TRIANGLE_C[1]),
177
- )
178
- left_side = (
179
- int((1 - t) * TRIANGLE_A[0] + t * TRIANGLE_B[0]),
180
- int((1 - t) * TRIANGLE_A[1] + t * TRIANGLE_B[1]),
181
- )
182
- right_side = (
183
- int((1 - t) * TRIANGLE_A[0] + t * TRIANGLE_C[0]),
184
- int((1 - t) * TRIANGLE_A[1] + t * TRIANGLE_C[1]),
185
- )
186
- draw.line((TRIANGLE_B, right_side), fill=(214, 224, 236), width=1)
187
- draw.line((TRIANGLE_C, left_side), fill=(214, 224, 236), width=1)
188
- draw.line((bottom, TRIANGLE_A), fill=(214, 224, 236), width=1)
189
-
190
- labels = [
191
- (TRIANGLE_A, "A", short_corner_label(prompt_a, "Prompt A"), (55, 94, 151)),
192
- (TRIANGLE_B, "B", short_corner_label(prompt_b, "Prompt B"), (5, 122, 85)),
193
- (TRIANGLE_C, "C", short_corner_label(prompt_c, "Prompt C"), (154, 72, 174)),
194
- ]
195
- for (x, y), letter, label, color in labels:
196
- draw.ellipse((x - 13, y - 13, x + 13, y + 13), fill=color, outline=(255, 255, 255), width=3)
197
- draw.text((x - 4, y - 7), letter, fill=(255, 255, 255))
198
- label_x = max(8, min(TRIANGLE_SIZE - 150, x - 70))
199
- label_y = y - 34 if y < TRIANGLE_SIZE / 2 else y + 18
200
- draw.text((label_x, label_y), label, fill=(30, 41, 59))
201
-
202
- x, y = weighted_triangle_point(weight_a, weight_b, weight_c)
203
- draw.ellipse((x - 9, y - 9, x + 9, y + 9), fill=(239, 68, 68), outline=(15, 23, 42), width=2)
204
- draw.text((12, 12), "Click inside the triangle to choose the embedding blend.", fill=(51, 65, 85))
205
- return image
206
-
207
-
208
- def barycentric_triangle_weights(x, y):
209
- ax, ay = TRIANGLE_A
210
- bx, by = TRIANGLE_B
211
- cx, cy = TRIANGLE_C
212
- denominator = (by - cy) * (ax - cx) + (cx - bx) * (ay - cy)
213
- if denominator == 0:
214
- return 1.0, 0.0, 0.0
215
-
216
- weight_a = ((by - cy) * (x - cx) + (cx - bx) * (y - cy)) / denominator
217
- weight_b = ((cy - ay) * (x - cx) + (ax - cx) * (y - cy)) / denominator
218
- weight_c = 1.0 - weight_a - weight_b
219
-
220
- if min(weight_a, weight_b, weight_c) < 0:
221
- weight_a, weight_b, weight_c = normalized_triangle_weights(weight_a, weight_b, weight_c)
222
-
223
- return normalized_triangle_weights(weight_a, weight_b, weight_c)
224
-
225
 
226
- def triangle_status(weight_a, weight_b, weight_c):
227
- weight_a, weight_b, weight_c = normalized_triangle_weights(weight_a, weight_b, weight_c)
228
- return f"A: {weight_a:.2f} B: {weight_b:.2f} C: {weight_c:.2f}"
229
 
230
-
231
- def update_triangle_from_weights(prompt_a, prompt_b, prompt_c, weight_a, weight_b, weight_c):
232
- weight_a, weight_b, weight_c = normalized_triangle_weights(weight_a, weight_b, weight_c)
233
  return (
234
- make_triangle_picker(prompt_a, prompt_b, prompt_c, weight_a, weight_b, weight_c),
235
- triangle_status(weight_a, weight_b, weight_c),
236
  weight_a,
237
  weight_b,
238
  weight_c,
239
  )
240
 
241
 
242
- def select_triangle_point(prompt_a, prompt_b, prompt_c, evt: gr.SelectData):
243
- index = evt.index
244
- if isinstance(index, dict):
245
- x, y = index.get("x", TRIANGLE_A[0]), index.get("y", TRIANGLE_A[1])
246
- else:
247
- x, y = index[:2]
248
- weight_a, weight_b, weight_c = barycentric_triangle_weights(float(x), float(y))
249
- return update_triangle_from_weights(prompt_a, prompt_b, prompt_c, weight_a, weight_b, weight_c)
 
250
 
251
 
252
  @lru_cache(maxsize=2)
253
- def load_pipe(model_id, scheduler_name, device_type):
254
  device = torch.device(device_type)
255
  dtype = model_dtype(device)
256
  pipe = StableDiffusionPipeline.from_pretrained(
@@ -259,11 +168,7 @@ def load_pipe(model_id, scheduler_name, device_type):
259
  use_safetensors=True,
260
  )
261
 
262
- if scheduler_name == "DDIM":
263
- pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
264
- else:
265
- pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
266
-
267
  pipe = pipe.to(device)
268
  pipe.set_progress_bar_config(disable=True)
269
  pipe.enable_vae_slicing()
@@ -312,38 +217,17 @@ def mix_prompt_embeddings(
312
  prompt_b,
313
  prompt_c,
314
  negative_prompt,
315
- embedding_mode,
316
- prompt_mix,
317
- triangle_weight_a,
318
- triangle_weight_b,
319
- triangle_weight_c,
320
- analogy_strength,
321
  renormalize_prompt,
322
  ):
323
  emb_a, negative_embeds = encode_prompt(pipe, prompt_a, negative_prompt, device)
324
  emb_b, _ = encode_prompt(pipe, prompt_b, negative_prompt, device)
325
  emb_c, _ = encode_prompt(pipe, prompt_c or "", negative_prompt, device)
326
 
327
- if embedding_mode == "Average prompt A and B":
328
- mixed = (1.0 - prompt_mix) * emb_a + prompt_mix * emb_b
329
- formula = f"prompt = {(1.0 - prompt_mix):.2f} * A + {prompt_mix:.2f} * B"
330
- elif embedding_mode == "Triangle blend: A/B/C":
331
- triangle_weight_a, triangle_weight_b, triangle_weight_c = normalized_triangle_weights(
332
- triangle_weight_a,
333
- triangle_weight_b,
334
- triangle_weight_c,
335
- )
336
- mixed = triangle_weight_a * emb_a + triangle_weight_b * emb_b + triangle_weight_c * emb_c
337
- formula = (
338
- f"prompt = {triangle_weight_a:.2f} * A + {triangle_weight_b:.2f} * B "
339
- f"+ {triangle_weight_c:.2f} * C"
340
- )
341
- elif embedding_mode == "Vector arithmetic: A + s * (B - C)":
342
- mixed = emb_a + analogy_strength * (emb_b - emb_c)
343
- formula = f"prompt = A + {analogy_strength:.2f} * (B - C)"
344
- else:
345
- mixed = emb_a
346
- formula = "prompt = A"
347
 
348
  original_norm = emb_a.detach().float().norm()
349
  mixed_norm = mixed.detach().float().norm()
@@ -389,37 +273,10 @@ def prepare_latents(pipe, device, height, width, seed_a, seed_b, noise_mix, reno
389
  return latents, formula, metrics
390
 
391
 
392
- def combine_noise_predictions(
393
- noise_negative,
394
- noise_prompt,
395
- equation_mode,
396
- guidance_scale,
397
- prediction_mix,
398
- negative_coeff,
399
- prompt_coeff,
400
- delta_coeff,
401
- ):
402
  delta = noise_prompt - noise_negative
403
-
404
- if equation_mode == "Standard CFG: negative + scale * (prompt - negative)":
405
- guided = noise_negative + guidance_scale * delta
406
- formula = f"guided = negative + {guidance_scale:.2f} * (prompt - negative)"
407
- elif equation_mode == "Direct blend: (1 - mix) * negative + mix * prompt":
408
- guided = (1.0 - prediction_mix) * noise_negative + prediction_mix * noise_prompt
409
- formula = f"guided = {(1.0 - prediction_mix):.2f} * negative + {prediction_mix:.2f} * prompt"
410
- elif equation_mode == "Prompt prediction only":
411
- guided = noise_prompt
412
- formula = "guided = prompt"
413
- elif equation_mode == "Negative prediction only":
414
- guided = noise_negative
415
- formula = "guided = negative"
416
- else:
417
- guided = negative_coeff * noise_negative + prompt_coeff * noise_prompt + delta_coeff * delta
418
- formula = (
419
- f"guided = {negative_coeff:.2f} * negative + {prompt_coeff:.2f} * prompt "
420
- f"+ {delta_coeff:.2f} * (prompt - negative)"
421
- )
422
-
423
  return guided, formula
424
 
425
 
@@ -452,29 +309,17 @@ def gpu_duration(*args):
452
  @spaces.GPU(duration=gpu_duration)
453
  @torch.inference_mode()
454
  def generate(
455
- model_id,
456
- scheduler_name,
457
  prompt_a,
458
  prompt_b,
459
  prompt_c,
460
- negative_prompt,
461
- embedding_mode,
462
- prompt_mix,
463
- triangle_weight_a,
464
- triangle_weight_b,
465
- triangle_weight_c,
466
- analogy_strength,
467
- renormalize_prompt,
468
  seed_a,
469
  seed_b,
470
  noise_mix,
471
- renormalize_noise,
472
- equation_mode,
473
  guidance_scale,
474
- prediction_mix,
475
- negative_coeff,
476
- prompt_coeff,
477
- delta_coeff,
478
  num_steps,
479
  width,
480
  height,
@@ -486,13 +331,13 @@ def generate(
486
  [],
487
  "No GPU was detected. The app is designed for CUDA or MPS. It can run on CPU, but it may take a very long time.",
488
  [],
489
- [],
490
  )
491
 
492
  width = round_to_multiple_of_8(width)
493
  height = round_to_multiple_of_8(height)
494
  num_steps = int(num_steps)
495
- pipe = load_pipe(model_id.strip() or DEFAULT_MODEL, scheduler_name, device.type)
 
496
 
497
  prompt_embeds, negative_prompt_embeds, prompt_formula, prompt_metrics = mix_prompt_embeddings(
498
  pipe,
@@ -501,13 +346,10 @@ def generate(
501
  prompt_b or "",
502
  prompt_c or "",
503
  negative_prompt or "",
504
- embedding_mode,
505
- float(prompt_mix),
506
- float(triangle_weight_a),
507
- float(triangle_weight_b),
508
- float(triangle_weight_c),
509
- float(analogy_strength),
510
- bool(renormalize_prompt),
511
  )
512
 
513
  pipe.scheduler.set_timesteps(num_steps, device=device)
@@ -519,12 +361,11 @@ def generate(
519
  int(seed_a),
520
  int(seed_b),
521
  float(noise_mix),
522
- bool(renormalize_noise),
523
  )
524
 
525
  text_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])
526
  snapshots = []
527
- trace_rows = []
528
  save_at = checkpoint_indices(num_steps)
529
  last_formula = ""
530
 
@@ -539,30 +380,14 @@ def generate(
539
  return_dict=False,
540
  )[0]
541
  noise_negative, noise_prompt = noise_pred.chunk(2)
542
- guided, last_formula = combine_noise_predictions(
543
  noise_negative,
544
  noise_prompt,
545
- equation_mode,
546
  float(guidance_scale),
547
- float(prediction_mix),
548
- float(negative_coeff),
549
- float(prompt_coeff),
550
- float(delta_coeff),
551
  )
552
 
553
  latents = pipe.scheduler.step(guided, timestep, latents, return_dict=False)[0]
554
 
555
- delta_norm = float((noise_prompt - noise_negative).detach().float().norm().cpu())
556
- guided_norm = float(guided.detach().float().norm().cpu())
557
- trace_rows.append(
558
- [
559
- step_index + 1,
560
- int(timestep.detach().cpu()) if torch.is_tensor(timestep) else int(timestep),
561
- round(delta_norm, 3),
562
- round(guided_norm, 3),
563
- ]
564
- )
565
-
566
  if step_index in save_at:
567
  snapshot = decode_latents(pipe, latents)[0]
568
  snapshots.append((snapshot, f"step {step_index + 1} of {num_steps}"))
@@ -575,19 +400,18 @@ def generate(
575
  ["guidance formula", last_formula],
576
  ]
577
  summary = (
578
- f"Model: {model_id.strip() or DEFAULT_MODEL}\n"
579
- f"Scheduler: {scheduler_name}; steps: {num_steps}; size: {width}x{height}\n\n"
580
- f"Prompt embedding math:\n{prompt_formula}\n\n"
581
- f"Initial latent noise math:\n{latent_formula}\n\n"
582
- f"Guidance equation used at each denoising step:\n{last_formula}\n\n"
583
- "Stable Diffusion starts from noise in latent space. The prompt does not get averaged with pixels; "
584
- "it changes the UNet's predicted noise at every step."
585
  )
586
 
587
  if device.type == "cuda":
588
  torch.cuda.empty_cache()
589
  gc.collect()
590
- return final_image, snapshots, summary, metrics, trace_rows
591
 
592
 
593
  def randomize_seeds():
@@ -609,91 +433,46 @@ def build_app():
609
  """
610
 
611
  metric_headers = ["quantity", "value"]
612
- trace_headers = ["step", "timestep", "prompt minus negative norm", "guided prediction norm"]
613
 
614
  with gr.Blocks(title=APP_TITLE, theme=theme, css=css) as demo:
615
  gr.Markdown(
616
  f"# {APP_TITLE}\n"
617
- "Change the starting noise, average prompt embeddings, and edit the classifier-free guidance equation. "
618
- "This is a GPU-oriented teaching app built on Diffusers."
619
  )
 
 
 
 
 
620
 
621
  with gr.Row(equal_height=False):
622
  with gr.Column(scale=1, min_width=320):
623
- model_id = gr.Textbox(value=DEFAULT_MODEL, label="Diffusers model id")
624
- scheduler_name = gr.Radio(["DPM++ 2M", "DDIM"], value="DPM++ 2M", label="Scheduler")
625
-
626
- with gr.Accordion("Prompt embeddings", open=True):
627
- prompt_a = gr.Textbox(value=DEFAULT_PROMPT_A, label="Prompt A", lines=3)
628
- prompt_b = gr.Textbox(value=DEFAULT_PROMPT_B, label="Prompt B", lines=3)
629
- prompt_c = gr.Textbox(value=DEFAULT_PROMPT_C, label="Prompt C for vector arithmetic", lines=2)
 
 
 
 
 
 
 
630
  negative_prompt = gr.Textbox(
631
- value="blurry, low quality, distorted",
632
- label="Negative prompt",
633
- lines=2,
634
- )
635
- embedding_mode = gr.Dropdown(
636
- [
637
- "Prompt A only",
638
- "Average prompt A and B",
639
- "Triangle blend: A/B/C",
640
- "Vector arithmetic: A + s * (B - C)",
641
- ],
642
- value="Triangle blend: A/B/C",
643
- label="Embedding equation",
644
- )
645
- prompt_mix = gr.Slider(0, 1, value=0.5, step=0.05, label="Prompt B weight")
646
- triangle_picker = gr.Image(
647
- value=make_triangle_picker(DEFAULT_PROMPT_A, DEFAULT_PROMPT_B, DEFAULT_PROMPT_C),
648
- label="Triangle embedding mixer",
649
- type="pil",
650
- interactive=False,
651
- )
652
- triangle_status_box = gr.Textbox(
653
- value=triangle_status(1 / 3, 1 / 3, 1 / 3),
654
- label="Triangle weights",
655
- interactive=False,
656
  )
657
- with gr.Row():
658
- triangle_weight_a = gr.Slider(0, 1, value=1 / 3, step=0.01, label="A weight")
659
- triangle_weight_b = gr.Slider(0, 1, value=1 / 3, step=0.01, label="B weight")
660
- triangle_weight_c = gr.Slider(0, 1, value=1 / 3, step=0.01, label="C weight")
661
- analogy_strength = gr.Slider(-2, 2, value=0.8, step=0.1, label="Vector arithmetic strength")
662
- renormalize_prompt = gr.Checkbox(value=True, label="Keep mixed prompt embedding norm near prompt A")
663
 
664
- with gr.Accordion("Starting latent noise", open=True):
665
- with gr.Row():
666
- seed_a = gr.Number(value=11, precision=0, label="Seed A")
667
- seed_b = gr.Number(value=2222, precision=0, label="Seed B")
668
- random_seeds = gr.Button("Randomize seeds")
669
- noise_mix = gr.Slider(0, 1, value=0.0, step=0.05, label="Seed B noise weight")
670
- renormalize_noise = gr.Checkbox(value=True, label="Renormalize mixed noise")
671
-
672
- with gr.Accordion("Guidance equation", open=True):
673
- equation_mode = gr.Dropdown(
674
- [
675
- "Standard CFG: negative + scale * (prompt - negative)",
676
- "Direct blend: (1 - mix) * negative + mix * prompt",
677
- "Prompt prediction only",
678
- "Negative prediction only",
679
- "Custom coefficients",
680
- ],
681
- value="Standard CFG: negative + scale * (prompt - negative)",
682
- label="Noise prediction equation",
683
- )
684
- guidance_scale = gr.Slider(0, 15, value=7.5, step=0.25, label="CFG scale")
685
- prediction_mix = gr.Slider(0, 2, value=0.5, step=0.05, label="Direct blend prompt weight")
686
  with gr.Row():
687
- negative_coeff = gr.Slider(-3, 3, value=1, step=0.1, label="negative coeff")
688
- prompt_coeff = gr.Slider(-3, 3, value=0, step=0.1, label="prompt coeff")
689
- delta_coeff = gr.Slider(-3, 15, value=7.5, step=0.25, label="delta coeff")
690
 
691
- with gr.Accordion("Render settings", open=True):
692
- with gr.Row():
693
- width = gr.Slider(256, 768, value=512, step=8, label="Width")
694
- height = gr.Slider(256, 768, value=512, step=8, label="Height")
695
- num_steps = gr.Slider(4, 50, value=20, step=1, label="Denoising steps")
696
- generate_button = gr.Button("Generate", variant="primary")
697
 
698
  with gr.Column(scale=1, min_width=420):
699
  output_image = gr.Image(
@@ -703,27 +482,23 @@ def build_app():
703
  interactive=False,
704
  )
705
  summary = gr.Textbox(label="What happened", lines=12, interactive=False)
706
- snapshots = gr.Gallery(
707
- label="Decoded latent snapshots",
708
- columns=2,
709
- height=420,
710
- object_fit="contain",
711
- elem_classes=["snapshot-gallery"],
712
- )
713
- metrics = gr.Dataframe(
714
- headers=metric_headers,
715
- datatype=["str", "str"],
716
- label="Embedding and latent measurements",
717
- interactive=False,
718
- )
719
- trace = gr.Dataframe(
720
- headers=trace_headers,
721
- datatype=["number", "number", "number", "number"],
722
- label="Denoising trace",
723
- interactive=False,
724
- )
725
 
726
- with gr.Tab("Code cells"):
727
  with gr.Row(equal_height=False):
728
  gr.Code(PROMPT_MATH_CODE, language="python", label="Prompt embedding math", interactive=False, elem_classes=["code-panel"])
729
  gr.Code(LATENT_MATH_CODE, language="python", label="Latent noise math", interactive=False, elem_classes=["code-panel"])
@@ -735,50 +510,32 @@ def build_app():
735
  outputs=[seed_a, seed_b],
736
  show_progress="hidden",
737
  )
738
- triangle_picker.select(
739
- select_triangle_point,
740
- inputs=[prompt_a, prompt_b, prompt_c],
741
- outputs=[triangle_picker, triangle_status_box, triangle_weight_a, triangle_weight_b, triangle_weight_c],
742
- show_progress="hidden",
743
- )
744
- for triangle_input in [prompt_a, prompt_b, prompt_c, triangle_weight_a, triangle_weight_b, triangle_weight_c]:
745
- triangle_input.change(
746
- update_triangle_from_weights,
747
- inputs=[prompt_a, prompt_b, prompt_c, triangle_weight_a, triangle_weight_b, triangle_weight_c],
748
- outputs=[triangle_picker, triangle_status_box, triangle_weight_a, triangle_weight_b, triangle_weight_c],
749
  show_progress="hidden",
750
  )
751
  generate_button.click(
752
  generate,
753
  inputs=[
754
- model_id,
755
- scheduler_name,
756
  prompt_a,
757
  prompt_b,
758
  prompt_c,
759
- negative_prompt,
760
- embedding_mode,
761
- prompt_mix,
762
- triangle_weight_a,
763
- triangle_weight_b,
764
- triangle_weight_c,
765
- analogy_strength,
766
- renormalize_prompt,
767
  seed_a,
768
  seed_b,
769
  noise_mix,
770
- renormalize_noise,
771
- equation_mode,
772
  guidance_scale,
773
- prediction_mix,
774
- negative_coeff,
775
- prompt_coeff,
776
- delta_coeff,
777
  num_steps,
778
  width,
779
  height,
780
  ],
781
- outputs=[output_image, snapshots, summary, metrics, trace],
782
  show_progress="full",
783
  )
784
 
 
22
  import torch
23
  from PIL import Image, ImageDraw
24
 
25
+ from diffusers import DPMSolverMultistepScheduler, StableDiffusionPipeline
26
 
27
 
28
  APP_TITLE = "Stable Diffusion Equation Playground"
29
  DEFAULT_MODEL = "stable-diffusion-v1-5/stable-diffusion-v1-5"
30
+ DEFAULT_PROMPT_A = "a cozy treehouse in a forest"
31
+ DEFAULT_PROMPT_B = "an underwater coral reef"
32
+ DEFAULT_PROMPT_C = "a colorful outer space nebula"
33
+ DEFAULT_NEGATIVE_PROMPT = "blurry, low quality, distorted"
34
  MAX_SEED = 2_147_483_647
35
 
36
+ PROMPT_MATH_CODE = """# Diffusers normally hides this inside pipe(prompt).
37
+ # In this app, each prompt becomes a CLIP text embedding first.
38
+ embed_a, negative = encode_prompt(prompt_a, negative_prompt)
39
+ embed_b, _ = encode_prompt(prompt_b, negative_prompt)
40
+ embed_c, _ = encode_prompt(prompt_c, negative_prompt)
41
+
42
+ # The sliders choose the strength of each idea.
43
+ total = strength_a + strength_b + strength_c
44
+ if total <= 0:
45
+ wa = wb = wc = 1 / 3
 
 
 
 
 
 
 
46
  else:
47
+ wa = strength_a / total
48
+ wb = strength_b / total
49
+ wc = strength_c / total
50
+
51
+ prompt_embeds = wa * embed_a + wb * embed_b + wc * embed_c
52
  """
53
 
54
  LATENT_MATH_CODE = """# Stable Diffusion does not start from pixels.
55
  # It starts from noisy latents in the VAE's compressed image space.
56
  noise_a = torch.randn(latent_shape, generator=seed_a)
57
  noise_b = torch.randn(latent_shape, generator=seed_b)
 
58
 
59
+ # This hidden lever lets students mix the starting noise too.
60
+ latents = (1 - noise_mix) * noise_a + noise_mix * noise_b
61
+ latents = (latents - latents.mean()) / latents.std()
62
 
63
  latents = latents * scheduler.init_noise_sigma
64
  """
 
70
 
71
  # Standard CFG is:
72
  # guided = noise_negative + guidance_scale * delta
73
+ guided = noise_negative + guidance_scale * delta
 
 
74
  """
75
 
76
 
 
123
  return image
124
 
125
 
126
+ def normalized_prompt_weights(weight_a, weight_b, weight_c):
 
 
 
 
 
 
127
  weights = [max(0.0, float(weight_a)), max(0.0, float(weight_b)), max(0.0, float(weight_c))]
128
  total = sum(weights)
129
  if total <= 0:
130
+ return 1 / 3, 1 / 3, 1 / 3
131
  return tuple(weight / total for weight in weights)
132
 
133
 
134
+ def compact_prompt(text, fallback):
 
 
 
 
 
 
 
135
  text = " ".join(str(text or fallback).split())
136
+ text = text.replace("`", "'").replace('"', "'")
137
+ return text[:52] + ("..." if len(text) > 52 else "")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
138
 
 
 
 
139
 
140
+ def prompt_equation(prompt_a, prompt_b, prompt_c, weight_a, weight_b, weight_c):
141
+ weight_a, weight_b, weight_c = normalized_prompt_weights(weight_a, weight_b, weight_c)
 
142
  return (
143
+ f"prompt_embedding = {weight_a:.2f} * A + {weight_b:.2f} * B + {weight_c:.2f} * C",
 
144
  weight_a,
145
  weight_b,
146
  weight_c,
147
  )
148
 
149
 
150
+ def equation_markdown(prompt_a, prompt_b, prompt_c, weight_a, weight_b, weight_c):
151
+ equation, weight_a, weight_b, weight_c = prompt_equation(prompt_a, prompt_b, prompt_c, weight_a, weight_b, weight_c)
152
+ return (
153
+ "### Current Equation\n"
154
+ f"`{equation}`\n\n"
155
+ f"**A** = {compact_prompt(prompt_a, 'Prompt A')} \n"
156
+ f"**B** = {compact_prompt(prompt_b, 'Prompt B')} \n"
157
+ f"**C** = {compact_prompt(prompt_c, 'Prompt C')}"
158
+ )
159
 
160
 
161
  @lru_cache(maxsize=2)
162
+ def load_pipe(model_id, device_type):
163
  device = torch.device(device_type)
164
  dtype = model_dtype(device)
165
  pipe = StableDiffusionPipeline.from_pretrained(
 
168
  use_safetensors=True,
169
  )
170
 
171
+ pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
 
 
 
 
172
  pipe = pipe.to(device)
173
  pipe.set_progress_bar_config(disable=True)
174
  pipe.enable_vae_slicing()
 
217
  prompt_b,
218
  prompt_c,
219
  negative_prompt,
220
+ weight_a,
221
+ weight_b,
222
+ weight_c,
 
 
 
223
  renormalize_prompt,
224
  ):
225
  emb_a, negative_embeds = encode_prompt(pipe, prompt_a, negative_prompt, device)
226
  emb_b, _ = encode_prompt(pipe, prompt_b, negative_prompt, device)
227
  emb_c, _ = encode_prompt(pipe, prompt_c or "", negative_prompt, device)
228
 
229
+ formula, weight_a, weight_b, weight_c = prompt_equation(prompt_a, prompt_b, prompt_c, weight_a, weight_b, weight_c)
230
+ mixed = weight_a * emb_a + weight_b * emb_b + weight_c * emb_c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
231
 
232
  original_norm = emb_a.detach().float().norm()
233
  mixed_norm = mixed.detach().float().norm()
 
273
  return latents, formula, metrics
274
 
275
 
276
+ def apply_classifier_free_guidance(noise_negative, noise_prompt, guidance_scale):
 
 
 
 
 
 
 
 
 
277
  delta = noise_prompt - noise_negative
278
+ guided = noise_negative + guidance_scale * delta
279
+ formula = f"guided = negative + {guidance_scale:.2f} * (prompt - negative)"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
280
  return guided, formula
281
 
282
 
 
309
  @spaces.GPU(duration=gpu_duration)
310
  @torch.inference_mode()
311
  def generate(
 
 
312
  prompt_a,
313
  prompt_b,
314
  prompt_c,
315
+ weight_a,
316
+ weight_b,
317
+ weight_c,
 
 
 
 
 
318
  seed_a,
319
  seed_b,
320
  noise_mix,
321
+ negative_prompt,
 
322
  guidance_scale,
 
 
 
 
323
  num_steps,
324
  width,
325
  height,
 
331
  [],
332
  "No GPU was detected. The app is designed for CUDA or MPS. It can run on CPU, but it may take a very long time.",
333
  [],
 
334
  )
335
 
336
  width = round_to_multiple_of_8(width)
337
  height = round_to_multiple_of_8(height)
338
  num_steps = int(num_steps)
339
+ scheduler_name = "DPM++ 2M"
340
+ pipe = load_pipe(DEFAULT_MODEL, device.type)
341
 
342
  prompt_embeds, negative_prompt_embeds, prompt_formula, prompt_metrics = mix_prompt_embeddings(
343
  pipe,
 
346
  prompt_b or "",
347
  prompt_c or "",
348
  negative_prompt or "",
349
+ float(weight_a),
350
+ float(weight_b),
351
+ float(weight_c),
352
+ True,
 
 
 
353
  )
354
 
355
  pipe.scheduler.set_timesteps(num_steps, device=device)
 
361
  int(seed_a),
362
  int(seed_b),
363
  float(noise_mix),
364
+ True,
365
  )
366
 
367
  text_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])
368
  snapshots = []
 
369
  save_at = checkpoint_indices(num_steps)
370
  last_formula = ""
371
 
 
380
  return_dict=False,
381
  )[0]
382
  noise_negative, noise_prompt = noise_pred.chunk(2)
383
+ guided, last_formula = apply_classifier_free_guidance(
384
  noise_negative,
385
  noise_prompt,
 
386
  float(guidance_scale),
 
 
 
 
387
  )
388
 
389
  latents = pipe.scheduler.step(guided, timestep, latents, return_dict=False)[0]
390
 
 
 
 
 
 
 
 
 
 
 
 
391
  if step_index in save_at:
392
  snapshot = decode_latents(pipe, latents)[0]
393
  snapshots.append((snapshot, f"step {step_index + 1} of {num_steps}"))
 
400
  ["guidance formula", last_formula],
401
  ]
402
  summary = (
403
+ f"Prompt blend: {prompt_formula}\n\n"
404
+ f"Starting noise: {latent_formula}\n\n"
405
+ f"Guidance: {last_formula}\n\n"
406
+ f"Steps: {num_steps}; size: {width}x{height}; scheduler: {scheduler_name}\n\n"
407
+ "The sliders blend text embeddings, not pixels. Stable Diffusion starts from noise and uses this blended prompt "
408
+ "to steer each denoising step."
 
409
  )
410
 
411
  if device.type == "cuda":
412
  torch.cuda.empty_cache()
413
  gc.collect()
414
+ return final_image, snapshots, summary, metrics
415
 
416
 
417
  def randomize_seeds():
 
433
  """
434
 
435
  metric_headers = ["quantity", "value"]
 
436
 
437
  with gr.Blocks(title=APP_TITLE, theme=theme, css=css) as demo:
438
  gr.Markdown(
439
  f"# {APP_TITLE}\n"
440
+ "Blend three ideas, then watch Stable Diffusion turn noise into an image using that blended prompt embedding."
 
441
  )
442
+ equation_preview = gr.Markdown(
443
+ equation_markdown(DEFAULT_PROMPT_A, DEFAULT_PROMPT_B, DEFAULT_PROMPT_C, 1, 1, 1)
444
+ )
445
+ width = gr.State(512)
446
+ height = gr.State(512)
447
 
448
  with gr.Row(equal_height=False):
449
  with gr.Column(scale=1, min_width=320):
450
+ with gr.Group():
451
+ prompt_a = gr.Textbox(value=DEFAULT_PROMPT_A, label="Prompt A", lines=2)
452
+ strength_a = gr.Slider(0, 3, value=1, step=0.05, label="Strength A")
453
+ prompt_b = gr.Textbox(value=DEFAULT_PROMPT_B, label="Prompt B", lines=2)
454
+ strength_b = gr.Slider(0, 3, value=1, step=0.05, label="Strength B")
455
+ prompt_c = gr.Textbox(value=DEFAULT_PROMPT_C, label="Prompt C", lines=2)
456
+ strength_c = gr.Slider(0, 3, value=1, step=0.05, label="Strength C")
457
+
458
+ with gr.Accordion("A Few Diffusers Levers", open=True):
459
+ guidance_scale = gr.Slider(1, 14, value=7.5, step=0.5, label="Prompt guidance")
460
+ num_steps = gr.Slider(8, 35, value=20, step=1, label="Denoising steps")
461
+ with gr.Row():
462
+ seed_a = gr.Number(value=11, precision=0, label="Starting noise seed")
463
+ random_seeds = gr.Button("Random seed")
464
  negative_prompt = gr.Textbox(
465
+ value=DEFAULT_NEGATIVE_PROMPT,
466
+ label="Things to avoid",
467
+ lines=1,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
468
  )
 
 
 
 
 
 
469
 
470
+ with gr.Accordion("Extra Noise Mixer", open=False):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
471
  with gr.Row():
472
+ seed_b = gr.Number(value=2222, precision=0, label="Second noise seed")
473
+ noise_mix = gr.Slider(0, 1, value=0.0, step=0.05, label="Second seed strength")
 
474
 
475
+ generate_button = gr.Button("Generate", variant="primary")
 
 
 
 
 
476
 
477
  with gr.Column(scale=1, min_width=420):
478
  output_image = gr.Image(
 
482
  interactive=False,
483
  )
484
  summary = gr.Textbox(label="What happened", lines=12, interactive=False)
485
+ with gr.Accordion("Denoising Snapshots", open=False):
486
+ snapshots = gr.Gallery(
487
+ label="Decoded latent snapshots",
488
+ columns=2,
489
+ height=420,
490
+ object_fit="contain",
491
+ elem_classes=["snapshot-gallery"],
492
+ )
493
+ with gr.Accordion("Embedding Measurements", open=False):
494
+ metrics = gr.Dataframe(
495
+ headers=metric_headers,
496
+ datatype=["str", "str"],
497
+ label="Embedding and latent measurements",
498
+ interactive=False,
499
+ )
 
 
 
 
500
 
501
+ with gr.Accordion("Code Cells", open=False):
502
  with gr.Row(equal_height=False):
503
  gr.Code(PROMPT_MATH_CODE, language="python", label="Prompt embedding math", interactive=False, elem_classes=["code-panel"])
504
  gr.Code(LATENT_MATH_CODE, language="python", label="Latent noise math", interactive=False, elem_classes=["code-panel"])
 
510
  outputs=[seed_a, seed_b],
511
  show_progress="hidden",
512
  )
513
+ for equation_input in [prompt_a, prompt_b, prompt_c, strength_a, strength_b, strength_c]:
514
+ equation_input.change(
515
+ equation_markdown,
516
+ inputs=[prompt_a, prompt_b, prompt_c, strength_a, strength_b, strength_c],
517
+ outputs=[equation_preview],
 
 
 
 
 
 
518
  show_progress="hidden",
519
  )
520
  generate_button.click(
521
  generate,
522
  inputs=[
 
 
523
  prompt_a,
524
  prompt_b,
525
  prompt_c,
526
+ strength_a,
527
+ strength_b,
528
+ strength_c,
 
 
 
 
 
529
  seed_a,
530
  seed_b,
531
  noise_mix,
532
+ negative_prompt,
 
533
  guidance_scale,
 
 
 
 
534
  num_steps,
535
  width,
536
  height,
537
  ],
538
+ outputs=[output_image, snapshots, summary, metrics],
539
  show_progress="full",
540
  )
541