Make experiments consistent & use shared utils.

experiments/diamond-crop-img2img.py

@@ -1,35 +1,49 @@
 from utils import filenames, images, wallpaper, pipes
 
 prompt = "seamless pattern of parrots. black and white, drawing, white background, seamless, ornament."
+inference_steps = 20
 generation_dir = "./generations/diamond_crop_img2img"
 
 sdxl_pipe = pipes.create_stable_diffusion_xl_pipeline()
-
-original_image = sdxl_pipe(prompt=prompt, num_inference_steps=20).images[0]
-original_image_filename = filenames.generate_filename(f"{generation_dir}/originals", prompt)
-original_image.save(original_image_filename)
-print("generated image generated", original_image_filename)
-
-inner_rotated_tile = images.extract_inner_rotated_tile_from_image(original_image)
-inner_rotated_tile_filename = filenames.generate_filename(f"{generation_dir}/inner_rotated_tiles", prompt,
-                                                          "-inner-rotated")
-inner_rotated_tile.save(inner_rotated_tile_filename)
-print("generated inner rotated tile", inner_rotated_tile_filename)
-
 sdxl_img2img_pipe = pipes.create_stable_diffusion_xl_img2img_pipe()
 
-tileable_image = sdxl_img2img_pipe(prompt, image=inner_rotated_tile.convert("RGB"), num_inference_steps=20).images[0]
-tileable_image_filename = filenames.generate_filename(f"{generation_dir}/tiles", prompt, "-tile")
-tileable_image.save(tileable_image_filename)
-print("generated tileable image", tileable_image_filename)
-
-half_drop = wallpaper.convert_tile_to_half_drop(tileable_image)
-half_drop_filename = filenames.generate_filename(f"{generation_dir}//half_drops", prompt, "_half-drop-tile")
-half_drop.save(half_drop_filename)
-print("generated half drop tile", tileable_image_filename)
-
-half_drop_tiled = wallpaper.generate_tile_test_arrangement(half_drop)
-half_drop_tiled_filename = filenames.generate_filename(f"{generation_dir}/half_drop_demo", prompt,
-                                                       "_half-drop-demo")
-half_drop_tiled.save(half_drop_tiled_filename)
-print("generated half drop tiled image", tileable_image_filename)
+prompts = [
+    "seamless pattern of parrots. black and white, drawing, white background, seamless, ornament.",
+    "highly detailed black and white geometric pattern, repetition, hexagon, fractal, sacred geometry, sharp angles, "
+    "symmetry."
+    "spring flowers, happy and beautiful, by jackdaws and william morris, thick black line art, anime.",
+    "individual, one, cartoon-style, great horned owl standing on a branch, coloring page, black and white.",
+    "geometric pattern, orange background."
+]
+
+for prompt in prompts:
+    print(f"using prompt: `{prompt}`")
+
+    original_image = sdxl_pipe(prompt=prompt, num_inference_steps=inference_steps).images[0]
+    original_image_filename = filenames.generate_filename(f"{generation_dir}/originals", prompt)
+    original_image.save(original_image_filename)
+    print("generated original image", original_image_filename)
+
+    inner_rotated_tile = images.extract_inner_rotated_tile_from_image(original_image)
+    inner_rotated_tile_filename = filenames.generate_filename(f"{generation_dir}/inner_rotated_tiles", prompt,
+                                                              "-inner-rotated")
+    inner_rotated_tile.save(inner_rotated_tile_filename)
+    print("generated inner rotated tile", inner_rotated_tile_filename)
+
+    tileable_image = \
+        sdxl_img2img_pipe(prompt, image=inner_rotated_tile.convert("RGB"), num_inference_steps=inference_steps).images[
+            0]
+    tileable_image_filename = filenames.generate_filename(f"{generation_dir}/tiles", prompt, "-tile")
+    tileable_image.save(tileable_image_filename)
+    print("generated tileable image", tileable_image_filename)
+
+    half_drop = wallpaper.convert_tile_to_half_drop(tileable_image)
+    half_drop_filename = filenames.generate_filename(f"{generation_dir}/half_drops", prompt, "_half-drop-tile")
+    half_drop.save(half_drop_filename)
+    print("generated half drop tile", tileable_image_filename)
+
+    half_drop_demo = wallpaper.generate_tile_test_arrangement(half_drop)
+    half_drop_demo_filename = filenames.generate_filename(f"{generation_dir}/half_drop_demos", prompt,
+                                                          "_half-drop-demo")
+    half_drop_demo.save(half_drop_demo_filename)
+    print("generated half drop demo", tileable_image_filename)

experiments/guided_prompt.py

@@ -1,70 +1,43 @@
-import diffusers
-import torch
-from diffusers import StableDiffusionXLPipeline
+from utils import images, filenames, wallpaper, pipes
 
-from utils import images, filenames, wallpaper
-
-pipe = StableDiffusionXLPipeline.from_pretrained(
-    "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True
-)
-pipe = pipe.to("mps")
-pipe.vae.disable_tiling()
-
-from typing import Optional
-from torch import Tensor
-from torch.nn import functional as F
-from torch.nn.modules.utils import _pair
-
-def asymmetricConv2DConvForward(self, input: Tensor, weight: Tensor, bias: Optional[Tensor]):
-    self.paddingX = (self._reversed_padding_repeated_twice[0], self._reversed_padding_repeated_twice[1], 0, 0)
-    self.paddingY = (0, 0, self._reversed_padding_repeated_twice[2], self._reversed_padding_repeated_twice[3])
-    working = F.pad(input, self.paddingX, mode='circular')
-    working = F.pad(working, self.paddingY, mode='circular')
-    return F.conv2d(working, weight, bias, self.stride, _pair(0), self.dilation, self.groups)
-
-
-targets = [pipe.vae, pipe.text_encoder, pipe.unet]
-conv_layers = []
-for target in targets:
-    for module in target.modules():
-        if isinstance(module, torch.nn.Conv2d):
-            conv_layers.append(module)
-
-for cl in conv_layers:
-    if isinstance(cl, diffusers.models.lora.LoRACompatibleConv) and cl.lora_layer is None:
-        cl.lora_layer = lambda *x: 0
-
-    cl._conv_forward = asymmetricConv2DConvForward.__get__(cl, torch.nn.Conv2d)
+generation_dir = "./generations/guided_prompt"
+inference_steps = 20
 
+sdxl_pipe = pipes.create_stable_diffusion_xl_pipeline()
 
 prompt_suffix = "-45 degree angle."
 
 prompts = [
     "seamless pattern of parrots. black and white, drawing, white background, seamless, ornament.",
-    "highly detailed black and white geometric pattern, repetition, hexagon, fractal, sacred geometry, sharp angles, symmetry."
-    "spring flowers, happy and beautiful, by jacdraws and william morris, thick black lineart, anime.",
+    "highly detailed black and white geometric pattern, repetition, hexagon, fractal, sacred geometry, sharp angles, "
+    "symmetry."
+    "spring flowers, happy and beautiful, by jackdaws and william morris, thick black line art, anime.",
     "individual, one, cartoon-style, great horned owl standing on a branch, coloring page, black and white.",
     "geometric pattern, orange background."
 ]
 
 for prompt in prompts:
-    prompt = f"{prompt} {prompt_suffix}"
-    image = pipe(prompt=prompt, num_inference_steps=20).images[0]
+    modified_prompt = f"{prompt} {prompt_suffix}"
+    print(f"using prompt: `{modified_prompt}`")
 
-    filename = filenames.generate_filename("./generations/45-degrees/tiles", prompt)
-    image.save(filename)
+    original_image = sdxl_pipe(prompt=modified_prompt, num_inference_steps=inference_steps).images[0]
+    original_image_filename = filenames.generate_filename(f"{generation_dir}/tiles", modified_prompt)
+    original_image.save(original_image_filename)
+    print("generated original image", original_image_filename)
 
-    rotated = images.rotate_image(image)
-    rotated_filename = filenames.generate_filename("./generations/45-degrees/rotations", prompt, "_rotated")
+    rotated = images.rotate_image(original_image)
+    rotated_filename = filenames.generate_filename(f"{generation_dir}/rotations", modified_prompt, "_rotated")
     rotated.save(rotated_filename)
+    print("generated rotation", rotated_filename)
 
-    half_drop = wallpaper.convert_tile_to_half_drop(image)
-    half_drop_filename = filenames.generate_filename("./generations/45-degrees/half_drops", prompt, "_half-drop-tile")
+    half_drop = wallpaper.convert_tile_to_half_drop(original_image)
+    half_drop_filename = filenames.generate_filename(f"{generation_dir}/half_drops", modified_prompt,
+                                                     "_half-drop-tile")
     half_drop.save(half_drop_filename)
+    print("generated half drop", half_drop_filename)
 
-    half_drop_tiled = wallpaper.generate_tile_test_arrangement(half_drop)
-    half_drop_tiled_filename = filenames.generate_filename("./generations/45-degrees/half_drop_tiled", prompt, "_half_drop_tiled")
-    half_drop_tiled.save(half_drop_tiled_filename)
-
-    print(f"generated images for prompt=`${prompt}`")
-
+    half_drop_demo = wallpaper.generate_tile_test_arrangement(half_drop)
+    half_drop_demo_filename = filenames.generate_filename(f"{generation_dir}/half_drop_demos", modified_prompt,
+                                                          "_half-drop-demo")
+    half_drop_demo.save(half_drop_demo_filename)
+    print("generated half drop demo", half_drop_demo_filename)

experiments/inpainting.py

@@ -1,77 +1,31 @@
-import diffusers
-import torch
-from diffusers import StableDiffusionXLPipeline, StableDiffusionXLImg2ImgPipeline
-from PIL import Image
-
-from utils import filenames, images, wallpaper
-
-# pipe = StableDiffusionXLPipeline.from_pretrained(
-#     "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True
-# )
-# pipe = pipe.to("mps")
-# pipe.vae.disable_tiling()
-#
-
-pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(
-    "stabilityai/stable-diffusion-xl-refiner-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True
-)
-pipe = pipe.to("mps")
-pipe.vae.disable_tiling()
-
-from typing import Optional
-from torch import Tensor
-from torch.nn import functional as F
-from torch.nn.modules.utils import _pair
-
-
-def asymmetricConv2DConvForward(self, input: Tensor, weight: Tensor, bias: Optional[Tensor]):
-    self.paddingX = (self._reversed_padding_repeated_twice[0], self._reversed_padding_repeated_twice[1], 0, 0)
-    self.paddingY = (0, 0, self._reversed_padding_repeated_twice[2], self._reversed_padding_repeated_twice[3])
-    working = F.pad(input, self.paddingX, mode='circular')
-    working = F.pad(working, self.paddingY, mode='circular')
-    return F.conv2d(working, weight, bias, self.stride, _pair(0), self.dilation, self.groups)
-
-# TODO: Are the targets here sufficient?
-targets = [pipe.vae, pipe.text_encoder_2, pipe.unet]
-conv_layers = []
-index = 0
-for target in targets:
-    print(index)
-    index += 1
-    for module in target.modules():
-        if isinstance(module, torch.nn.Conv2d):
-            conv_layers.append(module)
-
-for cl in conv_layers:
-    if isinstance(cl, diffusers.models.lora.LoRACompatibleConv) and cl.lora_layer is None:
-        cl.lora_layer = lambda *x: 0
-
-    cl._conv_forward = asymmetricConv2DConvForward.__get__(cl, torch.nn.Conv2d)
-
-# prompt = "seamless pattern of parrots. black and white, drawing, white background, seamless, ornament."
-# original_image = pipe(prompt=prompt, num_inference_steps=20).images[0]
-# original_image_filename = filenames.generate_filename("./generations/inpainting/originals", prompt)
-# original_image.save(original_image_filename)
-
-prompt = "seamless pattern of parrots. black and white, drawing, white background, seamless, ornament."
-original_image = Image.open("generations/inpainting/originals/seamless-pattern-of-parrots-black-and-white-drawing-white-background-seamless-ornament_20240109-170602.png")
-
-inner_rotated_tile = images.extract_inner_rotated_tile_from_image(original_image)
-inner_rotated_tile_filename = filenames.generate_filename("./generations/inpainting/inner_rotated_tiles", prompt,
-                                                          "-inner-rotated")
-inner_rotated_tile.save(inner_rotated_tile_filename)
-print("generated inner rotated tile", inner_rotated_tile_filename)
-
-tileable_image = pipe(prompt, image=inner_rotated_tile.convert("RGB"), num_inference_steps=20).images[0]
-tileable_image_filename = filenames.generate_filename("./generations/inpainting/tiles", prompt, "-tile")
-tileable_image.save(tileable_image_filename)
-
-half_drop = wallpaper.convert_tile_to_half_drop(tileable_image)
-half_drop_filename = filenames.generate_filename("./generations/inpainting/half_drops", prompt, "_half-drop-tile")
-half_drop.save(half_drop_filename)
-
-half_drop_tiled = wallpaper.generate_tile_test_arrangement(half_drop)
-half_drop_tiled_filename = filenames.generate_filename("./generations/inpainting/half_drop_tiled", prompt,
-                                                       "_half_drop_tiled")
-half_drop_tiled.save(half_drop_tiled_filename)
-
+from diffusers.utils import load_image
+
+from utils import pipes, filenames
+
+sdxl_pipe = pipes.create_stable_diffusion_xl_pipeline()
+sdxl_inpainting_pipe = pipes.create_stable_diffusion_xl_inpainting_pipe()
+
+generation_dir = "./generations/inpainting"
+mask = load_image("masks/mask.png").convert("RGB")
+
+prompts = [
+    "seamless pattern of parrots. black and white, drawing, white background, seamless, ornament.",
+    "highly detailed black and white geometric pattern, repetition, hexagon, fractal, sacred geometry, sharp angles, "
+    "symmetry."
+    "spring flowers, happy and beautiful, by jackdaws and william morris, thick black line art, anime.",
+    "individual, one, cartoon-style, great horned owl standing on a branch, coloring page, black and white.",
+    "geometric pattern, orange background."
+]
+
+for prompt in prompts:
+    original_image = sdxl_pipe(prompt=prompt, num_inference_steps=40).images[0]
+    original_image_filename = filenames.generate_filename(f"{generation_dir}/tiles", prompt)
+    original_image.save(original_image_filename)
+    print("generated original image", original_image_filename)
+
+    inpainted_image = \
+        sdxl_inpainting_pipe(prompt=prompt, init_image=original_image, mask_image=mask, num_inference_steps=10,
+                             strength=0.80).images[0]
+    inpainted_filename = filenames.generate_filename(f"{generation_dir}/inpaintings", prompt, "_inpainted")
+    inpainted_image.save(inpainted_filename)
+    print("generated inpainting", inpainted_filename)

utils/pipes.py

@@ -1,5 +1,5 @@
 import torch
-from diffusers import StableDiffusionXLImg2ImgPipeline, StableDiffusionXLPipeline
+from diffusers import StableDiffusionXLImg2ImgPipeline, StableDiffusionXLPipeline, StableDiffusionXLInpaintPipeline
 
 from utils import tiling
 
@@ -22,3 +22,13 @@ def create_stable_diffusion_xl_img2img_pipe():
     pipe.vae.disable_tiling()
     tiling.enable_circular_tiling([pipe.vae, pipe.text_encoder_2, pipe.unet])
     return pipe
+
+
+def create_stable_diffusion_xl_inpainting_pipe():
+    pipe = StableDiffusionXLInpaintPipeline.from_pretrained(
+        "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True
+    )
+    pipe = pipe.to("mps")
+    pipe.vae.disable_tiling()
+    tiling.enable_circular_tiling([pipe.vae, pipe.text_encoder, pipe.unet])
+    return pipe