img/stable-diffusion-server/img2imgsdr.py

import PIL.Image

from diffusers import DiffusionPipeline
import torch

import numpy as np

from stable_diffusion_server.utils import log_time

pipe = DiffusionPipeline.from_pretrained(
    "models/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True
)
pipe.to("cuda")

refiner = DiffusionPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-refiner-1.0",
    text_encoder_2=pipe.text_encoder_2,
    vae=pipe.vae,
    torch_dtype=torch.float16,
    use_safetensors=True,
    variant="fp16",
)
refiner.to("cuda")

prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"
use_refiner = True
with log_time('diffuse'):
    with torch.inference_mode():
        image = pipe(prompt=prompt, output_type="latent" if use_refiner else "pil").images[0]
        # experiment try deleting a whole bunch of pixels and see if the refiner can recreate them
        # delete top 30% of pixels
        # image = image[0:0.7]
        #pixels to delete
        # pixels_to_delete = int(0.3 * 1024)
        # delete top 30% of pixels
        # image.save("latent.png")
        # image_data = PIL.Image.fromarray(image)
        # image_data.save("latent.png")

        # image = np.array(image)
        pixels_to_delete = int(0.3 * image.shape[0])
        idx_to_delete = np.ones(image.shape[0], dtype=bool, device="cuda")
        idx_to_delete[:pixels_to_delete] = False
        image[idx_to_delete] = [0,0,0]

        # image_data = PIL.Image.fromarray(image)
        # image_data.save("latentcleared.png")


        image = refiner(prompt=prompt, image=image[None, :]).images[0]