| | import gradio as gr |
| | import torch |
| | import cv2 |
| | from diffusers import ControlNetModel, StableDiffusionControlNetPipeline |
| | from PIL import Image |
| |
|
| | from diffusion_webui.diffusion_models.base_controlnet_pipeline import ( |
| | ControlnetPipeline, |
| | ) |
| | from diffusion_webui.utils.model_list import ( |
| | controlnet_model_list, |
| | stable_model_list, |
| | ) |
| | from diffusion_webui.utils.preprocces_utils import PREPROCCES_DICT |
| | from diffusion_webui.utils.scheduler_list import ( |
| | SCHEDULER_MAPPING, |
| | get_scheduler, |
| | ) |
| |
|
| |
|
| | stable_model_list = [ |
| | "runwayml/stable-diffusion-v1-5", |
| | "dreamlike-art/dreamlike-diffusion-1.0", |
| | "kadirnar/maturemalemix_v0", |
| | "kadirnar/DreamShaper_v6" |
| | ] |
| |
|
| | stable_inpiant_model_list = [ |
| | "stabilityai/stable-diffusion-2-inpainting", |
| | "runwayml/stable-diffusion-inpainting", |
| | "saik0s/realistic_vision_inpainting", |
| | ] |
| |
|
| | controlnet_model_list = [ |
| | "lllyasviel/control_v11p_sd15_canny", |
| | "lllyasviel/control_v11f1p_sd15_depth", |
| | "lllyasviel/control_v11p_sd15_openpose", |
| | "lllyasviel/control_v11p_sd15_scribble", |
| | "lllyasviel/control_v11p_sd15_mlsd", |
| | "lllyasviel/control_v11e_sd15_shuffle", |
| | "lllyasviel/control_v11e_sd15_ip2p", |
| | "lllyasviel/control_v11p_sd15_lineart", |
| | "lllyasviel/control_v11p_sd15s2_lineart_anime", |
| | "lllyasviel/control_v11p_sd15_softedge", |
| | ] |
| |
|
| | class StableDiffusionControlNetGenerator(ControlnetPipeline): |
| | def __init__(self): |
| | self.pipe = None |
| | |
| | def load_model(self, stable_model_path, controlnet_model_path, scheduler): |
| | if self.pipe is None or self.pipe.model_name != stable_model_path or self.pipe.scheduler_name != scheduler: |
| | controlnet = ControlNetModel.from_pretrained( |
| | controlnet_model_path, torch_dtype=torch.float16 |
| | ) |
| | self.pipe = StableDiffusionControlNetPipeline.from_pretrained( |
| | pretrained_model_name_or_path=stable_model_path, |
| | controlnet=controlnet, |
| | safety_checker=None, |
| | torch_dtype=torch.float16, |
| | ) |
| | self.pipe.model_name = stable_model_path |
| | self.pipe.scheduler_name = scheduler |
| | |
| | self.pipe = get_scheduler(pipe=self.pipe, scheduler=scheduler) |
| | self.pipe.scheduler_name = scheduler |
| | self.pipe.to("cuda") |
| | self.pipe.enable_xformers_memory_efficient_attention() |
| | |
| | return self.pipe |
| |
|
| |
|
| | def controlnet_preprocces( |
| | self, |
| | read_image: str, |
| | preprocces_type: str, |
| | ): |
| | processed_image = PREPROCCES_DICT[preprocces_type](read_image) |
| | return processed_image |
| |
|
| | def generate_image( |
| | self, |
| | image_path: str, |
| | stable_model_path: str, |
| | controlnet_model_path: str, |
| | height: int, |
| | width: int, |
| | guess_mode: bool, |
| | controlnet_conditioning_scale: int, |
| | prompt: str, |
| | negative_prompt: str, |
| | num_images_per_prompt: int, |
| | guidance_scale: int, |
| | num_inference_step: int, |
| | scheduler: str, |
| | seed_generator: int, |
| | preprocces_type: str, |
| | ): |
| | pipe = self.load_model( |
| | stable_model_path=stable_model_path, |
| | controlnet_model_path=controlnet_model_path, |
| | scheduler=scheduler, |
| | ) |
| | if preprocces_type== "ScribbleXDOG": |
| | read_image = cv2.imread(image_path) |
| | controlnet_image = self.controlnet_preprocces(read_image=read_image, preprocces_type=preprocces_type)[0] |
| | controlnet_image = Image.fromarray(controlnet_image) |
| | |
| | elif preprocces_type== "None": |
| | controlnet_image = self.controlnet_preprocces(read_image=image_path, preprocces_type=preprocces_type) |
| | else: |
| | read_image = Image.open(image_path) |
| | controlnet_image = self.controlnet_preprocces(read_image=read_image, preprocces_type=preprocces_type) |
| |
|
| | if seed_generator == 0: |
| | random_seed = torch.randint(0, 1000000, (1,)) |
| | generator = torch.manual_seed(random_seed) |
| | else: |
| | generator = torch.manual_seed(seed_generator) |
| | |
| |
|
| | output = pipe( |
| | prompt=prompt, |
| | height=height, |
| | width=width, |
| | controlnet_conditioning_scale=float(controlnet_conditioning_scale), |
| | guess_mode=guess_mode, |
| | image=controlnet_image, |
| | negative_prompt=negative_prompt, |
| | num_images_per_prompt=num_images_per_prompt, |
| | num_inference_steps=num_inference_step, |
| | guidance_scale=guidance_scale, |
| | generator=generator, |
| | ).images |
| |
|
| | return output |
| |
|
| | def app(): |
| | with gr.Blocks(): |
| | with gr.Row(): |
| | with gr.Column(): |
| | controlnet_image_path = gr.Image( |
| | type="filepath", label="Image" |
| | ).style(height=260) |
| | controlnet_prompt = gr.Textbox( |
| | lines=1, placeholder="Prompt", show_label=False |
| | ) |
| | controlnet_negative_prompt = gr.Textbox( |
| | lines=1, placeholder="Negative Prompt", show_label=False |
| | ) |
| |
|
| | with gr.Row(): |
| | with gr.Column(): |
| | controlnet_stable_model_path = gr.Dropdown( |
| | choices=stable_model_list, |
| | value=stable_model_list[0], |
| | label="Stable Model Path", |
| | ) |
| | controlnet_preprocces_type = gr.Dropdown( |
| | choices=list(PREPROCCES_DICT.keys()), |
| | value=list(PREPROCCES_DICT.keys())[0], |
| | label="Preprocess Type", |
| | ) |
| | controlnet_conditioning_scale = gr.Slider( |
| | minimum=0.0, |
| | maximum=1.0, |
| | step=0.1, |
| | value=1.0, |
| | label="ControlNet Conditioning Scale", |
| | ) |
| | controlnet_guidance_scale = gr.Slider( |
| | minimum=0.1, |
| | maximum=15, |
| | step=0.1, |
| | value=7.5, |
| | label="Guidance Scale", |
| | ) |
| | controlnet_height = gr.Slider( |
| | minimum=128, |
| | maximum=1280, |
| | step=32, |
| | value=512, |
| | label="Height", |
| | ) |
| | controlnet_width = gr.Slider( |
| | minimum=128, |
| | maximum=1280, |
| | step=32, |
| | value=512, |
| | label="Width", |
| | ) |
| |
|
| | with gr.Row(): |
| | with gr.Column(): |
| | controlnet_model_path = gr.Dropdown( |
| | choices=controlnet_model_list, |
| | value=controlnet_model_list[0], |
| | label="ControlNet Model Path", |
| | ) |
| | controlnet_scheduler = gr.Dropdown( |
| | choices=list(SCHEDULER_MAPPING.keys()), |
| | value=list(SCHEDULER_MAPPING.keys())[0], |
| | label="Scheduler", |
| | ) |
| | controlnet_num_inference_step = gr.Slider( |
| | minimum=1, |
| | maximum=150, |
| | step=1, |
| | value=30, |
| | label="Num Inference Step", |
| | ) |
| |
|
| | controlnet_num_images_per_prompt = gr.Slider( |
| | minimum=1, |
| | maximum=4, |
| | step=1, |
| | value=1, |
| | label="Number Of Images", |
| | ) |
| | controlnet_seed_generator = gr.Slider( |
| | minimum=0, |
| | maximum=1000000, |
| | step=1, |
| | value=0, |
| | label="Seed(0 for random)", |
| | ) |
| | controlnet_guess_mode = gr.Checkbox( |
| | label="Guess Mode" |
| | ) |
| |
|
| | |
| | predict_button = gr.Button(value="Generate Image") |
| |
|
| | with gr.Column(): |
| | |
| | output_image = gr.Gallery( |
| | label="Generated images", |
| | show_label=False, |
| | elem_id="gallery", |
| | ).style(grid=(1, 2)) |
| |
|
| | predict_button.click( |
| | fn=StableDiffusionControlNetGenerator().generate_image, |
| | inputs=[ |
| | controlnet_image_path, |
| | controlnet_stable_model_path, |
| | controlnet_model_path, |
| | controlnet_height, |
| | controlnet_width, |
| | controlnet_guess_mode, |
| | controlnet_conditioning_scale, |
| | controlnet_prompt, |
| | controlnet_negative_prompt, |
| | controlnet_num_images_per_prompt, |
| | controlnet_guidance_scale, |
| | controlnet_num_inference_step, |
| | controlnet_scheduler, |
| | controlnet_seed_generator, |
| | controlnet_preprocces_type, |
| | ], |
| | outputs=[output_image], |
| | ) |
| |
|
