Duplicate from ArtGAN/Stable-Diffusion-ControlNet-WebUI

Co-authored-by: Kadir Nar <kadirnar@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,16 @@
+---
+title: Stable Diffusion ControlNet WebUI
+emoji: 🔥
+colorFrom: green
+colorTo: red
+sdk: gradio
+sdk_version: 3.19
+app_file: app.py
+pinned: true
+license: openrail
+tags:
+- making-demos
+duplicated_from: ArtGAN/Stable-Diffusion-ControlNet-WebUI
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,62 @@
+import gradio as gr
+
+from diffusion_webui.helpers import (
+    keras_stable_diffusion_app,
+    stable_diffusion_controlnet_canny_app,
+    stable_diffusion_controlnet_depth_app,
+    stable_diffusion_controlnet_hed_app,
+    stable_diffusion_controlnet_mlsd_app,
+    stable_diffusion_controlnet_pose_app,
+    stable_diffusion_controlnet_scribble_app,
+    stable_diffusion_controlnet_seg_app,
+    stable_diffusion_img2img_app,
+    stable_diffusion_inpaint_app,
+    stable_diffusion_text2img_app,
+)
+
+app = gr.Blocks()
+with app:
+    gr.HTML(
+        """
+        <h1 style='text-align: center'>
+        Stable Diffusion + ControlNet + Keras Diffusion WebUI
+        </h1>
+        """
+    )
+    gr.Markdown(
+        """
+        <h4 style='text-align: center'>
+        Follow me for more! 
+        <a href='https://twitter.com/kadirnar_ai' target='_blank'>Twitter</a> | <a href='https://github.com/kadirnar' target='_blank'>Github</a> | <a href='https://www.linkedin.com/in/kadir-nar/' target='_blank'>Linkedin</a>
+        </h4>
+        """
+    )
+    with gr.Row():
+        with gr.Column():
+            with gr.Tab("Text2Img"):
+                stable_diffusion_text2img_app()
+            with gr.Tab("Img2Img"):
+                stable_diffusion_img2img_app()
+            with gr.Tab("Inpaint"):
+                stable_diffusion_inpaint_app()
+
+            with gr.Tab("ControlNet"):
+                with gr.Tab("Canny"):
+                    stable_diffusion_controlnet_canny_app()
+                with gr.Tab("Depth"):
+                    stable_diffusion_controlnet_depth_app()
+                with gr.Tab("HED"):
+                    stable_diffusion_controlnet_hed_app()
+                with gr.Tab("MLSD"):
+                    stable_diffusion_controlnet_mlsd_app()
+                with gr.Tab("Pose"):
+                    stable_diffusion_controlnet_pose_app()
+                with gr.Tab("Seg"):
+                    stable_diffusion_controlnet_seg_app()
+                with gr.Tab("Scribble"):
+                    stable_diffusion_controlnet_scribble_app()
+
+            with gr.Tab("Keras Diffusion"):
+                keras_diffusion_app = keras_stable_diffusion_app()
+
+app.launch(debug=True)

data/bash.sh

@@ -0,0 +1,2 @@
+black . --config pyproject.toml
+isort .

diffusion_webui/controlnet/controlnet_canny.py

@@ -0,0 +1,176 @@
+import cv2
+import gradio as gr
+import numpy as np
+import torch
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+from PIL import Image
+
+stable_model_list = [
+    "runwayml/stable-diffusion-v1-5",
+    "stabilityai/stable-diffusion-2-1",
+]
+
+controlnet_canny_model_list = [
+    "lllyasviel/sd-controlnet-canny",
+    "thibaud/controlnet-sd21-canny-diffusers",
+]
+
+
+stable_prompt_list = ["a photo of a man.", "a photo of a girl."]
+
+stable_negative_prompt_list = ["bad, ugly", "deformed"]
+
+data_list = [
+    "data/test.png",
+]
+
+
+def controlnet_canny(
+    image_path: str,
+    controlnet_model_path: str,
+):
+    image = Image.open(image_path)
+    image = np.array(image)
+
+    image = cv2.Canny(image, 100, 200)
+    image = image[:, :, None]
+    image = np.concatenate([image, image, image], axis=2)
+    image = Image.fromarray(image)
+
+    controlnet = ControlNetModel.from_pretrained(
+        controlnet_model_path, torch_dtype=torch.float16
+    )
+    return controlnet, image
+
+
+def stable_diffusion_controlnet_canny(
+    image_path: str,
+    stable_model_path: str,
+    controlnet_model_path: str,
+    prompt: str,
+    negative_prompt: str,
+    guidance_scale: int,
+    num_inference_step: int,
+):
+
+    controlnet, image = controlnet_canny(
+        image_path=image_path, controlnet_model_path=controlnet_model_path
+    )
+
+    pipe = StableDiffusionControlNetPipeline.from_pretrained(
+        pretrained_model_name_or_path=stable_model_path,
+        controlnet=controlnet,
+        safety_checker=None,
+        torch_dtype=torch.float16,
+    )
+    pipe.to("cuda")
+    pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+    pipe.enable_xformers_memory_efficient_attention()
+
+    output = pipe(
+        prompt=prompt,
+        image=image,
+        negative_prompt=negative_prompt,
+        num_inference_steps=num_inference_step,
+        guidance_scale=guidance_scale,
+    ).images
+
+    return output[0]
+
+
+def stable_diffusion_controlnet_canny_app():
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                controlnet_canny_image_file = gr.Image(
+                    type="filepath", label="Image"
+                )
+
+                controlnet_canny_stable_model_id = gr.Dropdown(
+                    choices=stable_model_list,
+                    value=stable_model_list[0],
+                    label="Stable Model Id",
+                )
+
+                controlnet_canny_model_id = gr.Dropdown(
+                    choices=controlnet_canny_model_list,
+                    value=controlnet_canny_model_list[0],
+                    label="Controlnet Model Id",
+                )
+
+                controlnet_canny_prompt = gr.Textbox(
+                    lines=1, value=stable_prompt_list[0], label="Prompt"
+                )
+
+                controlnet_canny_negative_prompt = gr.Textbox(
+                    lines=1,
+                    value=stable_negative_prompt_list[0],
+                    label="Negative Prompt",
+                )
+
+                with gr.Accordion("Advanced Options", open=False):
+                    controlnet_canny_guidance_scale = gr.Slider(
+                        minimum=0.1,
+                        maximum=15,
+                        step=0.1,
+                        value=7.5,
+                        label="Guidance Scale",
+                    )
+
+                    controlnet_canny_num_inference_step = gr.Slider(
+                        minimum=1,
+                        maximum=100,
+                        step=1,
+                        value=50,
+                        label="Num Inference Step",
+                    )
+
+                controlnet_canny_predict = gr.Button(value="Generator")
+
+            with gr.Column():
+                output_image = gr.Image(label="Output")
+
+        gr.Examples(
+            fn=stable_diffusion_controlnet_canny,
+            examples=[
+                [
+                    data_list[0],
+                    stable_model_list[0],
+                    controlnet_canny_model_list[0],
+                    stable_prompt_list[0],
+                    stable_negative_prompt_list[0],
+                    7.5,
+                    50,
+                ]
+            ],
+            inputs=[
+                controlnet_canny_image_file,
+                controlnet_canny_stable_model_id,
+                controlnet_canny_model_id,
+                controlnet_canny_prompt,
+                controlnet_canny_negative_prompt,
+                controlnet_canny_guidance_scale,
+                controlnet_canny_num_inference_step,
+            ],
+            outputs=[output_image],
+            cache_examples=False,
+            label="Controlnet Canny Example",
+        )
+
+        controlnet_canny_predict.click(
+            fn=stable_diffusion_controlnet_canny,
+            inputs=[
+                controlnet_canny_image_file,
+                controlnet_canny_stable_model_id,
+                controlnet_canny_model_id,
+                controlnet_canny_prompt,
+                controlnet_canny_negative_prompt,
+                controlnet_canny_guidance_scale,
+                controlnet_canny_num_inference_step,
+            ],
+            outputs=[output_image],
+        )

diffusion_webui/controlnet/controlnet_depth.py

@@ -0,0 +1,176 @@
+import gradio as gr
+import numpy as np
+import torch
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+from PIL import Image
+from transformers import pipeline
+
+stable_model_list = [
+    "runwayml/stable-diffusion-v1-5",
+    "stabilityai/stable-diffusion-2-1",
+]
+
+controlnet_depth_model_list = [
+    "lllyasviel/sd-controlnet-depth",
+    "thibaud/controlnet-sd21-depth-diffusers",
+]
+
+
+stable_prompt_list = ["a photo of a man.", "a photo of a girl."]
+
+stable_negative_prompt_list = ["bad, ugly", "deformed"]
+
+data_list = [
+    "data/test.png",
+]
+
+
+def controlnet_depth(image_path: str, depth_model_path: str):
+    depth_estimator = pipeline("depth-estimation")
+
+    image = Image.open(image_path)
+    image = depth_estimator(image)["depth"]
+    image = np.array(image)
+    image = image[:, :, None]
+    image = np.concatenate([image, image, image], axis=2)
+    image = Image.fromarray(image)
+
+    controlnet = ControlNetModel.from_pretrained(
+        depth_model_path, torch_dtype=torch.float16
+    )
+
+    return controlnet, image
+
+
+def stable_diffusion_controlnet_depth(
+    image_path: str,
+    stable_model_path: str,
+    depth_model_path: str,
+    prompt: str,
+    negative_prompt: str,
+    guidance_scale: int,
+    num_inference_step: int,
+):
+
+    controlnet, image = controlnet_depth(
+        image_path=image_path, depth_model_path=depth_model_path
+    )
+
+    pipe = StableDiffusionControlNetPipeline.from_pretrained(
+        pretrained_model_name_or_path=stable_model_path,
+        controlnet=controlnet,
+        safety_checker=None,
+        torch_dtype=torch.float16,
+    )
+
+    pipe.to("cuda")
+    pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+    pipe.enable_xformers_memory_efficient_attention()
+
+    output = pipe(
+        prompt=prompt,
+        image=image,
+        negative_prompt=negative_prompt,
+        num_inference_steps=num_inference_step,
+        guidance_scale=guidance_scale,
+    ).images
+
+    return output[0]
+
+
+def stable_diffusion_controlnet_depth_app():
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                controlnet_depth_image_file = gr.Image(
+                    type="filepath", label="Image"
+                )
+
+                controlnet_depth_stable_model_id = gr.Dropdown(
+                    choices=stable_model_list,
+                    value=stable_model_list[0],
+                    label="Stable Model Id",
+                )
+
+                controlnet_depth_model_id = gr.Dropdown(
+                    choices=controlnet_depth_model_list,
+                    value=controlnet_depth_model_list[0],
+                    label="ControlNet Model Id",
+                )
+
+                controlnet_depth_prompt = gr.Textbox(
+                    lines=1, value=stable_prompt_list[0], label="Prompt"
+                )
+
+                controlnet_depth_negative_prompt = gr.Textbox(
+                    lines=1,
+                    value=stable_negative_prompt_list[0],
+                    label="Negative Prompt",
+                )
+
+                with gr.Accordion("Advanced Options", open=False):
+                    controlnet_depth_guidance_scale = gr.Slider(
+                        minimum=0.1,
+                        maximum=15,
+                        step=0.1,
+                        value=7.5,
+                        label="Guidance Scale",
+                    )
+
+                    controlnet_depth_num_inference_step = gr.Slider(
+                        minimum=1,
+                        maximum=100,
+                        step=1,
+                        value=50,
+                        label="Num Inference Step",
+                    )
+
+                controlnet_depth_predict = gr.Button(value="Generator")
+
+            with gr.Column():
+                output_image = gr.Image(label="Output")
+
+        gr.Examples(
+            fn=stable_diffusion_controlnet_depth,
+            examples=[
+                [
+                    data_list[0],
+                    stable_model_list[0],
+                    controlnet_depth_model_list[0],
+                    stable_prompt_list[0],
+                    stable_negative_prompt_list[0],
+                    7.5,
+                    50,
+                ]
+            ],
+            inputs=[
+                controlnet_depth_image_file,
+                controlnet_depth_stable_model_id,
+                controlnet_depth_model_id,
+                controlnet_depth_prompt,
+                controlnet_depth_negative_prompt,
+                controlnet_depth_guidance_scale,
+                controlnet_depth_num_inference_step,
+            ],
+            outputs=[output_image],
+            cache_examples=False,
+            label="ControlNet Depth Example",
+        )
+
+        controlnet_depth_predict.click(
+            fn=stable_diffusion_controlnet_depth,
+            inputs=[
+                controlnet_depth_image_file,
+                controlnet_depth_stable_model_id,
+                controlnet_depth_model_id,
+                controlnet_depth_prompt,
+                controlnet_depth_negative_prompt,
+                controlnet_depth_guidance_scale,
+                controlnet_depth_num_inference_step,
+            ],
+            outputs=output_image,
+        )

diffusion_webui/controlnet/controlnet_hed.py

@@ -0,0 +1,170 @@
+import gradio as gr
+import torch
+from controlnet_aux import HEDdetector
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+from PIL import Image
+
+stable_model_list = [
+    "runwayml/stable-diffusion-v1-5",
+    "stabilityai/stable-diffusion-2-1",
+]
+
+controlnet_hed_model_list = [
+    "lllyasviel/sd-controlnet-hed",
+    "thibaud/controlnet-sd21-hed-diffusers",
+]
+
+stable_prompt_list = ["a photo of a man.", "a photo of a girl."]
+
+stable_negative_prompt_list = ["bad, ugly", "deformed"]
+
+data_list = [
+    "data/test.png",
+]
+
+
+def controlnet_hed(image_path: str, controlnet_hed_model_path: str):
+    hed = HEDdetector.from_pretrained("lllyasviel/ControlNet")
+
+    image = Image.open(image_path)
+    image = hed(image)
+
+    controlnet = ControlNetModel.from_pretrained(
+        controlnet_hed_model_path, torch_dtype=torch.float16
+    )
+    return controlnet, image
+
+
+def stable_diffusion_controlnet_hed(
+    image_path: str,
+    stable_model_path: str,
+    controlnet_hed_model_path: str,
+    prompt: str,
+    negative_prompt: str,
+    guidance_scale: int,
+    num_inference_step: int,
+):
+
+    controlnet, image = controlnet_hed(
+        image_path=image_path,
+        controlnet_hed_model_path=controlnet_hed_model_path,
+    )
+
+    pipe = StableDiffusionControlNetPipeline.from_pretrained(
+        pretrained_model_name_or_path=stable_model_path,
+        controlnet=controlnet,
+        safety_checker=None,
+        torch_dtype=torch.float16,
+    )
+
+    pipe.to("cuda")
+    pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+    pipe.enable_xformers_memory_efficient_attention()
+
+    output = pipe(
+        prompt=prompt,
+        image=image,
+        negative_prompt=negative_prompt,
+        num_inference_steps=num_inference_step,
+        guidance_scale=guidance_scale,
+    ).images
+
+    return output[0]
+
+
+def stable_diffusion_controlnet_hed_app():
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                controlnet_hed_image_file = gr.Image(
+                    type="filepath", label="Image"
+                )
+
+                controlnet_hed_stable_model_id = gr.Dropdown(
+                    choices=stable_model_list,
+                    value=stable_model_list[0],
+                    label="Stable Model Id",
+                )
+
+                controlnet_hed_model_id = gr.Dropdown(
+                    choices=controlnet_hed_model_list,
+                    value=controlnet_hed_model_list[1],
+                    label="ControlNet Model Id",
+                )
+
+                controlnet_hed_prompt = gr.Textbox(
+                    lines=1, value=stable_prompt_list[0], label="Prompt"
+                )
+
+                controlnet_hed_negative_prompt = gr.Textbox(
+                    lines=1,
+                    value=stable_negative_prompt_list[0],
+                    label="Negative Prompt",
+                )
+
+                with gr.Accordion("Advanced Options", open=False):
+                    controlnet_hed_guidance_scale = gr.Slider(
+                        minimum=0.1,
+                        maximum=15,
+                        step=0.1,
+                        value=7.5,
+                        label="Guidance Scale",
+                    )
+
+                    controlnet_hed_num_inference_step = gr.Slider(
+                        minimum=1,
+                        maximum=100,
+                        step=1,
+                        value=50,
+                        label="Num Inference Step",
+                    )
+
+                controlnet_hed_predict = gr.Button(value="Generator")
+
+            with gr.Column():
+                output_image = gr.Image(label="Output")
+
+        gr.Examples(
+            fn=stable_diffusion_controlnet_hed,
+            examples=[
+                [
+                    data_list[0],
+                    stable_model_list[0],
+                    controlnet_hed_model_list[0],
+                    stable_prompt_list[0],
+                    stable_negative_prompt_list[0],
+                    7.5,
+                    50,
+                ]
+            ],
+            inputs=[
+                controlnet_hed_image_file,
+                controlnet_hed_stable_model_id,
+                controlnet_hed_model_id,
+                controlnet_hed_prompt,
+                controlnet_hed_negative_prompt,
+                controlnet_hed_guidance_scale,
+                controlnet_hed_num_inference_step,
+            ],
+            outputs=[output_image],
+            cache_examples=False,
+            label="ControlNet HED Example",
+        )
+
+        controlnet_hed_predict.click(
+            fn=stable_diffusion_controlnet_hed,
+            inputs=[
+                controlnet_hed_image_file,
+                controlnet_hed_stable_model_id,
+                controlnet_hed_model_id,
+                controlnet_hed_prompt,
+                controlnet_hed_negative_prompt,
+                controlnet_hed_guidance_scale,
+                controlnet_hed_num_inference_step,
+            ],
+            outputs=[output_image],
+        )

diffusion_webui/controlnet/controlnet_mlsd.py

@@ -0,0 +1,153 @@
+import gradio as gr
+import torch
+from controlnet_aux import MLSDdetector
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+from PIL import Image
+
+stable_model_list = [
+    "runwayml/stable-diffusion-v1-5",
+]
+
+stable_prompt_list = ["a photo of a man.", "a photo of a girl."]
+
+stable_negative_prompt_list = ["bad, ugly", "deformed"]
+
+data_list = [
+    "data/test.png",
+]
+
+
+def controlnet_mlsd(image_path: str):
+    mlsd = MLSDdetector.from_pretrained("lllyasviel/ControlNet")
+
+    image = Image.open(image_path)
+    image = mlsd(image)
+
+    controlnet = ControlNetModel.from_pretrained(
+        "lllyasviel/sd-controlnet-mlsd",
+        torch_dtype=torch.float16,
+    )
+
+    return controlnet, image
+
+
+def stable_diffusion_controlnet_mlsd(
+    image_path: str,
+    model_path: str,
+    prompt: str,
+    negative_prompt: str,
+    guidance_scale: int,
+    num_inference_step: int,
+):
+
+    controlnet, image = controlnet_mlsd(image_path=image_path)
+
+    pipe = StableDiffusionControlNetPipeline.from_pretrained(
+        pretrained_model_name_or_path=model_path,
+        controlnet=controlnet,
+        safety_checker=None,
+        torch_dtype=torch.float16,
+    )
+
+    pipe.to("cuda")
+    pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+    pipe.enable_xformers_memory_efficient_attention()
+
+    output = pipe(
+        prompt=prompt,
+        image=image,
+        negative_prompt=negative_prompt,
+        num_inference_steps=num_inference_step,
+        guidance_scale=guidance_scale,
+    ).images
+
+    return output[0]
+
+
+def stable_diffusion_controlnet_mlsd_app():
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                controlnet_mlsd_image_file = gr.Image(
+                    type="filepath", label="Image"
+                )
+
+                controlnet_mlsd_model_id = gr.Dropdown(
+                    choices=stable_model_list,
+                    value=stable_model_list[0],
+                    label="Stable Model Id",
+                )
+
+                controlnet_mlsd_prompt = gr.Textbox(
+                    lines=1, value=stable_prompt_list[0], label="Prompt"
+                )
+
+                controlnet_mlsd_negative_prompt = gr.Textbox(
+                    lines=1,
+                    value=stable_negative_prompt_list[0],
+                    label="Negative Prompt",
+                )
+
+                with gr.Accordion("Advanced Options", open=False):
+                    controlnet_mlsd_guidance_scale = gr.Slider(
+                        minimum=0.1,
+                        maximum=15,
+                        step=0.1,
+                        value=7.5,
+                        label="Guidance Scale",
+                    )
+
+                    controlnet_mlsd_num_inference_step = gr.Slider(
+                        minimum=1,
+                        maximum=100,
+                        step=1,
+                        value=50,
+                        label="Num Inference Step",
+                    )
+
+                controlnet_mlsd_predict = gr.Button(value="Generator")
+
+            with gr.Column():
+                output_image = gr.Image(label="Output")
+
+        gr.Examples(
+            fn=stable_diffusion_controlnet_mlsd,
+            examples=[
+                [
+                    data_list[0],
+                    stable_model_list[0],
+                    stable_prompt_list[0],
+                    stable_negative_prompt_list[0],
+                    7.5,
+                    50,
+                ]
+            ],
+            inputs=[
+                controlnet_mlsd_image_file,
+                controlnet_mlsd_model_id,
+                controlnet_mlsd_prompt,
+                controlnet_mlsd_negative_prompt,
+                controlnet_mlsd_guidance_scale,
+                controlnet_mlsd_num_inference_step,
+            ],
+            outputs=[output_image],
+            label="ControlNet-MLSD Example",
+            cache_examples=False,
+        )
+
+        controlnet_mlsd_predict.click(
+            fn=stable_diffusion_controlnet_mlsd,
+            inputs=[
+                controlnet_mlsd_image_file,
+                controlnet_mlsd_model_id,
+                controlnet_mlsd_prompt,
+                controlnet_mlsd_negative_prompt,
+                controlnet_mlsd_guidance_scale,
+                controlnet_mlsd_num_inference_step,
+            ],
+            outputs=output_image,
+        )

diffusion_webui/controlnet/controlnet_pose.py

@@ -0,0 +1,170 @@
+import gradio as gr
+import torch
+from controlnet_aux import OpenposeDetector
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+from PIL import Image
+
+stable_model_list = [
+    "runwayml/stable-diffusion-v1-5",
+    "stabilityai/stable-diffusion-2-1",
+]
+
+controlnet_pose_model_list = [
+    "lllyasviel/sd-controlnet-openpose",
+    "thibaud/controlnet-sd21-openpose-diffusers",
+]
+
+stable_prompt_list = ["a photo of a man.", "a photo of a girl."]
+
+stable_negative_prompt_list = ["bad, ugly", "deformed"]
+
+data_list = [
+    "data/test.png",
+]
+
+
+def controlnet_pose(image_path: str, controlnet_pose_model_path: str):
+    openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
+
+    image = Image.open(image_path)
+    image = openpose(image)
+
+    controlnet = ControlNetModel.from_pretrained(
+        controlnet_pose_model_path, torch_dtype=torch.float16
+    )
+
+    return controlnet, image
+
+
+def stable_diffusion_controlnet_pose(
+    image_path: str,
+    stable_model_path: str,
+    controlnet_pose_model_path: str,
+    prompt: str,
+    negative_prompt: str,
+    guidance_scale: int,
+    num_inference_step: int,
+):
+
+    controlnet, image = controlnet_pose(
+        image_path=image_path,
+        controlnet_pose_model_path=controlnet_pose_model_path,
+    )
+
+    pipe = StableDiffusionControlNetPipeline.from_pretrained(
+        pretrained_model_name_or_path=-stable_model_path,
+        controlnet=controlnet,
+        safety_checker=None,
+        torch_dtype=torch.float16,
+    )
+
+    pipe.to("cuda")
+    pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+    pipe.enable_xformers_memory_efficient_attention()
+
+    output = pipe(
+        prompt=prompt,
+        image=image,
+        negative_prompt=negative_prompt,
+        num_inference_steps=num_inference_step,
+        guidance_scale=guidance_scale,
+    ).images
+
+    return output[0]
+
+
+def stable_diffusion_controlnet_pose_app():
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                controlnet_pose_image_file = gr.Image(
+                    type="filepath", label="Image"
+                )
+
+                controlnet_pose_stable_model_id = gr.Dropdown(
+                    choices=stable_model_list,
+                    value=stable_model_list[0],
+                    label="Stable Model Id",
+                )
+
+                controlnet_pose_model_id = gr.Dropdown(
+                    choices=controlnet_pose_model_list,
+                    value=controlnet_pose_model_list[0],
+                    label="ControlNet Model Id",
+                )
+
+                controlnet_pose_prompt = gr.Textbox(
+                    lines=1, value=stable_prompt_list[0], label="Prompt"
+                )
+
+                controlnet_pose_negative_prompt = gr.Textbox(
+                    lines=1,
+                    value=stable_negative_prompt_list[0],
+                    label="Negative Prompt",
+                )
+
+                with gr.Accordion("Advanced Options", open=False):
+                    controlnet_pose_guidance_scale = gr.Slider(
+                        minimum=0.1,
+                        maximum=15,
+                        step=0.1,
+                        value=7.5,
+                        label="Guidance Scale",
+                    )
+
+                    controlnet_pose_num_inference_step = gr.Slider(
+                        minimum=1,
+                        maximum=100,
+                        step=1,
+                        value=50,
+                        label="Num Inference Step",
+                    )
+
+                controlnet_pose_predict = gr.Button(value="Generator")
+
+            with gr.Column():
+                output_image = gr.Image(label="Output")
+
+        gr.Examples(
+            fn=stable_diffusion_controlnet_pose,
+            examples=[
+                [
+                    data_list[0],
+                    stable_model_list[0],
+                    controlnet_pose_model_list[0],
+                    stable_prompt_list[0],
+                    stable_negative_prompt_list[0],
+                    7.5,
+                    50,
+                ]
+            ],
+            inputs=[
+                controlnet_pose_image_file,
+                controlnet_pose_stable_model_id,
+                controlnet_pose_model_id,
+                controlnet_pose_prompt,
+                controlnet_pose_negative_prompt,
+                controlnet_pose_guidance_scale,
+                controlnet_pose_num_inference_step,
+            ],
+            outputs=[output_image],
+            label="ControlNet Pose Example",
+            cache_examples=False,
+        )
+        controlnet_pose_predict.click(
+            fn=stable_diffusion_controlnet_pose,
+            inputs=[
+                controlnet_pose_image_file,
+                controlnet_pose_stable_model_id,
+                controlnet_pose_model_id,
+                controlnet_pose_prompt,
+                controlnet_pose_negative_prompt,
+                controlnet_pose_guidance_scale,
+                controlnet_pose_num_inference_step,
+            ],
+            outputs=output_image,
+        )

diffusion_webui/controlnet/controlnet_scribble.py

@@ -0,0 +1,170 @@
+import gradio as gr
+import torch
+from controlnet_aux import HEDdetector
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+from PIL import Image
+
+stable_model_list = [
+    "runwayml/stable-diffusion-v1-5",
+    "stabilityai/stable-diffusion-2-1",
+]
+
+controlnet_hed_model_list = [
+    "lllyasviel/sd-controlnet-scribble",
+    "thibaud/controlnet-sd21-scribble-diffusers",
+]
+
+stable_prompt_list = ["a photo of a man.", "a photo of a girl."]
+
+stable_negative_prompt_list = ["bad, ugly", "deformed"]
+
+data_list = [
+    "data/test.png",
+]
+
+
+def controlnet_scribble(image_path: str, controlnet_hed_model_path: str):
+    hed = HEDdetector.from_pretrained("lllyasviel/ControlNet")
+
+    image = Image.open(image_path)
+    image = hed(image, scribble=True)
+
+    controlnet = ControlNetModel.from_pretrained(
+        controlnet_hed_model_path, torch_dtype=torch.float16
+    )
+
+    return controlnet, image
+
+
+def stable_diffusion_controlnet_scribble(
+    image_path: str,
+    stable_model_path: str,
+    controlnet_hed_model_path: str,
+    prompt: str,
+    negative_prompt: str,
+    guidance_scale: int,
+    num_inference_step: int,
+):
+
+    controlnet, image = controlnet_scribble(
+        image_path=image_path,
+        controlnet_hed_model_path=controlnet_hed_model_path,
+    )
+
+    pipe = StableDiffusionControlNetPipeline.from_pretrained(
+        pretrained_model_name_or_path=stable_model_path,
+        controlnet=controlnet,
+        safety_checker=None,
+        torch_dtype=torch.float16,
+    )
+
+    pipe.to("cuda")
+    pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+    pipe.enable_xformers_memory_efficient_attention()
+
+    output = pipe(
+        prompt=prompt,
+        image=image,
+        negative_prompt=negative_prompt,
+        num_inference_steps=num_inference_step,
+        guidance_scale=guidance_scale,
+    ).images
+
+    return output[0]
+
+
+def stable_diffusion_controlnet_scribble_app():
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                controlnet_scribble_image_file = gr.Image(
+                    type="filepath", label="Image"
+                )
+
+                controlnet_scribble_stable_model_id = gr.Dropdown(
+                    choices=stable_model_list,
+                    value=stable_model_list[0],
+                    label="Stable v1.5 Model Id",
+                )
+
+                controlnet_scribble_model_id = gr.Dropdown(
+                    choices=controlnet_hed_model_list,
+                    value=controlnet_hed_model_list[1],
+                    label="ControlNet Model Id",
+                )
+
+                controlnet_scribble_prompt = gr.Textbox(
+                    lines=1, value=stable_prompt_list[0], label="Prompt"
+                )
+
+                controlnet_scribble_negative_prompt = gr.Textbox(
+                    lines=1,
+                    value=stable_negative_prompt_list[0],
+                    label="Negative Prompt",
+                )
+
+                with gr.Accordion("Advanced Options", open=False):
+                    controlnet_scribble_guidance_scale = gr.Slider(
+                        minimum=0.1,
+                        maximum=15,
+                        step=0.1,
+                        value=7.5,
+                        label="Guidance Scale",
+                    )
+
+                    controlnet_scribble_num_inference_step = gr.Slider(
+                        minimum=1,
+                        maximum=100,
+                        step=1,
+                        value=50,
+                        label="Num Inference Step",
+                    )
+
+                controlnet_scribble_predict = gr.Button(value="Generator")
+
+            with gr.Column():
+                output_image = gr.Image(label="Output")
+
+        gr.Examples(
+            fn=stable_diffusion_controlnet_scribble,
+            examples=[
+                [
+                    data_list[0],
+                    stable_model_list[0],
+                    controlnet_hed_model_list[0],
+                    stable_prompt_list[0],
+                    stable_negative_prompt_list[0],
+                    7.5,
+                    50,
+                ],
+            ],
+            inputs=[
+                controlnet_scribble_image_file,
+                controlnet_scribble_stable_model_id,
+                controlnet_scribble_model_id,
+                controlnet_scribble_prompt,
+                controlnet_scribble_negative_prompt,
+                controlnet_scribble_guidance_scale,
+                controlnet_scribble_num_inference_step,
+            ],
+            outputs=[output_image],
+            label="ControlNet Scribble Example",
+            cache_examples=False,
+        )
+        controlnet_scribble_predict.click(
+            fn=stable_diffusion_controlnet_scribble,
+            inputs=[
+                controlnet_scribble_image_file,
+                controlnet_scribble_stable_model_id,
+                controlnet_scribble_model_id,
+                controlnet_scribble_prompt,
+                controlnet_scribble_negative_prompt,
+                controlnet_scribble_guidance_scale,
+                controlnet_scribble_num_inference_step,
+            ],
+            outputs=output_image,
+        )

diffusion_webui/controlnet/controlnet_seg.py

@@ -0,0 +1,329 @@
+import gradio as gr
+import numpy as np
+import torch
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+from PIL import Image
+from transformers import AutoImageProcessor, UperNetForSemanticSegmentation
+
+stable_model_list = [
+    "runwayml/stable-diffusion-v1-5",
+]
+
+stable_prompt_list = ["a photo of a man.", "a photo of a girl."]
+
+stable_negative_prompt_list = ["bad, ugly", "deformed"]
+
+data_list = [
+    "data/test.png",
+]
+
+
+def ade_palette():
+    """ADE20K palette that maps each class to RGB values."""
+    return [
+        [120, 120, 120],
+        [180, 120, 120],
+        [6, 230, 230],
+        [80, 50, 50],
+        [4, 200, 3],
+        [120, 120, 80],
+        [140, 140, 140],
+        [204, 5, 255],
+        [230, 230, 230],
+        [4, 250, 7],
+        [224, 5, 255],
+        [235, 255, 7],
+        [150, 5, 61],
+        [120, 120, 70],
+        [8, 255, 51],
+        [255, 6, 82],
+        [143, 255, 140],
+        [204, 255, 4],
+        [255, 51, 7],
+        [204, 70, 3],
+        [0, 102, 200],
+        [61, 230, 250],
+        [255, 6, 51],
+        [11, 102, 255],
+        [255, 7, 71],
+        [255, 9, 224],
+        [9, 7, 230],
+        [220, 220, 220],
+        [255, 9, 92],
+        [112, 9, 255],
+        [8, 255, 214],
+        [7, 255, 224],
+        [255, 184, 6],
+        [10, 255, 71],
+        [255, 41, 10],
+        [7, 255, 255],
+        [224, 255, 8],
+        [102, 8, 255],
+        [255, 61, 6],
+        [255, 194, 7],
+        [255, 122, 8],
+        [0, 255, 20],
+        [255, 8, 41],
+        [255, 5, 153],
+        [6, 51, 255],
+        [235, 12, 255],
+        [160, 150, 20],
+        [0, 163, 255],
+        [140, 140, 140],
+        [250, 10, 15],
+        [20, 255, 0],
+        [31, 255, 0],
+        [255, 31, 0],
+        [255, 224, 0],
+        [153, 255, 0],
+        [0, 0, 255],
+        [255, 71, 0],
+        [0, 235, 255],
+        [0, 173, 255],
+        [31, 0, 255],
+        [11, 200, 200],
+        [255, 82, 0],
+        [0, 255, 245],
+        [0, 61, 255],
+        [0, 255, 112],
+        [0, 255, 133],
+        [255, 0, 0],
+        [255, 163, 0],
+        [255, 102, 0],
+        [194, 255, 0],
+        [0, 143, 255],
+        [51, 255, 0],
+        [0, 82, 255],
+        [0, 255, 41],
+        [0, 255, 173],
+        [10, 0, 255],
+        [173, 255, 0],
+        [0, 255, 153],
+        [255, 92, 0],
+        [255, 0, 255],
+        [255, 0, 245],
+        [255, 0, 102],
+        [255, 173, 0],
+        [255, 0, 20],
+        [255, 184, 184],
+        [0, 31, 255],
+        [0, 255, 61],
+        [0, 71, 255],
+        [255, 0, 204],
+        [0, 255, 194],
+        [0, 255, 82],
+        [0, 10, 255],
+        [0, 112, 255],
+        [51, 0, 255],
+        [0, 194, 255],
+        [0, 122, 255],
+        [0, 255, 163],
+        [255, 153, 0],
+        [0, 255, 10],
+        [255, 112, 0],
+        [143, 255, 0],
+        [82, 0, 255],
+        [163, 255, 0],
+        [255, 235, 0],
+        [8, 184, 170],
+        [133, 0, 255],
+        [0, 255, 92],
+        [184, 0, 255],
+        [255, 0, 31],
+        [0, 184, 255],
+        [0, 214, 255],
+        [255, 0, 112],
+        [92, 255, 0],
+        [0, 224, 255],
+        [112, 224, 255],
+        [70, 184, 160],
+        [163, 0, 255],
+        [153, 0, 255],
+        [71, 255, 0],
+        [255, 0, 163],
+        [255, 204, 0],
+        [255, 0, 143],
+        [0, 255, 235],
+        [133, 255, 0],
+        [255, 0, 235],
+        [245, 0, 255],
+        [255, 0, 122],
+        [255, 245, 0],
+        [10, 190, 212],
+        [214, 255, 0],
+        [0, 204, 255],
+        [20, 0, 255],
+        [255, 255, 0],
+        [0, 153, 255],
+        [0, 41, 255],
+        [0, 255, 204],
+        [41, 0, 255],
+        [41, 255, 0],
+        [173, 0, 255],
+        [0, 245, 255],
+        [71, 0, 255],
+        [122, 0, 255],
+        [0, 255, 184],
+        [0, 92, 255],
+        [184, 255, 0],
+        [0, 133, 255],
+        [255, 214, 0],
+        [25, 194, 194],
+        [102, 255, 0],
+        [92, 0, 255],
+    ]
+
+
+def controlnet_mlsd(image_path: str):
+    image_processor = AutoImageProcessor.from_pretrained(
+        "openmmlab/upernet-convnext-small"
+    )
+    image_segmentor = UperNetForSemanticSegmentation.from_pretrained(
+        "openmmlab/upernet-convnext-small"
+    )
+
+    image = Image.open(image_path).convert("RGB")
+    pixel_values = image_processor(image, return_tensors="pt").pixel_values
+
+    with torch.no_grad():
+        outputs = image_segmentor(pixel_values)
+
+    seg = image_processor.post_process_semantic_segmentation(
+        outputs, target_sizes=[image.size[::-1]]
+    )[0]
+
+    color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8)
+    palette = np.array(ade_palette())
+
+    for label, color in enumerate(palette):
+        color_seg[seg == label, :] = color
+
+    color_seg = color_seg.astype(np.uint8)
+    image = Image.fromarray(color_seg)
+    controlnet = ControlNetModel.from_pretrained(
+        "lllyasviel/sd-controlnet-seg", torch_dtype=torch.float16
+    )
+
+    return controlnet, image
+
+
+def stable_diffusion_controlnet_seg(
+    image_path: str,
+    model_path: str,
+    prompt: str,
+    negative_prompt: str,
+    guidance_scale: int,
+    num_inference_step: int,
+):
+
+    controlnet, image = controlnet_mlsd(image_path=image_path)
+
+    pipe = StableDiffusionControlNetPipeline.from_pretrained(
+        pretrained_model_name_or_path=model_path,
+        controlnet=controlnet,
+        safety_checker=None,
+        torch_dtype=torch.float16,
+    )
+
+    pipe.to("cuda")
+    pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+    pipe.enable_xformers_memory_efficient_attention()
+
+    output = pipe(
+        prompt=prompt,
+        image=image,
+        negative_prompt=negative_prompt,
+        num_inference_steps=num_inference_step,
+        guidance_scale=guidance_scale,
+    ).images
+
+    return output[0]
+
+
+def stable_diffusion_controlnet_seg_app():
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                controlnet_seg_image_file = gr.Image(
+                    type="filepath", label="Image"
+                )
+
+                controlnet_seg_model_id = gr.Dropdown(
+                    choices=stable_model_list,
+                    value=stable_model_list[0],
+                    label="Stable Model Id",
+                )
+
+                controlnet_seg_prompt = gr.Textbox(
+                    lines=1, value=stable_prompt_list[0], label="Prompt"
+                )
+
+                controlnet_seg_negative_prompt = gr.Textbox(
+                    lines=1,
+                    value=stable_negative_prompt_list[0],
+                    label="Negative Prompt",
+                )
+
+                with gr.Accordion("Advanced Options", open=False):
+                    controlnet_seg_guidance_scale = gr.Slider(
+                        minimum=0.1,
+                        maximum=15,
+                        step=0.1,
+                        value=7.5,
+                        label="Guidance Scale",
+                    )
+
+                    controlnet_seg_num_inference_step = gr.Slider(
+                        minimum=1,
+                        maximum=100,
+                        step=1,
+                        value=50,
+                        label="Num Inference Step",
+                    )
+
+                controlnet_seg_predict = gr.Button(value="Generator")
+
+            with gr.Column():
+                output_image = gr.Image(label="Output")
+
+        gr.Examples(
+            fn=stable_diffusion_controlnet_seg,
+            examples=[
+                [
+                    data_list[0],
+                    stable_model_list[0],
+                    stable_prompt_list[0],
+                    stable_negative_prompt_list[0],
+                    7.5,
+                    50,
+                ],
+            ],
+            inputs=[
+                controlnet_seg_image_file,
+                controlnet_seg_model_id,
+                controlnet_seg_prompt,
+                controlnet_seg_negative_prompt,
+                controlnet_seg_guidance_scale,
+                controlnet_seg_num_inference_step,
+            ],
+            outputs=[output_image],
+            cache_examples=False,
+            label="ControlNet Segmentation Example",
+        )
+
+        controlnet_seg_predict.click(
+            fn=stable_diffusion_controlnet_seg,
+            inputs=[
+                controlnet_seg_image_file,
+                controlnet_seg_model_id,
+                controlnet_seg_prompt,
+                controlnet_seg_negative_prompt,
+                controlnet_seg_guidance_scale,
+                controlnet_seg_num_inference_step,
+            ],
+            outputs=[output_image],
+        )

diffusion_webui/helpers.py

@@ -0,0 +1,33 @@
+from diffusion_webui.controlnet.controlnet_canny import (
+    stable_diffusion_controlnet_canny_app,
+)
+from diffusion_webui.controlnet.controlnet_depth import (
+    stable_diffusion_controlnet_depth_app,
+)
+from diffusion_webui.controlnet.controlnet_hed import (
+    stable_diffusion_controlnet_hed_app,
+)
+from diffusion_webui.controlnet.controlnet_mlsd import (
+    stable_diffusion_controlnet_mlsd_app,
+)
+from diffusion_webui.controlnet.controlnet_pose import (
+    stable_diffusion_controlnet_pose_app,
+)
+from diffusion_webui.controlnet.controlnet_scribble import (
+    stable_diffusion_controlnet_scribble_app,
+)
+from diffusion_webui.controlnet.controlnet_seg import (
+    stable_diffusion_controlnet_seg_app,
+)
+from diffusion_webui.stable_diffusion.img2img_app import (
+    stable_diffusion_img2img_app,
+)
+from diffusion_webui.stable_diffusion.inpaint_app import (
+    stable_diffusion_inpaint_app,
+)
+from diffusion_webui.stable_diffusion.keras_txt2img import (
+    keras_stable_diffusion_app,
+)
+from diffusion_webui.stable_diffusion.text2img_app import (
+    stable_diffusion_text2img_app,
+)

diffusion_webui/stable_diffusion/img2img_app.py

@@ -0,0 +1,131 @@
+import gradio as gr
+import torch
+from diffusers import DDIMScheduler, StableDiffusionImg2ImgPipeline
+from PIL import Image
+
+stable_model_list = [
+    "runwayml/stable-diffusion-v1-5",
+    "stabilityai/stable-diffusion-2-1",
+]
+
+stable_prompt_list = ["a photo of a man.", "a photo of a girl."]
+
+stable_negative_prompt_list = ["bad, ugly", "deformed"]
+
+data_list = [
+    "data/test.png",
+]
+
+
+def stable_diffusion_img2img(
+    image_path: str,
+    model_path: str,
+    prompt: str,
+    negative_prompt: str,
+    guidance_scale: int,
+    num_inference_step: int,
+):
+
+    image = Image.open(image_path)
+
+    pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
+        model_path, safety_checker=None, torch_dtype=torch.float16
+    )
+    pipe.to("cuda")
+    pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
+    pipe.enable_xformers_memory_efficient_attention()
+
+    output = pipe(
+        prompt=prompt,
+        image=image,
+        negative_prompt=negative_prompt,
+        num_inference_steps=num_inference_step,
+        guidance_scale=guidance_scale,
+    ).images
+
+    return output[0]
+
+
+def stable_diffusion_img2img_app():
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                image2image2_image_file = gr.Image(
+                    type="filepath", label="Image"
+                )
+
+                image2image_model_path = gr.Dropdown(
+                    choices=stable_model_list,
+                    value=stable_model_list[0],
+                    label="Image-Image Model Id",
+                )
+
+                image2image_prompt = gr.Textbox(
+                    lines=1, value=stable_prompt_list[0], label="Prompt"
+                )
+
+                image2image_negative_prompt = gr.Textbox(
+                    lines=1,
+                    value=stable_negative_prompt_list[0],
+                    label="Negative Prompt",
+                )
+
+                with gr.Accordion("Advanced Options", open=False):
+                    image2image_guidance_scale = gr.Slider(
+                        minimum=0.1,
+                        maximum=15,
+                        step=0.1,
+                        value=7.5,
+                        label="Guidance Scale",
+                    )
+
+                    image2image_num_inference_step = gr.Slider(
+                        minimum=1,
+                        maximum=100,
+                        step=1,
+                        value=50,
+                        label="Num Inference Step",
+                    )
+
+                image2image_predict = gr.Button(value="Generator")
+
+            with gr.Column():
+                output_image = gr.Image(label="Output")
+
+        gr.Examples(
+            fn=stable_diffusion_img2img,
+            examples=[
+                [
+                    data_list[0],
+                    stable_model_list[0],
+                    stable_prompt_list[0],
+                    stable_negative_prompt_list[0],
+                    7.5,
+                    50,
+                ],
+            ],
+            inputs=[
+                image2image2_image_file,
+                image2image_model_path,
+                image2image_prompt,
+                image2image_negative_prompt,
+                image2image_guidance_scale,
+                image2image_num_inference_step,
+            ],
+            outputs=[output_image],
+            cache_examples=False,
+            label="Image-Image Generator",
+        )
+
+        image2image_predict.click(
+            fn=stable_diffusion_img2img,
+            inputs=[
+                image2image2_image_file,
+                image2image_model_path,
+                image2image_prompt,
+                image2image_negative_prompt,
+                image2image_guidance_scale,
+                image2image_num_inference_step,
+            ],
+            outputs=[output_image],
+        )

diffusion_webui/stable_diffusion/inpaint_app.py

@@ -0,0 +1,108 @@
+import gradio as gr
+import torch
+from diffusers import DDIMScheduler, DiffusionPipeline
+
+stable_inpiant_model_list = [
+    "stabilityai/stable-diffusion-2-inpainting",
+    "runwayml/stable-diffusion-inpainting",
+]
+
+stable_prompt_list = ["a photo of a man.", "a photo of a girl."]
+
+stable_negative_prompt_list = ["bad, ugly", "deformed"]
+
+
+def stable_diffusion_inpaint(
+    dict: str,
+    model_path: str,
+    prompt: str,
+    negative_prompt: str,
+    guidance_scale: int,
+    num_inference_step: int,
+):
+
+    image = dict["image"].convert("RGB").resize((512, 512))
+    mask_image = dict["mask"].convert("RGB").resize((512, 512))
+    pipe = DiffusionPipeline.from_pretrained(
+        model_path,
+        revision="fp16",
+        torch_dtype=torch.float16,
+    )
+    pipe.to("cuda")
+    pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
+    pipe.enable_xformers_memory_efficient_attention()
+
+    output = pipe(
+        prompt=prompt,
+        image=image,
+        mask_image=mask_image,
+        negative_prompt=negative_prompt,
+        num_inference_steps=num_inference_step,
+        guidance_scale=guidance_scale,
+    ).images
+
+    return output[0]
+
+
+def stable_diffusion_inpaint_app():
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                inpaint_image_file = gr.Image(
+                    source="upload",
+                    tool="sketch",
+                    elem_id="image_upload",
+                    type="pil",
+                    label="Upload",
+                )
+
+                inpaint_model_id = gr.Dropdown(
+                    choices=stable_inpiant_model_list,
+                    value=stable_inpiant_model_list[0],
+                    label="Inpaint Model Id",
+                )
+
+                inpaint_prompt = gr.Textbox(
+                    lines=1, value=stable_prompt_list[0], label="Prompt"
+                )
+
+                inpaint_negative_prompt = gr.Textbox(
+                    lines=1,
+                    value=stable_negative_prompt_list[0],
+                    label="Negative Prompt",
+                )
+
+                with gr.Accordion("Advanced Options", open=False):
+                    inpaint_guidance_scale = gr.Slider(
+                        minimum=0.1,
+                        maximum=15,
+                        step=0.1,
+                        value=7.5,
+                        label="Guidance Scale",
+                    )
+
+                    inpaint_num_inference_step = gr.Slider(
+                        minimum=1,
+                        maximum=100,
+                        step=1,
+                        value=50,
+                        label="Num Inference Step",
+                    )
+
+                inpaint_predict = gr.Button(value="Generator")
+
+            with gr.Column():
+                output_image = gr.Gallery(label="Outputs")
+
+        inpaint_predict.click(
+            fn=stable_diffusion_inpaint,
+            inputs=[
+                inpaint_image_file,
+                inpaint_model_id,
+                inpaint_prompt,
+                inpaint_negative_prompt,
+                inpaint_guidance_scale,
+                inpaint_num_inference_step,
+            ],
+            outputs=output_image,
+        )

diffusion_webui/stable_diffusion/keras_txt2img.py

@@ -0,0 +1,148 @@
+import gradio as gr
+import tensorflow as tf
+from huggingface_hub import from_pretrained_keras
+from keras_cv import models
+from tensorflow import keras
+
+keras_model_list = [
+    "keras-dreambooth/keras_diffusion_lowpoly_world",
+    "keras-dreambooth/pink-floyd-division-bell",
+    "keras-dreambooth/dreambooth_diffusion_model",
+]
+
+stable_prompt_list = [
+    "a photo of lowpoly_world",
+    "Flower vase inspired by pink floyd division bell",
+]
+
+stable_negative_prompt_list = ["bad, ugly", "deformed"]
+
+
+def keras_stable_diffusion(
+    model_path: str,
+    prompt: str,
+    negative_prompt: str,
+    guidance_scale: int,
+    num_inference_step: int,
+    height: int,
+    width: int,
+):
+
+    with tf.device("/GPU:0"):
+        keras.mixed_precision.set_global_policy("mixed_float16")
+
+        sd_dreambooth_model = models.StableDiffusion(
+            img_width=height, img_height=width
+        )
+
+        db_diffusion_model = from_pretrained_keras(model_path)
+        sd_dreambooth_model._diffusion_model = db_diffusion_model
+
+        generated_images = sd_dreambooth_model.text_to_image(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            num_steps=num_inference_step,
+            unconditional_guidance_scale=guidance_scale,
+        )
+
+    return generated_images
+
+
+def keras_stable_diffusion_app():
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                keras_text2image_model_path = gr.Dropdown(
+                    choices=keras_model_list,
+                    value=keras_model_list[0],
+                    label="Text-Image Model Id",
+                )
+
+                keras_text2image_prompt = gr.Textbox(
+                    lines=1, value=stable_prompt_list[0], label="Prompt"
+                )
+
+                keras_text2image_negative_prompt = gr.Textbox(
+                    lines=1,
+                    value=stable_negative_prompt_list[0],
+                    label="Negative Prompt",
+                )
+
+                with gr.Accordion("Advanced Options", open=False):
+                    keras_text2image_guidance_scale = gr.Slider(
+                        minimum=0.1,
+                        maximum=15,
+                        step=0.1,
+                        value=7.5,
+                        label="Guidance Scale",
+                    )
+
+                    keras_text2image_num_inference_step = gr.Slider(
+                        minimum=1,
+                        maximum=100,
+                        step=1,
+                        value=50,
+                        label="Num Inference Step",
+                    )
+
+                    keras_text2image_height = gr.Slider(
+                        minimum=128,
+                        maximum=1280,
+                        step=32,
+                        value=512,
+                        label="Image Height",
+                    )
+
+                    keras_text2image_width = gr.Slider(
+                        minimum=128,
+                        maximum=1280,
+                        step=32,
+                        value=512,
+                        label="Image Height",
+                    )
+
+                keras_text2image_predict = gr.Button(value="Generator")
+
+            with gr.Column():
+                output_image = gr.Gallery(label="Output")
+
+        gr.Examples(
+            fn=keras_stable_diffusion,
+            inputs=[
+                keras_text2image_model_path,
+                keras_text2image_prompt,
+                keras_text2image_negative_prompt,
+                keras_text2image_guidance_scale,
+                keras_text2image_num_inference_step,
+                keras_text2image_height,
+                keras_text2image_width,
+            ],
+            outputs=[output_image],
+            examples=[
+                [
+                    keras_model_list[0],
+                    stable_prompt_list[0],
+                    stable_negative_prompt_list[0],
+                    7.5,
+                    50,
+                    512,
+                    512,
+                ],
+            ],
+            label="Keras Stable Diffusion Example",
+            cache_examples=False,
+        )
+
+        keras_text2image_predict.click(
+            fn=keras_stable_diffusion,
+            inputs=[
+                keras_text2image_model_path,
+                keras_text2image_prompt,
+                keras_text2image_negative_prompt,
+                keras_text2image_guidance_scale,
+                keras_text2image_num_inference_step,
+                keras_text2image_height,
+                keras_text2image_width,
+            ],
+            outputs=output_image,
+        )

diffusion_webui/stable_diffusion/text2img_app.py

@@ -0,0 +1,146 @@
+import gradio as gr
+import torch
+from diffusers import DDIMScheduler, StableDiffusionPipeline
+
+stable_model_list = [
+    "runwayml/stable-diffusion-v1-5",
+    "stabilityai/stable-diffusion-2-1",
+    "sd-dreambooth-library/disco-diffusion-style",
+    "prompthero/openjourney-v2",
+    "andite/anything-v4.0",
+    "Lykon/DreamShaper",
+    "nitrosocke/Nitro-Diffusion",
+    "dreamlike-art/dreamlike-diffusion-1.0",
+]
+
+stable_prompt_list = ["a photo of a man.", "a photo of a girl."]
+
+stable_negative_prompt_list = ["bad, ugly", "deformed"]
+
+
+def stable_diffusion_text2img(
+    model_path: str,
+    prompt: str,
+    negative_prompt: str,
+    guidance_scale: int,
+    num_inference_step: int,
+    height: int,
+    width: int,
+):
+
+    pipe = StableDiffusionPipeline.from_pretrained(
+        model_path, safety_checker=None, torch_dtype=torch.float16
+    ).to("cuda")
+
+    pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
+    pipe.enable_xformers_memory_efficient_attention()
+
+    images = pipe(
+        prompt,
+        height=height,
+        width=width,
+        negative_prompt=negative_prompt,
+        num_inference_steps=num_inference_step,
+        guidance_scale=guidance_scale,
+    ).images
+
+    return images[0]
+
+
+def stable_diffusion_text2img_app():
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                text2image_model_path = gr.Dropdown(
+                    choices=stable_model_list,
+                    value=stable_model_list[0],
+                    label="Text-Image Model Id",
+                )
+
+                text2image_prompt = gr.Textbox(
+                    lines=1, value=stable_prompt_list[0], label="Prompt"
+                )
+
+                text2image_negative_prompt = gr.Textbox(
+                    lines=1,
+                    value=stable_negative_prompt_list[0],
+                    label="Negative Prompt",
+                )
+
+                with gr.Accordion("Advanced Options", open=False):
+                    text2image_guidance_scale = gr.Slider(
+                        minimum=0.1,
+                        maximum=15,
+                        step=0.1,
+                        value=7.5,
+                        label="Guidance Scale",
+                    )
+
+                    text2image_num_inference_step = gr.Slider(
+                        minimum=1,
+                        maximum=100,
+                        step=1,
+                        value=50,
+                        label="Num Inference Step",
+                    )
+
+                    text2image_height = gr.Slider(
+                        minimum=128,
+                        maximum=1280,
+                        step=32,
+                        value=512,
+                        label="Image Height",
+                    )
+
+                    text2image_width = gr.Slider(
+                        minimum=128,
+                        maximum=1280,
+                        step=32,
+                        value=768,
+                        label="Image Width",
+                    )
+
+                text2image_predict = gr.Button(value="Generator")
+
+            with gr.Column():
+                output_image = gr.Image(label="Output")
+
+        gr.Examples(
+            examples=[
+                [
+                    stable_model_list[0],
+                    stable_prompt_list[0],
+                    stable_negative_prompt_list[0],
+                    7.5,
+                    50,
+                    512,
+                    768,
+                ]
+            ],
+            inputs=[
+                text2image_model_path,
+                text2image_prompt,
+                text2image_negative_prompt,
+                text2image_guidance_scale,
+                text2image_num_inference_step,
+                text2image_height,
+                text2image_width,
+            ],
+            outputs=[output_image],
+            cache_examples=False,
+            fn=stable_diffusion_text2img,
+            label="Text2Image Example",
+        )
+        text2image_predict.click(
+            fn=stable_diffusion_text2img,
+            inputs=[
+                text2image_model_path,
+                text2image_prompt,
+                text2image_negative_prompt,
+                text2image_guidance_scale,
+                text2image_num_inference_step,
+                text2image_height,
+                text2image_width,
+            ],
+            outputs=output_image,
+        )

pyproject.toml

@@ -0,0 +1,6 @@
+[tool.black]
+line-length = 80
+
+[tool.isort]
+line_length = 80
+profile = "black"

requirements.txt

@@ -0,0 +1,12 @@
+transformers 
+bitsandbytes==0.35.0 
+xformers 
+controlnet_aux 
+diffusers
+imageio
+gradio
+triton
+tensorflow
+huggingface-hub
+keras-cv
+pycocotools

script/code_formatter.sh

@@ -0,0 +1,2 @@
+black . --config pyproject.toml
+isort .