Update app.py

app.py

@@ -9,16 +9,12 @@ import torchvision.transforms.functional as TF
 
 from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, AutoencoderKL
 from diffusers import DDIMScheduler, EulerAncestralDiscreteScheduler
-from controlnet_aux import PidiNetDetector, HEDdetector
+from controlnet_aux import OpenposeDetector, MidasDetector, CannyDetector, LineartDetector, LineartAnimeDetector, MLSDdetector, HEDdetector, PidiNetDetector, NormalBaeDetector, SamDetector
 from diffusers.utils import load_image
 from huggingface_hub import HfApi
 from pathlib import Path
 from PIL import Image, ImageOps
-import torch
-import numpy as np
 import cv2
-import os
-import random
 import spaces
 from gradio_imageslider import ImageSlider
 
@@ -27,6 +23,7 @@ function refresh() {
     const url = new URL(window.location);
 }
 """
+
 def nms(x, t, s):
     x = cv2.GaussianBlur(x.astype(np.float32), (0, 0), s)
 
@@ -78,72 +75,38 @@ style_list = [
         "prompt": "cinematic still {prompt} . emotional, harmonious, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy",
         "negative_prompt": "anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured",
     },
-    {
-        "name": "3D Model",
-        "prompt": "professional 3d model {prompt} . octane render, highly detailed, volumetric, dramatic lighting",
-        "negative_prompt": "ugly, deformed, noisy, low poly, blurry, painting",
-    },
-    {
-        "name": "Anime",
-        "prompt": "anime artwork {prompt} . anime style, key visual, vibrant, studio anime,  highly detailed",
-        "negative_prompt": "photo, deformed, black and white, realism, disfigured, low contrast",
-    },
-    {
-        "name": "Digital Art",
-        "prompt": "concept art {prompt} . digital artwork, illustrative, painterly, matte painting, highly detailed",
-        "negative_prompt": "photo, photorealistic, realism, ugly",
-    },
-    {
-        "name": "Photographic",
-        "prompt": "cinematic photo {prompt} . 35mm photograph, film, bokeh, professional, 4k, highly detailed",
-        "negative_prompt": "drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly",
-    },
-    {
-        "name": "Pixel art",
-        "prompt": "pixel-art {prompt} . low-res, blocky, pixel art style, 8-bit graphics",
-        "negative_prompt": "sloppy, messy, blurry, noisy, highly detailed, ultra textured, photo, realistic",
-    },
-    {
-        "name": "Fantasy art",
-        "prompt": "ethereal fantasy concept art of  {prompt} . magnificent, celestial, ethereal, painterly, epic, majestic, magical, fantasy art, cover art, dreamy",
-        "negative_prompt": "photographic, realistic, realism, 35mm film, dslr, cropped, frame, text, deformed, glitch, noise, noisy, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, sloppy, duplicate, mutated, black and white",
-    },
-    {
-        "name": "Neonpunk",
-        "prompt": "neonpunk style {prompt} . cyberpunk, vaporwave, neon, vibes, vibrant, stunningly beautiful, crisp, detailed, sleek, ultramodern, magenta highlights, dark purple shadows, high contrast, cinematic, ultra detailed, intricate, professional",
-        "negative_prompt": "painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured",
-    },
-    {
-        "name": "Manga",
-        "prompt": "manga style {prompt} . vibrant, high-energy, detailed, iconic, Japanese comic style",
-        "negative_prompt": "ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, Western comic style",
-    },
+    # ... (other styles)
 ]
 
 styles = {k["name"]: (k["prompt"], k["negative_prompt"]) for k in style_list}
 STYLE_NAMES = list(styles.keys())
 DEFAULT_STYLE_NAME = "(No style)"
 
-
 def apply_style(style_name: str, positive: str, negative: str = "") -> tuple[str, str]:
     p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
     return p.replace("{prompt}", positive), n + negative
 
-
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
 eulera_scheduler = EulerAncestralDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler")
 
+# Initialize all detectors
+openpose_detector = OpenposeDetector.from_pretrained('lllyasviel/ControlNet')
+midas_detector = MidasDetector.from_pretrained('lllyasviel/ControlNet')
+canny_detector = CannyDetector()
+lineart_detector = LineartDetector.from_pretrained("lllyasviel/Annotators")
+anime_lineart_detector = LineartAnimeDetector.from_pretrained("lllyasviel/Annotators")
+mlsd_detector = MLSDdetector.from_pretrained('lllyasviel/ControlNet')
+hed_detector = HEDdetector.from_pretrained('lllyasviel/ControlNet')
+pidi_detector = PidiNetDetector.from_pretrained('lllyasviel/Annotators')
+normal_detector = NormalBaeDetector.from_pretrained('lllyasviel/Annotators')
+sam_detector = SamDetector.from_pretrained('ybelkada/segment-anything', subfolder='checkpoints')
 
 controlnet = ControlNetModel.from_pretrained(
     "xinsir/controlnet-union-sdxl-1.0",
     torch_dtype=torch.float16
 )
-controlnet_canny = ControlNetModel.from_pretrained(
-    "xinsir/controlnet-canny-sdxl-1.0",
-    torch_dtype=torch.float16
-)
-# when test with other base model, you need to change the vae also.
+
 vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
 
 pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
@@ -154,36 +117,8 @@ pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
     scheduler=eulera_scheduler,
 )
 pipe.to(device)
-# Load model.
-pipe_canny = StableDiffusionXLControlNetPipeline.from_pretrained(
-    "stabilityai/stable-diffusion-xl-base-1.0",
-    controlnet=controlnet_canny,
-    vae=vae,
-    safety_checker=None,
-    torch_dtype=torch.float16,
-    scheduler=eulera_scheduler,
-)
-
-pipe_canny.to(device)
 
 MAX_SEED = np.iinfo(np.int32).max
-processor = HEDdetector.from_pretrained('lllyasviel/Annotators')
-def nms(x, t, s):
-    x = cv2.GaussianBlur(x.astype(np.float32), (0, 0), s)
-
-    f1 = np.array([[0, 0, 0], [1, 1, 1], [0, 0, 0]], dtype=np.uint8)
-    f2 = np.array([[0, 1, 0], [0, 1, 0], [0, 1, 0]], dtype=np.uint8)
-    f3 = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype=np.uint8)
-    f4 = np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]], dtype=np.uint8)
-
-    y = np.zeros_like(x)
-
-    for f in [f1, f2, f3, f4]:
-        np.putmask(y, cv2.dilate(x, kernel=f) == x, x)
-
-    z = np.zeros_like(y, dtype=np.uint8)
-    z[y > t] = 255
-    return z
 
 def randomize_seed_fn(seed: int, randomize_seed: bool) -> int:
     if randomize_seed:
@@ -200,71 +135,116 @@ def run(
     guidance_scale: float = 5,
     controlnet_conditioning_scale: float = 1.0,
     seed: int = 0,
-    use_hed: bool = False,
+    use_openpose: bool = False,
+    use_depth: bool = False,
     use_canny: bool = False,
+    use_lineart: bool = False,
+    use_anime_lineart: bool = False,
+    use_mlsd: bool = False,
+    use_scribble: bool = False,
+    use_hed: bool = False,
+    use_softedge: bool = False,
+    use_teed: bool = False,
+    use_segment: bool = False,
+    use_normal: bool = False,
     progress=gr.Progress(track_tqdm=True),
 ) -> PIL.Image.Image:
-    # Get the composite image from the EditorValue dict
     composite_image = image['composite']
     width, height = composite_image.size
     
-    # Calculate new dimensions to fit within 1024x1024 while maintaining aspect ratio
     max_size = 1024
     ratio = min(max_size / width, max_size / height)
     new_width = int(width * ratio)
     new_height = int(height * ratio)
     
-    # Resize the image
     resized_image = composite_image.resize((new_width, new_height), Image.LANCZOS)
     
+    control_images = []
+    control_type = [0, 0, 0, 0, 0, 0]  # Initialize control type
+
+    if use_openpose:
+        openpose_image = openpose_detector(resized_image)
+        control_images.append(openpose_image)
+        control_type[0] = 1
+    
+    if use_depth:
+        depth_image = midas_detector(resized_image)
+        control_images.append(depth_image)
+        control_type[1] = 1
+    
     if use_canny:
-        controlnet_img = np.array(resized_image)
-        controlnet_img = cv2.Canny(controlnet_img, 100, 200)
-        controlnet_img = HWC3(controlnet_img)
-        image = Image.fromarray(controlnet_img)
-    elif not use_hed:
-        controlnet_img = resized_image
-        image = resized_image
-    else:
-        controlnet_img = processor(resized_image, scribble=False)
-        controlnet_img = np.array(controlnet_img)
-        controlnet_img = nms(controlnet_img, 127, 3)
-        controlnet_img = cv2.GaussianBlur(controlnet_img, (0, 0), 3)
-        random_val = int(round(random.uniform(0.01, 0.10), 2) * 255)
-        controlnet_img[controlnet_img > random_val] = 255
-        controlnet_img[controlnet_img < 255] = 0
-        image = Image.fromarray(controlnet_img)
+        canny_image = canny_detector(resized_image)
+        control_images.append(canny_image)
+        control_type[3] = 1
+    
+    if use_lineart:
+        lineart_image = lineart_detector(resized_image)
+        control_images.append(lineart_image)
+        control_type[3] = 1
+    
+    if use_anime_lineart:
+        anime_lineart_image = anime_lineart_detector(resized_image)
+        control_images.append(anime_lineart_image)
+        control_type[3] = 1
+    
+    if use_mlsd:
+        mlsd_image = mlsd_detector(resized_image)
+        control_images.append(mlsd_image)
+        control_type[3] = 1
+    
+    if use_scribble:
+        # Assuming scribble is the same as the input image
+        control_images.append(resized_image)
+        control_type[2] = 1
+    
+    if use_hed:
+        hed_image = hed_detector(resized_image)
+        control_images.append(hed_image)
+        control_type[2] = 1
     
+    if use_softedge:
+        softedge_image = pidi_detector(resized_image)
+        control_images.append(softedge_image)
+        control_type[2] = 1
+    
+    if use_teed:
+        # Assuming TEED is similar to HED but with thinner lines
+        teed_image = hed_detector(resized_image, detect_resolution=512)
+        control_images.append(teed_image)
+        control_type[2] = 1
+    
+    if use_segment:
+        segment_image = sam_detector(resized_image)
+        control_images.append(segment_image)
+        control_type[5] = 1
+    
+    if use_normal:
+        normal_image = normal_detector(resized_image)
+        control_images.append(normal_image)
+        control_type[4] = 1
+
+    if not control_images:
+        control_images.append(resized_image)
+        control_type[2] = 1  # Default to thick line type if no specific control is used
+
     prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
 
     generator = torch.Generator(device=device).manual_seed(seed)
     
-    if use_canny:
-        out = pipe_canny(
-            prompt=prompt,
-            negative_prompt=negative_prompt,
-            image=image,
-            num_inference_steps=num_steps,
-            generator=generator,
-            controlnet_conditioning_scale=controlnet_conditioning_scale,
-            guidance_scale=guidance_scale,
-            width=new_width,
-            height=new_height,
-        ).images[0]
-    else:
-        out = pipe(
-            prompt=prompt,
-            negative_prompt=negative_prompt,
-            image=image,
-            num_inference_steps=num_steps,
-            generator=generator,
-            controlnet_conditioning_scale=controlnet_conditioning_scale,
-            guidance_scale=guidance_scale,
-            width=new_width,
-            height=new_height,
-        ).images[0]
-
-    return (controlnet_img, out)
+    out = pipe(
+        prompt=prompt,
+        negative_prompt=negative_prompt,
+        image=control_images,
+        num_inference_steps=num_steps,
+        generator=generator,
+        controlnet_conditioning_scale=[controlnet_conditioning_scale] * len(control_images),
+        control_type=control_type,
+        guidance_scale=guidance_scale,
+        width=new_width,
+        height=new_height,
+    ).images[0]
+
+    return (control_images[0], out)
 
 with gr.Blocks(css="style.css", js=js_func) as demo:
     gr.Markdown(DESCRIPTION, elem_id="description")
@@ -280,8 +260,19 @@ with gr.Blocks(css="style.css", js=js_func) as demo:
                 image = gr.ImageEditor(type="pil", label="Sketch your image or upload one", width=512, height=512)
                 prompt = gr.Textbox(label="Prompt")
                 style = gr.Dropdown(label="Style", choices=STYLE_NAMES, value=DEFAULT_STYLE_NAME)
-                use_hed = gr.Checkbox(label="use HED detector", value=False, info="check this box if you upload an image and want to turn it to a sketch")
-                use_canny = gr.Checkbox(label="use Canny", value=False, info="check this to use ControlNet canny instead of scribble")
+                with gr.Accordion("Control options", open=False):
+                    use_openpose = gr.Checkbox(label="Use Openpose", value=False)
+                    use_depth = gr.Checkbox(label="Use Depth", value=False)
+                    use_canny = gr.Checkbox(label="Use Canny", value=False)
+                    use_lineart = gr.Checkbox(label="Use Lineart", value=False)
+                    use_anime_lineart = gr.Checkbox(label="Use Anime Lineart", value=False)
+                    use_mlsd = gr.Checkbox(label="Use MLSD", value=False)
+                    use_scribble = gr.Checkbox(label="Use Scribble", value=False)
+                    use_hed = gr.Checkbox(label="Use HED", value=False)
+                    use_softedge = gr.Checkbox(label="Use Softedge (PIDI)", value=False)
+                    use_teed = gr.Checkbox(label="Use TEED", value=False)
+                    use_segment = gr.Checkbox(label="Use Segment", value=False)
+                    use_normal = gr.Checkbox(label="Use Normal", value=False)
                 run_button = gr.Button("Run")
             with gr.Accordion("Advanced options", open=False):
                 negative_prompt = gr.Textbox(
@@ -303,7 +294,7 @@ with gr.Blocks(css="style.css", js=js_func) as demo:
                     value=5,
                 )
                 controlnet_conditioning_scale = gr.Slider(
-                    label="controlnet conditioning scale",
+                    label="ControlNet conditioning scale",
                     minimum=0.5,
                     maximum=5.0,
                     step=0.1,
@@ -322,6 +313,14 @@ with gr.Blocks(css="style.css", js=js_func) as demo:
             with gr.Group():
                 image_slider = ImageSlider(position=0.5)
 
+    gr.Markdown("""
+    ## Multi Control Visual Examples
+    - Openpose + Canny
+    - Openpose + Depth
+    - Openpose + Scribble
+    - Openpose + Normal
+    - Openpose + Segment
+    """)
 
     inputs = [
         image,
@@ -332,8 +331,18 @@ with gr.Blocks(css="style.css", js=js_func) as demo:
         guidance_scale,
         controlnet_conditioning_scale,
         seed,
+        use_openpose,
+        use_depth,
+        use_canny,
+        use_lineart,
+        use_anime_lineart,
+        use_mlsd,
+        use_scribble,
         use_hed,
-        use_canny
+        use_softedge,
+        use_teed,
+        use_segment,
+        use_normal
     ]
     outputs = [image_slider]
     run_button.click(
@@ -345,7 +354,5 @@ with gr.Blocks(css="style.css", js=js_func) as demo:
     ).then(lambda x: None, inputs=None, outputs=image_slider).then(
         fn=run, inputs=inputs, outputs=outputs
     )
-    
-    
 
-demo.queue().launch(show_error=True, ssl_verify=False)
+demo.queue().launch