updated pipeline example and enabled cache

app.py

@@ -351,7 +351,7 @@ with gr.Blocks(css=css, theme=Platinum(), title="MoD ControlNet Tile Upscaler")
                     ],                    
                     fn=predict,
                     outputs=result,
-                    cache_examples=False,
+                    cache_examples=True,
                 )
 
     max_tile_size.select(fn=set_maximum_resolution, inputs=[max_tile_size, resolution], outputs=resolution)

pipeline/mod_controlnet_tile_sr_sdxl.py

@@ -75,94 +75,86 @@ logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
 EXAMPLE_DOC_STRING = """
     Examples:
         ```py
-        # !pip install controlnet_aux
-        from diffusers import (
-            StableDiffusionXLControlNetUnionImg2ImgPipeline,
-            ControlNetUnionModel,
-            AutoencoderKL,
-        )
-        from diffusers.utils import load_image
         import torch
+        from diffusers import ControlNetUnionModel, AutoencoderKL, UniPCMultistepScheduler
+        from pipeline.mod_controlnet_tile_sr_sdxl import StableDiffusionXLControlNetTileSRPipeline, TileWeightingMethod, calculate_overlap
+        from diffusers.utils import load_image
         from PIL import Image
-        import numpy as np
 
-        prompt = "A cat"
-        # download an image
-        image = load_image(
-            "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky/cat.png"
-        )
-        # initialize the models and pipeline
+        device = "cuda"
+
+        # Initialize the models and pipeline
         controlnet = ControlNetUnionModel.from_pretrained(
             "brad-twinkl/controlnet-union-sdxl-1.0-promax", torch_dtype=torch.float16
-        )
-        vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
-        pipe = StableDiffusionXLControlNetUnionImg2ImgPipeline.from_pretrained(
-            "stabilityai/stable-diffusion-xl-base-1.0",
-            controlnet=controlnet,
-            vae=vae,
-            torch_dtype=torch.float16,
-            variant="fp16",
-        ).to("cuda")
-        # `enable_model_cpu_offload` is not recommended due to multiple generations
-        height = image.height
-        width = image.width
-        ratio = np.sqrt(1024.0 * 1024.0 / (width * height))
-        # 3 * 3 upscale correspond to 16 * 3 multiply, 2 * 2 correspond to 16 * 2 multiply and so on.
-        scale_image_factor = 3
-        base_factor = 16
-        factor = scale_image_factor * base_factor
-        W, H = int(width * ratio) // factor * factor, int(height * ratio) // factor * factor
-        image = image.resize((W, H))
-        target_width = W // scale_image_factor
-        target_height = H // scale_image_factor
-        images = []
-        crops_coords_list = [
-            (0, 0),
-            (0, width // 2),
-            (height // 2, 0),
-            (width // 2, height // 2),
-            0,
-            0,
-            0,
-            0,
-            0,
-        ]
-        for i in range(scale_image_factor):
-            for j in range(scale_image_factor):
-                left = j * target_width
-                top = i * target_height
-                right = left + target_width
-                bottom = top + target_height
-                cropped_image = image.crop((left, top, right, bottom))
-                cropped_image = cropped_image.resize((W, H))
-                images.append(cropped_image)
-        # set ControlNetUnion input
-        result_images = []
-        for sub_img, crops_coords in zip(images, crops_coords_list):
-            new_width, new_height = W, H
-            out = pipe(
-                prompt=[prompt] * 1,
-                image=sub_img,
-                control_image=[sub_img],
-                control_mode=[6],
-                width=new_width,
-                height=new_height,
-                num_inference_steps=30,
-                crops_coords_top_left=(W, H),
-                target_size=(W, H),
-                original_size=(W * 2, H * 2),
-            )
-            result_images.append(out.images[0])
-        new_im = Image.new("RGB", (new_width * scale_image_factor, new_height * scale_image_factor))
-        new_im.paste(result_images[0], (0, 0))
-        new_im.paste(result_images[1], (new_width, 0))
-        new_im.paste(result_images[2], (new_width * 2, 0))
-        new_im.paste(result_images[3], (0, new_height))
-        new_im.paste(result_images[4], (new_width, new_height))
-        new_im.paste(result_images[5], (new_width * 2, new_height))
-        new_im.paste(result_images[6], (0, new_height * 2))
-        new_im.paste(result_images[7], (new_width, new_height * 2))
-        new_im.paste(result_images[8], (new_width * 2, new_height * 2))
+        ).to(device=device)
+        vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16).to(device=device)
+
+        model_id = "SG161222/RealVisXL_V5.0"
+        pipe = StableDiffusionXLControlNetTileSRPipeline.from_pretrained(
+            model_id, controlnet=controlnet, vae=vae, torch_dtype=torch.float16, use_safetensors=True, variant="fp16"
+        ).to(device)
+
+        pipe.enable_model_cpu_offload()  # << Enable this if you have limited VRAM
+        pipe.enable_vae_tiling() # << Enable this if you have limited VRAM
+        pipe.enable_vae_slicing() # << Enable this if you have limited VRAM
+
+        # Set selected scheduler
+        pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+
+        # Load image
+        control_image = load_image("https://huggingface.co/datasets/DEVAIEXP/assets/resolve/main/1.jpg")
+        original_height = control_image.height
+        original_width = control_image.width
+        print(f"Current resolution: H:{original_height} x W:{original_width}")
+
+        # Pre-upscale image for tiling
+        resolution = 4096
+        tile_gaussian_sigma = 0.3
+        max_tile_size = 1024 # or 1280
+
+        current_size = max(control_image.size)
+        scale_factor = max(2, resolution / current_size)
+        new_size = (int(control_image.width * scale_factor), int(control_image.height * scale_factor))
+        image = control_image.resize(new_size, Image.LANCZOS)
+
+        # Update target height and width
+        target_height = image.height
+        target_width = image.width
+        print(f"Target resolution: H:{target_height} x W:{target_width}")
+
+        # Calculate overlap size
+        normal_tile_overlap, border_tile_overlap = calculate_overlap(target_width, target_height)
+
+        # Set other params
+        tile_weighting_method = TileWeightingMethod.COSINE.value
+        guidance_scale = 4
+        num_inference_steps = 35
+        denoising_strenght = 0.65
+        controlnet_strength = 1.0
+        prompt = "high-quality, noise-free edges, high quality, 4k, hd, 8k"
+        negative_prompt = "blurry, pixelated, noisy, low resolution, artifacts, poor details"
+
+        # Image generation
+        control_image = pipe(
+            image=image,
+            control_image=control_image,
+            control_mode=[6],
+            controlnet_conditioning_scale=float(controlnet_strength),
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            normal_tile_overlap=normal_tile_overlap,
+            border_tile_overlap=border_tile_overlap,
+            height=target_height,
+            width=target_width,
+            original_size=(original_width, original_height),
+            target_size=(target_width, target_height),
+            guidance_scale=guidance_scale,        
+            strength=float(denoising_strenght),
+            tile_weighting_method=tile_weighting_method,
+            max_tile_size=max_tile_size,
+            tile_gaussian_sigma=float(tile_gaussian_sigma),
+            num_inference_steps=num_inference_steps,
+        )["images"][0]
         ```
 """