anyline preprocessor

app.py

@@ -8,7 +8,7 @@ from diffusers import (
     StableDiffusionXLControlNetImg2ImgPipeline,
     DDIMScheduler,
 )
-from canny_gpu import SobelOperator
+from controlnet_aux import AnylineDetector
 from compel import Compel, ReturnedEmbeddingsType
 from PIL import Image
 import os
@@ -63,7 +63,10 @@ if not IS_SPACES_ZERO:
     # pipe.enable_xformers_memory_efficient_attention()
     pipe.enable_model_cpu_offload()
 pipe.enable_vae_tiling()
-canny_torch = SobelOperator()
+
+anyline = AnylineDetector.from_pretrained(
+    "TheMistoAI/MistoLine", filename="MTEED.pth", subfolder="Anyline"
+).to(device)
 
 
 def pad_image(image):
@@ -96,23 +99,20 @@ def predict(
     strength=1.0,
     controlnet_start=0.0,
     controlnet_end=1.0,
-    canny_low=0.01,
-    canny_high=0.2,
     progress=gr.Progress(track_tqdm=True),
 ):
     if IS_SPACES_ZERO:
         apply_hidiffusion(pipe)
-        canny_torch.to(device)
     if input_image is None:
         raise gr.Error("Please upload an image.")
     padded_image = pad_image(input_image).resize((1024, 1024)).convert("RGB")
     conditioning, pooled = compel([prompt, negative_prompt])
     generator = torch.manual_seed(seed)
     last_time = time.time()
-    canny_image = canny_torch(padded_image, canny_low, canny_high)
+    anyline_image = anyline(padded_image, detect_resolution=1024)
     images = pipe(
         image=padded_image,
-        control_image=canny_image,
+        control_image=anyline_image,
         strength=strength,
         prompt_embeds=conditioning[0:1],
         pooled_prompt_embeds=pooled[0:1],
@@ -129,7 +129,7 @@ def predict(
         eta=1.0,
     )
     print(f"Time taken: {time.time() - last_time}")
-    return (padded_image, images.images[0]), padded_image, canny_image
+    return (padded_image, images.images[0]), padded_image, anyline_image
 
 
 css = """
@@ -221,20 +221,6 @@ SDXL Controlnet [TheMistoAI/MistoLine](https://huggingface.co/TheMistoAI/MistoLi
                     value=1.0,
                     label="ControlNet End",
                 )
-                canny_low = gr.Slider(
-                    minimum=0,
-                    maximum=1,
-                    step=0.001,
-                    value=0.0,
-                    label="Canny Low Threshold",
-                )
-                canny_high = gr.Slider(
-                    minimum=0.0,
-                    maximum=1.0,
-                    step=0.001,
-                    value=0.2,
-                    label="Canny High Threshold",
-                )
 
             btn = gr.Button()
         with gr.Column(scale=2):
@@ -242,7 +228,7 @@ SDXL Controlnet [TheMistoAI/MistoLine](https://huggingface.co/TheMistoAI/MistoLi
                 image_slider = ImageSlider(position=0.5)
             with gr.Row():
                 padded_image = gr.Image(type="pil", label="Padded Image")
-                canny_image = gr.Image(type="pil", label="Canny Image")
+                anyline_image = gr.Image(type="pil", label="Anyline Image")
     inputs = [
         image_input,
         prompt,
@@ -254,10 +240,8 @@ SDXL Controlnet [TheMistoAI/MistoLine](https://huggingface.co/TheMistoAI/MistoLi
         strength,
         controlnet_start,
         controlnet_end,
-        canny_low,
-        canny_high,
     ]
-    outputs = [image_slider, padded_image, canny_image]
+    outputs = [image_slider, padded_image, anyline_image]
     btn.click(lambda x: None, inputs=None, outputs=image_slider).then(
         fn=predict, inputs=inputs, outputs=outputs
     )
@@ -277,8 +261,6 @@ SDXL Controlnet [TheMistoAI/MistoLine](https://huggingface.co/TheMistoAI/MistoLi
                 1.0,
                 0.0,
                 1.0,
-                0.01,
-                0.2,
             ],
             [
                 "./examples/cybetruck.jpeg",
@@ -291,8 +273,6 @@ SDXL Controlnet [TheMistoAI/MistoLine](https://huggingface.co/TheMistoAI/MistoLi
                 0.8,
                 0.0,
                 1.0,
-                0.01,
-                0.2,
             ],
             [
                 "./examples/jesus.png",
@@ -305,8 +285,6 @@ SDXL Controlnet [TheMistoAI/MistoLine](https://huggingface.co/TheMistoAI/MistoLi
                 0.8,
                 0.0,
                 1.0,
-                0.01,
-                0.2,
             ],
             [
                 "./examples/anna-sullivan-DioLM8ViiO8-unsplash.jpg",
@@ -319,8 +297,6 @@ SDXL Controlnet [TheMistoAI/MistoLine](https://huggingface.co/TheMistoAI/MistoLi
                 0.8,
                 0.0,
                 1.0,
-                0.01,
-                0.2,
             ],
             [
                 "./examples/img_aef651cb-2919-499d-aa49-6d4e2e21a56e_1024.jpg",
@@ -333,8 +309,6 @@ SDXL Controlnet [TheMistoAI/MistoLine](https://huggingface.co/TheMistoAI/MistoLi
                 0.8,
                 0.0,
                 1.0,
-                0.01,
-                0.2,
             ],
             [
                 "./examples/huggingface.jpg",
@@ -347,8 +321,6 @@ SDXL Controlnet [TheMistoAI/MistoLine](https://huggingface.co/TheMistoAI/MistoLi
                 0.8,
                 0.0,
                 1.0,
-                0.0,
-                0.95,
             ],
         ],
         cache_examples="lazy",

requirements.txt

@@ -17,4 +17,5 @@ accelerate
 safetensors
 hidiffusion==0.1.8
 spaces
-torch
+torch
+controlnet-aux @ git+https://github.com/huggingface/controlnet_aux