fix pipe

app.py

@@ -11,6 +11,7 @@ import os
 import numpy as np
 from PIL import Image
 import zipfile
+import spaces
 
 path = os.getcwd()
 output_dir = f"{path}/output"
@@ -19,6 +20,36 @@ cn_lineart_dir = f"{path}/controlnet/lineart"
 
 load_cn_model(cn_lineart_dir)
 load_cn_config(cn_lineart_dir)
+pipe = get_cn_pipeline()
+
+@spaces.GPU(duration=120)
+def generate(detectors, prompt, negative_prompt, reference_flg=False, reference_img=None):
+    default_pos = ""
+    default_neg = ""
+    prompt = default_pos + prompt 
+    negative_prompt = default_neg + negative_prompt 
+    
+
+    if reference_flg==False:
+        image = pipe(
+                    prompt=prompt,
+                    negative_prompt = negative_prompt,
+                    image=detectors,
+                    num_inference_steps=50,
+                    controlnet_conditioning_scale=[1.0, 0.2],
+                    ip_adapter_image=None,
+                ).images[0]
+    else:
+
+        image = pipe(
+                    prompt=prompt,
+                    negative_prompt = negative_prompt,
+                    image=detectors,
+                    num_inference_steps=50,
+                    controlnet_conditioning_scale=[1.0, 0.2],
+                    ip_adapter_image=reference_img,
+                ).images[0]
+    return image
 
 
 def zip_png_files(folder_path):
@@ -60,7 +91,6 @@ def resize_image(img, max_size=1024):
         return img
 
 
-
 class webui:
     def __init__(self):
         self.demo = gr.Blocks()
@@ -75,9 +105,7 @@ class webui:
         image[index] = [255, 255, 255, 255]
         input_image = cv2pil(image)
 
-        pipe = get_cn_pipeline(reference_flg)
         detectors = get_cn_detector(input_image.resize((1024, 1024), Image.ANTIALIAS))
-        
 
         gen_image = generate(pipe, detectors, pos_prompt, neg_prompt, reference_flg, reference_img)
         color_img, unfinished = process(gen_image.resize((image.shape[1], image.shape[0]), Image.ANTIALIAS) , org_line_image, alpha_th, thickness)

sd_model.py

@@ -2,11 +2,11 @@ from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCM
 from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL
 import torch
 import pickle as pkl
-import spaces
+
 
 device = "cuda"
 
-def get_cn_pipeline(reference_flg):
+def get_cn_pipeline():
     controlnets = [
         ControlNetModel.from_pretrained("./controlnet/lineart", torch_dtype=torch.float16, use_safetensors=True),
         ControlNetModel.from_pretrained("mattyamonaca/controlnet_line2line_xl", torch_dtype=torch.float16)
@@ -16,6 +16,11 @@ def get_cn_pipeline(reference_flg):
     pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
         "cagliostrolab/animagine-xl-3.1", controlnet=controlnets, vae=vae, torch_dtype=torch.float16
     )
+    pipe.load_ip_adapter(
+        "ozzygt/sdxl-ip-adapter",
+        "", 
+        weight_name="ip-adapter_sdxl_vit-h.safetensors"
+    )
 
     return pipe
 
@@ -30,47 +35,7 @@ def invert_image(img):
 
 
 def get_cn_detector(image):
-    #lineart_anime = LineartAnimeDetector.from_pretrained("lllyasviel/Annotators")
-    #canny = CannyDetector()
-    #lineart_anime_img = lineart_anime(image)
-    #canny_img = canny(image)
-    #canny_img = canny_img.resize((lineart_anime(image).width, lineart_anime(image).height))
-    re_image = invert_image(image)
-    
-    
+    re_image = invert_image(image)    
     detectors = [re_image, image]
-    print(detectors)
     return detectors
 
-@spaces.GPU(duration=120)
-def generate(pipe, detectors, prompt, negative_prompt, reference_flg=False, reference_img=None):
-    pipe.to("cuda")
-    default_pos = ""
-    default_neg = ""
-    prompt = default_pos + prompt 
-    negative_prompt = default_neg + negative_prompt 
-    
-
-    if reference_flg==False:
-        image = pipe(
-                    prompt=prompt,
-                    negative_prompt = negative_prompt,
-                    image=detectors,
-                    num_inference_steps=50,
-                    controlnet_conditioning_scale=[1.0, 0.2],
-                ).images[0]
-    else:
-        pipe.load_ip_adapter(
-            "ozzygt/sdxl-ip-adapter",
-            "", 
-            weight_name="ip-adapter_sdxl_vit-h.safetensors")
-        image = pipe(
-                    prompt=prompt,
-                    negative_prompt = negative_prompt,
-                    image=detectors,
-                    num_inference_steps=50,
-                    controlnet_conditioning_scale=[1.0, 0.2],
-                    ip_adapter_image=reference_img,
-                ).images[0]
-
-    return image