Update app.py

app.py

@@ -17,6 +17,9 @@ using our implementation of the RAFT model. We will also see how to convert the
 predicted flows to RGB images for visualization.
 """
 
+from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
+from diffusers import UniPCMultistepScheduler
+
 import cv2
 import numpy as np
 import os
@@ -39,6 +42,53 @@ from scipy.interpolate import LinearNDInterpolator
 from imageio import imread, imwrite
 
 
+# Constants
+low_threshold = 100
+high_threshold = 200
+
+# Models
+controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
+    "runwayml/stable-diffusion-v1-5", controlnet=controlnet, safety_checker=None, torch_dtype=torch.float16
+)
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+
+# This command loads the individual model components on GPU on-demand. So, we don't
+# need to explicitly call pipe.to("cuda").
+pipe.enable_model_cpu_offload()
+
+pipe.enable_xformers_memory_efficient_attention()
+
+# Generator seed,
+generator = torch.manual_seed(0)
+
+def get_canny_filter(image):
+    if not isinstance(image, np.ndarray):
+        image = np.array(image) 
+        
+    image = cv2.Canny(image, low_threshold, high_threshold)
+    image = image[:, :, None]
+    image = np.concatenate([image, image, image], axis=2)
+    canny_image = Image.fromarray(image)
+    return canny_image
+
+
+def generate_images(image, prompt):
+    canny_image = get_canny_filter(image)
+    output = pipe(
+        prompt,
+        canny_image,
+        generator=generator,
+        num_images_per_prompt=1,
+        num_inference_steps=20,
+    )
+    all_outputs = []
+    all_outputs.append(canny_image)
+    for image in output.images:
+        all_outputs.append(image)
+    return all_outputs
+
+
 def write_flo(flow, filename):
     """
     Write optical flow in Middlebury .flo format
@@ -74,6 +124,9 @@ def infer():
     #frames, _, _ = read_video(str("./spacex.mp4"), output_format="TCHW")
     #print(f"FRAME BEFORE stack: {frames[100]}")
 
+    pil2diff_img = Image.open("./frame1.jpg")
+    diffused_img = generate_images(pil2diff_img)
+    print(f"DIFFUSED IMG: {diffused_img[1]}")
     
     input_frame_1 = read_image(str("./frame1.jpg"), ImageReadMode.UNCHANGED)
     print(f"FRAME 1: {input_frame_1}")