1fps streaming hooray

app/bot.py

@@ -86,7 +86,7 @@ class DailyVision(EventHandler):
             self.__camera = Daily.create_camera_device("camera",
                                                        width = video_frame.width,
                                                        height = video_frame.height,
-                                                       color_format="RGBA")
+                                                       color_format="RGB")
             self.__client.update_inputs({
                 "camera": {
                     "isEnabled": True,
@@ -97,6 +97,8 @@ class DailyVision(EventHandler):
             })
 
     def process_frames(self):
+        params = Pipeline.InputParams()
+
         while not self.__app_quit:
             # Is anyone watching?
             if not self.__idle and len(self.__client.participants()) < 2:
@@ -113,16 +115,16 @@ class DailyVision(EventHandler):
 
                 if video_frame:
                     image = Image.frombytes("RGBA", (video_frame.width, video_frame.height), video_frame.buffer)
-                    #result = self.__pipeline(image)
+                    result_image = self.__pipeline.predict(params, image).convert("RGB")
+                    self.__camera.write_frame(result_image.tobytes())
                     #pil = Image.fromarray(result.render()[0], mode="RGB").tobytes()
-
-                    self.__camera.write_frame(image.tobytes())
+                    #self.__camera.write_frame(result_image.tobytes())
             except queue.Empty:
                 pass
 
     def on_video_frame(self, participant_id, video_frame):
         # Process ~15 frames per second (considering incoming frames at 30fps).
-        if time.time() - self.__time > 0.05:
+        if time.time() - self.__time > 1: #0.05:
             self.__time = time.time()
             self.setup_camera(video_frame)
             self.__queue.put(video_frame)    

app/pipeline.py

@@ -51,10 +51,10 @@ class Pipeline:
             1, min=1, max=15, title="Steps", field="range", hide=True, id="steps"
         )
         width: int = Field(
-            512, min=2, max=15, title="Width", disabled=True, hide=True, id="width"
+            640, min=2, max=15, title="Width", disabled=True, hide=True, id="width"
         )
         height: int = Field(
-            512, min=2, max=15, title="Height", disabled=True, hide=True, id="height"
+            480, min=2, max=15, title="Height", disabled=True, hide=True, id="height"
         )
         guidance_scale: float = Field(
             1.0,
@@ -181,8 +181,8 @@ class Pipeline:
             
             self.pipe(
                 prompt="warmup",
-                image=[Image.new("RGB", (768, 768))],
-                control_image=[Image.new("RGB", (768, 768))],
+                image=[Image.new("RGB", (640, 480))],
+                control_image=[Image.new("RGB", (640, 480))],
         )
 
     def predict(self, params: "Pipeline.InputParams", image) -> Image.Image:
@@ -222,7 +222,7 @@ class Pipeline:
             return None
         result_image = results.images[0]
 
-        if os.getenv("CONTROL_NET_OVERLAY"):
+        if os.getenv("CONTROL_NET_OVERLAY", True):
             # paste control_image on top of result_image
             w0, h0 = (200, 200)
             control_image = control_image.resize((w0, h0))