got image to vector

charles_actor.py

@@ -2,6 +2,7 @@ import ray
 import time
 import asyncio
 import os
+from clip_transform import CLIPTransform
 
 @ray.remote
 class CharlesActor:
@@ -9,6 +10,7 @@ class CharlesActor:
         self._needs_init = True
         self._system_one_audio_history_output = ""
         self._state = "Initializing"
+        self._clip_transform = CLIPTransform()
         
     def get_state(self):
         return self._state
@@ -54,8 +56,11 @@ class CharlesActor:
         
         self._state = "Waiting for input"
         total_video_frames = 0
+        skipped_video_frames = 0
         total_audio_frames = 0
         loops = 0
+        start_time = time.time()
+        vector_debug = "--n/a--"
         
         process_speech_to_text_future = []
 
@@ -88,19 +93,18 @@ class CharlesActor:
                         self._system_one_audio_history_output = table_content
                         await self._respond_to_prompt_actor.enqueue_prompt.remote(prompt)
                         
-            # video_frames = await self._streamlit_av_queue.get_video_frames_async()
-            # if len(video_frames) > 0:
-            #     total_video_frames += len(video_frames)
-            # #     for video_frame in video_frames:
-            # #         system_one_video_output.image(video_frame.to_ndarray())
-            # #         pass
+            video_frames = await self._streamlit_av_queue.get_video_frames_async()
+            if len(video_frames) > 0:
+                vector_debug = f"found {len(video_frames)} video frames"
+                total_video_frames += 1
+                skipped_video_frames += (len(video_frames) -1)
+                image_as_array = video_frames[-1]
+                last_frame_vector = self._clip_transform.image_to_embeddings(image_as_array)
+                vector_debug = f"Last frame vector: {last_frame_vector.shape}"
             
-            # update debug output
-            if (total_video_frames >0 or total_audio_frames > 0):
-                self._state = f"Processed {total_video_frames} video frames and {total_audio_frames} audio frames"
             await asyncio.sleep(0.01)
             loops+=1
-            self._state = f"Processed {total_video_frames} video frames and {total_audio_frames} audio frames, loops: {loops}"
+            self._state = f"Processed {total_video_frames} video frames and {total_audio_frames} audio frames, loops: {loops}. loops per second: {loops/(time.time()-start_time):.2f}. {vector_debug}"
 
 if __name__ == "__main__":
     if not ray.is_initialized():

streamlit_av_queue.py

@@ -27,13 +27,14 @@ class StreamlitAVQueue:
             
     async def queued_video_frames_callback(
                 self,
-                frames: List[av.AudioFrame],
-            ) -> av.AudioFrame:
+                frames: List[av.VideoFrame],
+            ) -> av.VideoFrame:
         try:
             for frame in frames:
-                shared_tensor = torch.from_numpy(frame.to_ndarray())
+                shared_tensor = frame.to_ndarray(format="rgb24")
                 shared_tensor_ref = ray.put(shared_tensor)
                 await self.queue_actor.enqueue_in_video_frame.remote(shared_tensor_ref)
+                # print (f"tesnor len: {len(shared_tensor)}, tensor shape: {shared_tensor.shape}, tensor type:{shared_tensor.dtype} tensor ref: {shared_tensor_ref}")
         except Exception as e:
             print (e)
         return frames