get access to the image frames

app.py

@@ -60,14 +60,24 @@ def do_work(data: bytearray) -> tuple[str, bool]:
     return text, speaker_finished
 
 
-frames_deque_lock = threading.Lock()
-frames_deque: deque = deque([])
+audio_frames_deque_lock = threading.Lock()
+audio_frames_deque: deque = deque([])
+
+video_frames_deque_lock = threading.Lock()
+video_frames_deque: deque = deque([])
+
+async def queued_video_frames_callback(
+            frames: List[av.AudioFrame],
+        ) -> av.AudioFrame:
+    with video_frames_deque_lock:
+        video_frames_deque.extend(frames)
+    return frames
 
 async def queued_audio_frames_callback(
             frames: List[av.AudioFrame],
         ) -> av.AudioFrame:
-    with frames_deque_lock:
-        frames_deque.extend(frames)
+    with audio_frames_deque_lock:
+        audio_frames_deque.extend(frames)
 
     # create frames to be returned.
     new_frames = []
@@ -88,8 +98,8 @@ webrtc_ctx = webrtc_streamer(
     key="charles",
     desired_playing_state=playing,
     # audio_receiver_size=4096,
-    # audio_frame_callback=process_audio,
     queued_audio_frames_callback=queued_audio_frames_callback,
+    queued_video_frames_callback=queued_video_frames_callback,
     mode=WebRtcMode.SENDRECV,
     rtc_configuration={"iceServers": get_ice_servers()},
     async_processing=True,
@@ -109,10 +119,18 @@ system_one_audio_history_output = st.empty()
 sound_chunk = pydub.AudioSegment.empty()
 while True:
     if webrtc_ctx.state.playing:
+        # handle video
+        video_frames = []
+        with video_frames_deque_lock:
+            while len(video_frames_deque) > 0:
+                frame = video_frames_deque.popleft()
+                video_frames.append(frame)
+
+        # handle audio
         audio_frames = []
-        with frames_deque_lock:
-            while len(frames_deque) > 0:
-                frame = frames_deque.popleft()
+        with audio_frames_deque_lock:
+            while len(audio_frames_deque) > 0:
+                frame = audio_frames_deque.popleft()
                 audio_frames.append(frame)
 
         if len(audio_frames) == 0: