Camera working on JS

camera_utils.py

@@ -0,0 +1,169 @@
+import logging
+import flet as ft
+from js_execution import set_camera_frame_callback
+from flet_extensions import eval_js
+import asyncio
+from typing import List, Dict
+
+logger = logging.getLogger(__name__)
+
+class Frame:
+    def __init__(self, data: str, timestamp: float):
+        self.data = data
+        self.timestamp = timestamp
+
+class CameraImage(ft.Container):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.stream_active = False
+        self.video_image1 = ft.Image(
+            src="",
+            width=self.width,
+            height=self.height,
+            fit=ft.ImageFit.CONTAIN,
+            visible=False,
+        )
+        self.video_image2 = ft.Image(
+            src="",
+            width=self.width,
+            height=self.height,
+            fit=ft.ImageFit.CONTAIN,
+            visible=False,
+        )
+        self.current_video_image = self.video_image1
+        self.video_stack = ft.Stack([self.video_image1, self.video_image2])
+        self.content = self.video_stack
+        set_camera_frame_callback(self.handle_camera_frame)
+        self.captured_frames: Dict[int, Frame] = {}
+        self.stream_start_time = None
+
+    async def start_camera_stream_async(self) -> str:
+        try:
+            result = await eval_js(self.page, """
+            window.cameraStreamActive = false;
+            window.cameraWebSocket = null;
+
+            async function setupCameraStream() {
+                console.log("setupCameraStream called");
+                try {
+                    const stream = await navigator.mediaDevices.getUserMedia({ video: true });
+                    console.log("Camera access granted");
+                    const video = document.createElement('video');
+                    video.srcObject = stream;
+                    await video.play();
+
+                    const canvas = document.createElement('canvas');
+                    canvas.width = video.videoWidth / 2;
+                    canvas.height = video.videoHeight / 2;
+                    const ctx = canvas.getContext('2d');
+
+                    window.cameraWebSocket = new WebSocket(`ws://${window.location.host}/ws/camera`);
+                    
+                    function sendFrame() {
+                        if (window.cameraWebSocket && window.cameraWebSocket.readyState === WebSocket.OPEN) {
+                            ctx.drawImage(video, 0, 0, video.videoWidth, video.videoHeight, 0, 0, canvas.width, canvas.height);
+                            window.cameraWebSocket.send(canvas.toDataURL('image/jpeg', 0.2)); // Added quality parameter
+                            window.cameraStreamActive && setTimeout(sendFrame, 1000 / 60);
+                        }
+                    }
+
+                    window.cameraWebSocket.onopen = () => {
+                        console.log("WebSocket connection opened");
+                        window.cameraStreamActive = true;
+                        sendFrame();
+                    };
+
+                    window.cameraWebSocket.onclose = () => {
+                        console.log("WebSocket connection closed");
+                        window.cameraStreamActive = false;
+                        stream.getTracks().forEach(track => track.stop());
+                    };
+                    
+                    return "Camera stream setup complete";
+                } catch (error) {
+                    console.error("Error accessing camera:", error);
+                    throw error;
+                }
+            }
+
+            return await setupCameraStream();
+            """)
+            logger.info(f"start_camera_stream_async result: {result}")
+            self.stream_active = True
+            self.stream_start_time = asyncio.get_event_loop().time()
+            self.captured_frames.clear()
+            return result
+        except Exception as e:
+            logger.error(f"Error in start_camera_stream_async: {str(e)}")
+            return f"Error: {str(e)}"
+
+    async def stop_camera_stream_async(self) -> str:
+        try:
+            result = await eval_js(self.page, """
+            if (window.cameraWebSocket) {
+                window.cameraStreamActive = false;
+                window.cameraWebSocket.close();
+                window.cameraWebSocket = null;
+            }
+            return 'Camera stream stopped';
+            """)
+            logger.info(f"stop_camera_stream_async result: {result}")
+            self.stream_active = False
+            self.stream_start_time = None
+            return result
+        except Exception as e:
+            logger.error(f"Error in stop_camera_stream_async: {str(e)}")
+            return f"Error: {str(e)}"
+
+    async def handle_camera_frame(self, frame_data: str):
+        if self.stream_active:
+            await self.update_webcam_image(frame_data)
+            self.capture_timed_frames(frame_data)
+
+    def capture_timed_frames(self, frame_data: str):
+        current_time = asyncio.get_event_loop().time()
+        if self.stream_start_time is None:
+            self.stream_start_time = current_time
+
+        elapsed_time = current_time - self.stream_start_time
+
+        for seconds in [1, 2, 3]:
+            if elapsed_time >= seconds and seconds not in self.captured_frames:
+                self.captured_frames[seconds] = Frame(frame_data, current_time)
+
+    def get_captured_frames(self) -> List[Frame]:
+        return list(self.captured_frames.values())
+
+    async def update_webcam_image(self, frame_data: str) -> None:
+        if not frame_data:
+            print("Empty data!")
+            return
+        
+        # Frame data is already in Data URI format
+        new_src = frame_data 
+
+        self.video_image1.visible = True
+        self.video_image2.visible = True
+
+        # Determine which image to update
+        if self.current_video_image == self.video_image1:
+            image_to_update = self.video_image2
+            image_to_hide = self.video_image1
+        else:
+            image_to_update = self.video_image1
+            image_to_hide = self.video_image2
+
+        # Update the hidden image
+        image_to_update.src = new_src
+        image_to_update.update()
+        
+        # Wait for the browser to load the new image
+        await asyncio.sleep(0.02)  # Adjust this delay if needed
+        
+        # Update opacity
+        image_to_update.opacity = 1
+        image_to_hide.opacity = 0
+        self.current_video_image = image_to_update
+
+        # Final update to apply opacity changes
+        self.video_stack.update()

flet_app.py

@@ -5,9 +5,12 @@ from domain.translations import spanish_translations
 from domain.localapi_dialog_processor import LocalApiDialogueProcessor
 import logging
 import asyncio
-from typing import Optional, List, Tuple
+from typing import Optional, List, Callable, Tuple
 from audio_utils import get_base64_audio
 
+from pydantic import BaseModel, Field
+from camera_utils import CameraImage
+
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
@@ -163,106 +166,11 @@ class ChessModel:
             logger.info(f"Piece at {chess.SQUARE_NAMES[square]}: {piece} (Empathy: {piece.empathy:.2f})")
 
 
-from pydantic import BaseModel, Field
-import time
-from typing import List, Callable
-import cv2
-import numpy as np
-import base64
-
-
 class StreamConfig(BaseModel):
     max_width: int = Field(default=320, gt=0)
     fps_target: int = Field(default=40, gt=0)
     jpeg_quality: int = Field(default=40, ge=0, le=100)
-
-class Frame(BaseModel):
-    data: str
-    width: int
-    height: int
-
-import cv2
-import asyncio
-
-class CameraModel:
-    def __init__(self):
-        self.cap = None
-
-    def __enter__(self):
-        self.cap = cv2.VideoCapture(0)
-        if not self.cap.isOpened():
-            raise RuntimeError("Failed to open camera")
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self.cap:
-            self.cap.release()
-
-    def get_frame(self):
-        if not self.cap:
-            raise RuntimeError("Camera is not initialized")
-        ret, frame = self.cap.read()
-        if not ret:
-            raise RuntimeError("Failed to capture frame")
-        return frame
-
-class ImageProcessorModel:
-    @staticmethod
-    def process(frame, config: StreamConfig) -> Frame:
-        height, width = frame.shape[:2]
-        if width > config.max_width:
-            scale = config.max_width / width
-            new_height = int(height * scale)
-            frame = cv2.resize(frame, (config.max_width, new_height))
-        
-        _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, config.jpeg_quality])
-        # return Frame(
-        #     data="", #base64.b64encode(buffer).decode('utf-8'),
-        #     width=0, #frame.shape[1],
-        #     height=0, #frame.shape[0]
-        # )
-        return Frame(
-            data=base64.b64encode(buffer).decode('utf-8'),
-            width=frame.shape[1],
-            height=frame.shape[0]
-        )
-
-
-class VideoCaptureService:
-    def __init__(self, camera_model: CameraModel, image_processor: ImageProcessorModel):
-        self.camera_model = camera_model
-        self.image_processor = image_processor
-        self.stream_config = StreamConfig()
-        self.is_capturing = False
-        self.captured_frames: List[Frame] = []
-        self.frame_callback: Callable[[Frame], None] = None
-
-    async def start_capturing(self, frame_callback: Callable[[Frame], None]):
-        self.is_capturing = True
-        self.captured_frames.clear()
-        self.frame_callback = frame_callback
-        await self._capture_frames()
-
-    def stop_capturing(self):
-        self.is_capturing = False
-
-    async def _capture_frames(self):
-        with self.camera_model as camera:
-            while self.is_capturing:
-                frame = camera.get_frame()
-                processed_frame = self.image_processor.process(frame, self.stream_config)
-                
-                if self.frame_callback:
-                    self.frame_callback(processed_frame)
-                
-                if len(self.captured_frames) < 3:
-                    self.captured_frames.append(processed_frame)
-
-                await asyncio.sleep(1 / self.stream_config.fps_target)
-
-    def get_captured_frames(self) -> List[Frame]:
-        return self.captured_frames
-
+    
 class RecordingCountdown(ft.ProgressBar):
     def __init__(self, seconds):
         super().__init__()
@@ -304,32 +212,22 @@ class ChessPresenter:
         self.recording_timer = None
         self.countdown_timer = None
 
-        camera_model = CameraModel()
-        image_processor = ImageProcessorModel()
-        video_capture_service = VideoCaptureService(camera_model, image_processor)
-        self.video_capture_service = video_capture_service
-
     def toggle_video_recording(self):
-        if self.video_capture_service.is_capturing:
+        if self.view.is_camera_active():
             self.stop_video_recording()
             self.view.show_processing_feedback()
         else:
             self.start_video_recording()
 
     def start_video_recording(self):
-        self.view.video_container.visible = True
-        self.view.page.run_task(self.video_capture_service.start_capturing, self.on_frame_captured)
+        self.view.start_camera_stream()
+        self.view.show_video_container()
         self.view.update_video_button(is_recording=True)
 
     def stop_video_recording(self):
-        self.video_capture_service.stop_capturing()
+        self.view.stop_camera_stream()
         self.view.update_video_button(is_recording=False)
-        self.view.video_container.visible = False
-        self.view.video_container.update()
-
-    def on_frame_captured(self, frame: Frame):
-        self.view.page.run_task(self.view.update_webcam_image, frame)
-        
+        self.view.hide_video_container()
 
     def process_audio_move_argument(self, audio_base64: str):
         piece_symbol, chess_piece = self.model.get_piece_at(self.pending_move.from_square)
@@ -684,25 +582,9 @@ class ChessView(ft.Container):
                 self.audio_player.autoplay = False
                 self.audio_player.update()
 
-        self.video_image1 = ft.Image(
-            src="",
-            width=self.square_size * 4,
-            height=self.square_size * 4,
-            fit=ft.ImageFit.CONTAIN,
-            visible=True,
-        )
-        self.video_image2 = ft.Image(
-            src="",
-            width=self.square_size * 4,
-            height=self.square_size * 4,
-            fit=ft.ImageFit.CONTAIN,
-            visible=True,
-        )
-        self.current_video_image = self.video_image1
-
-        self.video_stack = ft.Stack([self.video_image1, self.video_image2])
+        self.camera_image = CameraImage(width=320, height=240)
         self.video_container = ft.Container(
-            content=self.video_stack,
+            content=self.camera_image,
             visible=False,
         )
 
@@ -965,7 +847,6 @@ class ChessView(ft.Container):
     def update_video_button(self, is_recording: bool):
         self.video_record_button.icon = ft.icons.STOP if is_recording else ft.icons.VIDEO_CALL
         self.video_record_button.tooltip = "Stop recording" if is_recording else "Record video move argument"
-        self.video_container.visible = is_recording
         self.update()
 
     def show_response(self, response, piece: str, transcription: str):
@@ -1313,37 +1194,22 @@ class ChessView(ft.Container):
                 bar.controls[i].bgcolor = color
             self.update()
             
-    async def update_webcam_image(self, frame: Frame) -> None:
-        if not frame.data:
-            print("Empty data!")
-            return
+    def is_camera_active(self):
+        return self.camera_image.stream_active
 
-        new_src = f"data:image/jpeg;base64,{frame.data}"
+    def start_camera_stream(self):
+        self.my_page.run_task(self.camera_image.start_camera_stream_async)
 
-        # Determine which image to update
-        if self.current_video_image == self.video_image1:
-            image_to_update = self.video_image2
-            image_to_hide = self.video_image1
-        else:
-            image_to_update = self.video_image1
-            image_to_hide = self.video_image2
-
-        # Update the hidden image
-        image_to_update.src = new_src
-        image_to_update.width = frame.width
-        image_to_update.height = frame.height
-        image_to_update.update()
-        
-        # Wait for the browser to load the new image
-        await asyncio.sleep(0.03)  # Adjust this delay if needed
-        
-        # Update opacity
-        image_to_update.opacity = 1
-        image_to_hide.opacity = 0
-        self.current_video_image = image_to_update
+    def stop_camera_stream(self):
+        self.my_page.run_task(self.camera_image.stop_camera_stream_async)
+
+    def show_video_container(self):
+        self.video_container.visible = True
+        self.update()
 
-        # Final update to apply opacity changes
-        self.video_stack.update()
+    def hide_video_container(self):
+        self.video_container.visible = False
+        self.update()
 
     def on_screen_click(self, e):
         if self.empathy_bar_visible:

flet_extensions.py

@@ -1,11 +1,13 @@
 import flet as ft
 from js_execution import js_execution_router
 
+
+async def eval_js(self: ft.Page, js_code: str) -> str:
+    async with js_execution_router.create_dynamic_endpoint(js_code) as context:
+        self.launch_url(context.endpoint, web_window_name=context.hidden_iframe_name)
+        return await context.result_future
+        
 def extend_flet_page():
-    async def eval_js(self: ft.Page, js_code: str) -> str:
-        async with js_execution_router.create_dynamic_endpoint(js_code) as context:
-            self.launch_url(context.endpoint, web_window_name=context.hidden_iframe_name)
-            return await context.result_future
 
     if not hasattr(ft.Page, 'eval_js'):
         ft.Page.eval_js = eval_js

js_execution.py

@@ -1,9 +1,10 @@
 import asyncio
 import logging
 import contextlib
-from fastapi import FastAPI, Request
+from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect
 from fastapi.responses import HTMLResponse
 import uuid
+import html
 
 logger = logging.getLogger(__name__)
 
@@ -20,8 +21,10 @@ class JSExecutionRouter:
 
     def init_app(self, app: FastAPI):
         self.app = app
+        
     def get_eval_js_html(self, js_code: str, result_id: str):
-        escaped_js = js_code.replace('"', '"')
+        # escaped_js = html.escape(js_code)
+        logger.info(f"Generating HTML for JavaScript execution. Result ID: {result_id}")
         return f"""
         <!DOCTYPE html>
         <html lang="en">
@@ -31,9 +34,12 @@ class JSExecutionRouter:
         </head>
         <body>
             <script>
+            console.log("JavaScript execution page loaded");
             async function executeAndReturn() {{
+                console.log("executeAndReturn function called");
                 try {{
-                    const result = await (async () => {{ {escaped_js} }})();
+                    const result = await (async () => {{ {js_code} }})();
+                    console.log("JavaScript execution result:", result);
                     await fetch('/js-result/{result_id}', {{
                         method: 'POST',
                         headers: {{ 'Content-Type': 'application/json' }},
@@ -46,8 +52,6 @@ class JSExecutionRouter:
                         headers: {{ 'Content-Type': 'application/json' }},
                         body: JSON.stringify({{ error: error.toString() }})
                     }});
-                }} finally {{
-                    window.close();
                 }}
             }}
             executeAndReturn();
@@ -66,10 +70,12 @@ class JSExecutionRouter:
         
         @self.app.get(f"/eval-js/{result_id}", response_class=HTMLResponse)
         async def serve_eval_js_page():
+            logger.info(f"Serving eval-js page for result ID: {result_id}")
             return HTMLResponse(content=self.get_eval_js_html(js_code, result_id))
         
         @self.app.post(f"/js-result/{result_id}")
         async def receive_js_result(data: dict):
+            logger.info(f"Received result for ID: {result_id}")
             if 'error' in data:
                 result_future.set_exception(Exception(data['error']))
             else:
@@ -92,9 +98,32 @@ js_execution_router = JSExecutionRouter()
 def add_js_execution_routes(app: FastAPI):
     js_execution_router.init_app(app)
 
+    @app.websocket("/ws/camera")
+    async def websocket_endpoint(websocket: WebSocket):
+        await websocket.accept()
+        try:
+            while True:
+                data = await websocket.receive_text()
+                await broadcast_camera_frame(data)
+        except WebSocketDisconnect:
+            pass
+        except Exception as e:
+            logger.error(f"WebSocket error: {str(e)}")
+
+# Add this new function to handle broadcasting
+async def broadcast_camera_frame(frame_data: str):
+    # This function will be called from the Flet app to set the callback
+    if hasattr(broadcast_camera_frame, 'callback'):
+        await broadcast_camera_frame.callback(frame_data)
+
+# Function to set the callback from the Flet app
+def set_camera_frame_callback(callback):
+    broadcast_camera_frame.callback = callback
+
 def mount_flet_app(app: FastAPI, route: str, flet_app):
     # Mount the Flet app
     app.mount("/flet-app-internal", flet_app)
+    app.flet_app = flet_app  # Store reference to the Flet app
 
     @app.get(route, response_class=HTMLResponse)
     async def serve_main_page(request: Request):
@@ -104,7 +133,7 @@ def mount_flet_app(app: FastAPI, route: str, flet_app):
         <head>
             <meta charset="UTF-8">
             <meta name="viewport" content="width=device-width, initial-scale=1.0">
-            <title>Audio Recorder App</title>
+            <title>Camera Stream App</title>
             <style>
                 body, html {{
                     margin: 0;