feat: websocket endpoint, frontend utilities, face detection and pulse hook

backend/main.py

@@ -1,15 +1,49 @@
-from fastapi import FastAPI
+import logging
+import os
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
 from fastapi.staticfiles import StaticFiles
 from fastapi.responses import FileResponse
+from pydantic import BaseModel
+from .signal_processor import SignalProcessor
 
 app = FastAPI()
 
+
+class FrameData(BaseModel):
+    r: float
+    g: float
+    b: float
+
+
 @app.get("/health")
 async def health():
     return {"status": "ok"}
 
-app.mount("/assets", StaticFiles(directory="static/assets"), name="assets")
 
-@app.get("/{full_path:path}")
-async def serve_frontend(full_path: str):
-    return FileResponse("static/index.html")
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+    await websocket.accept()
+    processor = SignalProcessor()
+    try:
+        while True:
+            try:
+                data = await websocket.receive_json()
+                frame = FrameData(**data)
+                result = processor.process(frame.r, frame.g, frame.b)
+                if processor.just_computed:
+                    await websocket.send_json(result)
+            except WebSocketDisconnect:
+                break
+            except Exception as e:
+                logging.getLogger("uvicorn").exception("WebSocket processing error")
+                await websocket.send_json({"error": str(e), "status": "error"})
+    finally:
+        del processor
+
+
+if os.path.isdir("static"):
+    app.mount("/assets", StaticFiles(directory="static/assets"), name="assets")
+
+    @app.get("/{full_path:path}")
+    async def serve_frontend(full_path: str):
+        return FileResponse("static/index.html")

backend/signal_processor.py

@@ -24,6 +24,7 @@ class SignalProcessor:
             "waveform": [],
             "status": "buffering",
         }
+        self.just_computed: bool = False
 
     def append(self, r: float, g: float, b: float) -> None:
         self.r_buffer.append(r)
@@ -88,6 +89,7 @@ class SignalProcessor:
 
         n = len(self.r_buffer)
         if n < self.min_calibrate_samples:
+            self.just_computed = True
             self._last_result = {
                 "bpm": 0,
                 "confidence": 0.0,
@@ -97,9 +99,11 @@ class SignalProcessor:
             return self._last_result
 
         if now - self.last_compute_time < self.compute_interval:
+            self.just_computed = False
             return self._last_result
 
         self.last_compute_time = now
+        self.just_computed = True
 
         r_arr = np.array(self.r_buffer)
         g_arr = np.array(self.g_buffer)

frontend/src/hooks/useFaceDetection.ts

@@ -0,0 +1,79 @@
+import { useRef, useEffect, useState, useCallback } from "react";
+import { FaceDetector, FilesetResolver } from "@mediapipe/tasks-vision";
+
+export interface DetectedFace {
+  leftEye: { x: number; y: number };
+  rightEye: { x: number; y: number };
+  boundingBox: { x: number; y: number; width: number; height: number };
+}
+
+export function useFaceDetection() {
+  const detectorRef = useRef<FaceDetector | null>(null);
+  const [loading, setLoading] = useState(true);
+  const [error, setError] = useState<string | null>(null);
+
+  useEffect(() => {
+    let cancelled = false;
+
+    async function init() {
+      try {
+        const vision = await FilesetResolver.forVisionTasks(
+          "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.35/wasm"
+        );
+        if (cancelled) return;
+        const detector = await FaceDetector.createFromOptions(vision, {
+          baseOptions: {
+            modelAssetPath:
+              "https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_short_range/float16/latest/blaze_face_short_range.tflite",
+            delegate: "GPU",
+          },
+          runningMode: "VIDEO",
+        });
+        if (cancelled) {
+          detector.close();
+          return;
+        }
+        detectorRef.current = detector;
+        setLoading(false);
+      } catch (e) {
+        if (!cancelled) {
+          setError(e instanceof Error ? e.message : "Failed to load face detector");
+          setLoading(false);
+        }
+      }
+    }
+
+    init();
+    return () => {
+      cancelled = true;
+      detectorRef.current?.close();
+    };
+  }, []);
+
+  const detectFace = useCallback(
+    (video: HTMLVideoElement, timestampMs: number): DetectedFace | null => {
+      if (!detectorRef.current) return null;
+      const result = detectorRef.current.detectForVideo(video, timestampMs);
+      if (!result.detections || result.detections.length === 0) return null;
+
+      const detection = result.detections[0];
+      const keypoints = detection.keypoints;
+      if (!keypoints || keypoints.length < 2) return null;
+
+      const bb = detection.boundingBox;
+      if (!bb) return null;
+
+      const leftEye = keypoints[0];
+      const rightEye = keypoints[1];
+
+      return {
+        leftEye: { x: leftEye.x, y: leftEye.y },
+        rightEye: { x: rightEye.x, y: rightEye.y },
+        boundingBox: { x: bb.originX, y: bb.originY, width: bb.width, height: bb.height },
+      };
+    },
+    []
+  );
+
+  return { detectFace, loading, error };
+}

frontend/src/hooks/usePulseCam.ts

@@ -0,0 +1,131 @@
+import { useRef, useEffect, useState, useCallback } from "react";
+import {
+  WS_URL,
+  HEALTH_CHECK_URL,
+  COLD_START_MAX_WAIT_MS,
+  COLD_START_CHECK_INTERVAL_MS,
+  COLD_START_ABORT_TIMEOUT_MS,
+  MAX_WS_RETRIES,
+  WS_RETRY_BASE_MS,
+  type Phase,
+  type MeasurementStatus,
+  type WSResponse,
+} from "../lib/constants";
+
+async function waitForServer(maxWaitMs = COLD_START_MAX_WAIT_MS): Promise<boolean> {
+  const start = Date.now();
+  while (Date.now() - start < maxWaitMs) {
+    try {
+      const res = await fetch(HEALTH_CHECK_URL, {
+        signal: AbortSignal.timeout(COLD_START_ABORT_TIMEOUT_MS),
+      });
+      if (res.ok) return true;
+    } catch {}
+    await new Promise((r) => setTimeout(r, COLD_START_CHECK_INTERVAL_MS));
+  }
+  return false;
+}
+
+export function usePulseCam() {
+  const [phase, setPhase] = useState<Phase>("idle");
+  const [bpm, setBpm] = useState(0);
+  const [confidence, setConfidence] = useState(0);
+  const [waveform, setWaveform] = useState<number[]>([]);
+  const [status, setStatus] = useState<MeasurementStatus>("buffering");
+  const wsRef = useRef<WebSocket | null>(null);
+  const intentionalStop = useRef(false);
+  const retryCount = useRef(0);
+  const reconnectScheduled = useRef(false);
+
+  const connect = useCallback(() => {
+    if (intentionalStop.current) return;
+    const ws = new WebSocket(WS_URL);
+    wsRef.current = ws;
+
+    ws.onopen = () => {
+      setPhase("measuring");
+      retryCount.current = 0;
+    };
+
+    ws.onmessage = (event) => {
+      try {
+        const raw = JSON.parse(event.data);
+        if (raw.error) {
+          setStatus("error");
+          return;
+        }
+        const data: WSResponse = raw;
+        setBpm(data.bpm);
+        setConfidence(data.confidence);
+        setWaveform(data.waveform);
+        setStatus(data.status);
+      } catch (e) {
+        console.error("Failed to parse WebSocket message:", e);
+      }
+    };
+
+    ws.onerror = () => {
+      if (intentionalStop.current) return;
+      if (retryCount.current < MAX_WS_RETRIES) {
+        const delay = WS_RETRY_BASE_MS * Math.pow(2, retryCount.current);
+        retryCount.current++;
+        reconnectScheduled.current = true;
+        setTimeout(() => {
+          reconnectScheduled.current = false;
+          connect();
+        }, delay);
+      } else {
+        setPhase("error");
+      }
+    };
+
+    ws.onclose = () => {
+      if (intentionalStop.current) return;
+      if (!reconnectScheduled.current) {
+        setTimeout(() => connect(), 3000);
+      }
+    };
+  }, []);
+
+  const sendFrame = useCallback((r: number, g: number, b: number) => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      wsRef.current.send(JSON.stringify({ r, g, b }));
+    }
+  }, []);
+
+  const start = useCallback(async () => {
+    intentionalStop.current = false;
+    wsRef.current?.close();
+    wsRef.current = null;
+    setPhase("checking");
+
+    const isUp = await waitForServer();
+    if (!isUp) {
+      setPhase("error");
+      return;
+    }
+
+    setPhase("connecting");
+    connect();
+  }, [connect]);
+
+  const stop = useCallback(() => {
+    intentionalStop.current = true;
+    wsRef.current?.close();
+    wsRef.current = null;
+    setPhase("idle");
+    setBpm(0);
+    setConfidence(0);
+    setWaveform([]);
+    setStatus("buffering");
+  }, []);
+
+  useEffect(() => {
+    return () => {
+      intentionalStop.current = true;
+      wsRef.current?.close();
+    };
+  }, []);
+
+  return { phase, bpm, confidence, waveform, status, sendFrame, start, stop };
+}

frontend/src/lib/constants.ts

@@ -0,0 +1,34 @@
+export const FPS = 30;
+export const BUFFER_SECONDS = 15;
+export const COLD_START_MAX_WAIT_MS = 90000;
+export const COLD_START_CHECK_INTERVAL_MS = 3000;
+export const COLD_START_ABORT_TIMEOUT_MS = 5000;
+export const HEALTH_CHECK_URL = `${window.location.protocol}//${window.location.host}/health`;
+export const WS_URL = `${window.location.protocol === "https:" ? "wss:" : "ws:"}//${window.location.host}/ws`;
+export const MAX_WS_RETRIES = 5;
+export const WS_RETRY_BASE_MS = 2000;
+export const ROI_WIDTH_RATIO = 0.8;
+export const ROI_HEIGHT_RATIO = 0.4;
+export const ROI_OFFSET_UP_RATIO = 0.4;
+
+export type Phase =
+  | "idle"
+  | "checking"
+  | "waking"
+  | "connecting"
+  | "measuring"
+  | "error";
+
+export type MeasurementStatus =
+  | "buffering"
+  | "calibrating"
+  | "measuring"
+  | "poor_signal"
+  | "error";
+
+export interface WSResponse {
+  bpm: number;
+  confidence: number;
+  waveform: number[];
+  status: MeasurementStatus;
+}

frontend/src/lib/extractROI.ts

@@ -0,0 +1,67 @@
+import { ROI_WIDTH_RATIO, ROI_HEIGHT_RATIO, ROI_OFFSET_UP_RATIO } from "./constants";
+
+interface Keypoint {
+  x: number;
+  y: number;
+}
+
+export function computeForeheadROI(
+  leftEye: Keypoint,
+  rightEye: Keypoint,
+  canvasWidth: number,
+  canvasHeight: number
+): { x: number; y: number; width: number; height: number } {
+  const midX = (leftEye.x + rightEye.x) / 2;
+  const midY = (leftEye.y + rightEye.y) / 2;
+  const interEyeDist = Math.sqrt(
+    (rightEye.x - leftEye.x) ** 2 + (rightEye.y - leftEye.y) ** 2
+  );
+
+  const width = interEyeDist * ROI_WIDTH_RATIO;
+  const height = interEyeDist * ROI_HEIGHT_RATIO;
+  const offsetX = midX * canvasWidth;
+  const offsetY = midY * canvasHeight - interEyeDist * ROI_OFFSET_UP_RATIO * canvasHeight;
+
+  const x = Math.max(0, offsetX - width * canvasWidth / 2);
+  const y = Math.max(0, offsetY - height * canvasHeight);
+  const w = width * canvasWidth;
+  const h = height * canvasHeight;
+
+  return {
+    x,
+    y,
+    width: Math.min(w, canvasWidth - x),
+    height: Math.min(h, canvasHeight - y),
+  };
+}
+
+export function extractRGBMeans(
+  ctx: CanvasRenderingContext2D,
+  roi: { x: number; y: number; width: number; height: number }
+): { r: number; g: number; b: number } {
+  const x = Math.floor(roi.x);
+  const y = Math.floor(roi.y);
+  const w = Math.floor(roi.width);
+  const h = Math.floor(roi.height);
+
+  if (w <= 0 || h <= 0) return { r: 0, g: 0, b: 0 };
+
+  const imageData = ctx.getImageData(x, y, w, h);
+  const data = imageData.data;
+  let totalR = 0;
+  let totalG = 0;
+  let totalB = 0;
+  const pixelCount = data.length / 4;
+
+  for (let i = 0; i < data.length; i += 4) {
+    totalR += data[i];
+    totalG += data[i + 1];
+    totalB += data[i + 2];
+  }
+
+  return {
+    r: totalR / pixelCount,
+    g: totalG / pixelCount,
+    b: totalB / pixelCount,
+  };
+}