Spaces:
Sleeping
Sleeping
File size: 5,516 Bytes
5029d29 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | """Frame-push interpreter for the web app (browser-camera architecture).
Unlike `CameraInterpreter` (which owns a server-side webcam — only usable
locally), this receives frames pushed from the browser over HTTP, runs the
holistic tracker + recognizer, and returns compact landmarks + glosses for the
browser to draw. This is what makes the app deployable to Hugging Face, where
the server has no camera.
Tuned for CPU-only hosts (HF free tier): MediaPipe at `model_complexity=0`,
face refinement off, server-side drawing skipped. MediaPipe's graph is stateful
and not thread-safe, so all processing is serialised behind one lock — fine for
a demo; for many concurrent users you'd shard sessions.
"""
from __future__ import annotations
import threading
import time
from collections import deque
from typing import Deque, List, Optional
import numpy as np
from .camera import InterpreterPipeline
from .landmarks import points_payload
from .recognizer import Recognition
class FrameSession:
"""Process-wide singleton that turns pushed frames into landmarks + glosses."""
_instance: Optional["FrameSession"] = None
_init_lock = threading.Lock()
def __init__(
self,
*,
templates_path: str = "signs/lsf_signs.json",
model_path: Optional[str] = None,
model_complexity: int = 0,
refine_face_landmarks: bool = False,
max_transcript: int = 200,
) -> None:
self._pipeline = InterpreterPipeline(
templates_path=templates_path,
model_path=model_path,
model_complexity=model_complexity,
refine_face_landmarks=refine_face_landmarks,
)
self._lock = threading.Lock()
self._transcript: Deque[dict] = deque(maxlen=max_transcript)
self._seq = 0
self._fps = 0.0
self._last_t = 0.0
# Depth capability reported by the client device (dual-pixel detection).
self._depth_source = "mediapipe"
self._depth_reason = "No client depth reported yet."
@classmethod
def instance(cls, **kwargs) -> "FrameSession":
with cls._init_lock:
if cls._instance is None:
cls._instance = cls(**kwargs)
return cls._instance
# -- depth capability (set by the browser) -------------------------------
def set_depth_capability(self, *, dual_pixel: bool, reason: str = "", label: str = "") -> None:
if dual_pixel:
self._depth_source = "dual_pixel"
self._depth_reason = f"Dual-pixel/depth-capable device: {label or 'detected'}."
else:
self._depth_source = "mediapipe"
self._depth_reason = reason or "No dual-pixel depth on this device; using MediaPipe z."
# -- main entry point -----------------------------------------------------
def process_jpeg(self, jpeg_bytes: bytes) -> dict:
"""Decode a pushed JPEG, run detection + recognition, return a payload."""
import cv2 # local import keeps module import cheap
arr = np.frombuffer(jpeg_bytes, dtype=np.uint8)
frame = cv2.imdecode(arr, cv2.IMREAD_COLOR)
if frame is None:
return {"ok": False, "error": "Could not decode frame."}
with self._lock:
result = self._pipeline.process(frame, draw=False)
reco = result.recognition
if reco is not None:
self._append(reco)
now = time.time()
if self._last_t:
self._fps = 0.85 * self._fps + 0.15 * (1.0 / max(now - self._last_t, 1e-3))
self._last_t = now
return {
"ok": True,
"landmarks": points_payload(result.landmarks),
"reco": self._reco_dict(reco) if reco else None,
"status": self.status,
}
# -- transcript -----------------------------------------------------------
def _append(self, reco: Recognition) -> None:
self._seq += 1
self._transcript.append(self._reco_dict(reco, seq=self._seq))
@staticmethod
def _reco_dict(reco: Recognition, seq: int = 0) -> dict:
return {
"seq": seq,
"gloss": reco.gloss,
"confidence": round(reco.confidence, 3),
"source": reco.source,
"t": reco.timestamp,
}
def transcript(self, since: int = 0) -> List[dict]:
with self._lock:
return [e for e in self._transcript if e["seq"] > since]
def clear_transcript(self) -> None:
with self._lock:
self._transcript.clear()
# -- recording / vocabulary proxies --------------------------------------
def start_recording(self, label: str) -> None:
self._pipeline.recognizer.start_recording(label)
def stop_recording(self) -> Optional[str]:
return self._pipeline.recognizer.stop_recording()
def cancel_recording(self) -> None:
self._pipeline.recognizer.cancel_recording()
def template_counts(self) -> dict:
return self._pipeline.recognizer.template_counts()
def delete_gloss(self, gloss: str) -> bool:
return self._pipeline.recognizer.delete_gloss(gloss)
@property
def status(self) -> dict:
rec = self._pipeline.recognizer
return {
"depth_source": self._depth_source,
"depth_reason": self._depth_reason,
"fps": round(self._fps, 1),
"recording": rec.is_recording,
"trained": len(rec.vocabulary),
}
|