| from __future__ import annotations |
|
|
| import tempfile |
| import time |
| from pathlib import Path |
| from typing import Any |
|
|
| from .asl.mediapipe_utils import draw_holistic_landmarks |
|
|
|
|
| def create_debug_overlay_video(video_path: str | Path, result: dict[str, Any]) -> str: |
| cv2 = _load_cv2() |
| mp, holistic = _load_holistic() |
| path = Path(video_path) |
| cap = cv2.VideoCapture(str(path)) |
| if not cap.isOpened(): |
| raise ValueError(f"Could not open video for debug overlay: {path}") |
|
|
| width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) or 640) |
| height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) or 480) |
| fps = float(cap.get(cv2.CAP_PROP_FPS) or 24.0) |
| if fps <= 0: |
| fps = 24.0 |
|
|
| output_path = Path(tempfile.gettempdir()) / f"signspeak_debug_{int(time.time() * 1000)}.mp4" |
| writer = cv2.VideoWriter( |
| str(output_path), |
| cv2.VideoWriter_fourcc(*"mp4v"), |
| fps, |
| (width, height), |
| ) |
| if not writer.isOpened(): |
| cap.release() |
| raise RuntimeError(f"Could not create debug overlay video: {output_path}") |
|
|
| asl = result.get("asl", {}) |
| emotion = result.get("emotion", {}) |
| intent = result.get("intent_input", {}) |
| glosses = intent.get("detected_glosses") or asl.get("gloss_sequence") or [] |
| gloss_text = " ".join(str(gloss) for gloss in glosses) if glosses else "NO ASL WORDS DETECTED" |
| emotion_text = str(emotion.get("dominant_emotion", "unknown")).upper() |
| top_prediction = asl.get("top_prediction") or "none" |
| confidence = float(asl.get("confidence", 0.0) or 0.0) |
| threshold = float(asl.get("confidence_threshold", 0.0) or 0.0) |
| windows_used = int(asl.get("windows_used", 0) or 0) |
| status_text = ( |
| f"ASL {asl.get('status', 'unknown')} | top {top_prediction} " |
| f"{confidence:.2f}/{threshold:.2f} | windows {windows_used} | EMOTION {emotion.get('status', 'unknown')}" |
| ) |
|
|
| try: |
| while True: |
| ok, frame = cap.read() |
| if not ok or frame is None: |
| break |
| if mp is not None and holistic is not None: |
| frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) |
| results = holistic.process(frame_rgb) |
| draw_holistic_landmarks(mp, frame, results) |
| _draw_overlay(cv2, frame, gloss_text, emotion_text, status_text) |
| writer.write(frame) |
| finally: |
| cap.release() |
| if holistic is not None: |
| holistic.close() |
| writer.release() |
|
|
| return str(output_path) |
|
|
|
|
| def _draw_overlay(cv2, frame, gloss_text: str, emotion_text: str, status_text: str) -> None: |
| height, width = frame.shape[:2] |
| pad = 14 |
| panel_height = 112 |
| cv2.rectangle(frame, (0, 0), (width, panel_height), (8, 11, 16), -1) |
| cv2.rectangle(frame, (0, panel_height - 2), (width, panel_height), (45, 212, 191), -1) |
|
|
| _put_text(cv2, frame, "DETECTED ASL", (pad, 28), 0.52, (203, 213, 225), 1) |
| _put_text(cv2, frame, gloss_text, (pad, 64), 0.86, (248, 250, 252), 2) |
| _put_text(cv2, frame, f"EMOTION: {emotion_text}", (pad, 96), 0.58, (245, 158, 11), 2) |
|
|
| status_size = cv2.getTextSize(status_text, cv2.FONT_HERSHEY_SIMPLEX, 0.46, 1)[0] |
| x = max(pad, width - status_size[0] - pad) |
| _put_text(cv2, frame, status_text, (x, height - 18), 0.46, (203, 213, 225), 1) |
|
|
|
|
| def _put_text(cv2, frame, text: str, origin: tuple[int, int], scale: float, color: tuple[int, int, int], thickness: int) -> None: |
| cv2.putText( |
| frame, |
| text[:90], |
| origin, |
| cv2.FONT_HERSHEY_SIMPLEX, |
| scale, |
| color, |
| thickness, |
| cv2.LINE_AA, |
| ) |
|
|
|
|
| def _load_cv2(): |
| try: |
| import cv2 |
|
|
| return cv2 |
| except Exception as exc: |
| raise RuntimeError("OpenCV is required for debug overlay video generation.") from exc |
|
|
|
|
| def _load_holistic(): |
| try: |
| import mediapipe as mp |
|
|
| return ( |
| mp, |
| mp.solutions.holistic.Holistic( |
| min_detection_confidence=0.5, |
| min_tracking_confidence=0.5, |
| ), |
| ) |
| except Exception: |
| return None, None |
|
|
