Spaces:
Running
Running
File size: 4,933 Bytes
ff324d9 52709db ff324d9 52709db ff324d9 52709db ff324d9 52709db ff324d9 52709db ff324d9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
import { useState, useRef, useEffect, useCallback } from "react";
import WebcamCapture from "./WebcamCapture";
import VideoScrubber from "./VideoScrubber";
import DraggableContainer from "./DraggableContainer";
import PromptInput from "./PromptInput";
import LiveCaption from "./LiveCaption";
import { useVLMContext } from "../context/useVLMContext";
import { PROMPTS, TIMING } from "../constants";
interface CaptioningViewProps {
videoRef: React.RefObject<HTMLVideoElement | null>;
sourceType?: 'webcam' | 'screen' | 'file' | null;
}
function useCaptioningLoop(
videoRef: React.RefObject<HTMLVideoElement | null>,
isRunning: boolean,
promptRef: React.RefObject<string>,
onCaptionUpdate: (caption: string) => void,
onError: (error: string) => void,
) {
const { isLoaded, runInference } = useVLMContext();
const abortControllerRef = useRef<AbortController | null>(null);
const onCaptionUpdateRef = useRef(onCaptionUpdate);
const onErrorRef = useRef(onError);
useEffect(() => {
onCaptionUpdateRef.current = onCaptionUpdate;
}, [onCaptionUpdate]);
useEffect(() => {
onErrorRef.current = onError;
}, [onError]);
useEffect(() => {
abortControllerRef.current?.abort();
if (!isRunning || !isLoaded) return;
abortControllerRef.current = new AbortController();
const signal = abortControllerRef.current.signal;
const video = videoRef.current;
const captureLoop = async () => {
while (!signal.aborted) {
if (video && video.readyState >= 2 && !video.paused && video.videoWidth > 0) {
try {
const currentPrompt = promptRef.current || "";
const result = await runInference(video, currentPrompt, onCaptionUpdateRef.current);
if (result && !signal.aborted) onCaptionUpdateRef.current(result);
} catch (error) {
if (!signal.aborted) {
const message = error instanceof Error ? error.message : String(error);
onErrorRef.current(message);
console.error("Error processing frame:", error);
}
}
}
if (signal.aborted) break;
await new Promise((resolve) => setTimeout(resolve, TIMING.FRAME_CAPTURE_DELAY));
}
};
// NB: Wrap with a setTimeout to ensure abort controller can run before starting the loop
// This is necessary for React's strict mode which calls effects twice in development.
setTimeout(captureLoop, 0);
return () => {
abortControllerRef.current?.abort();
};
}, [isRunning, isLoaded, runInference, promptRef, videoRef]);
}
export default function CaptioningView({ videoRef, sourceType }: CaptioningViewProps) {
const [caption, setCaption] = useState<string>("");
const [isLoopRunning, setIsLoopRunning] = useState<boolean>(true);
const [currentPrompt, setCurrentPrompt] = useState<string>(PROMPTS.default);
const [error, setError] = useState<string | null>(null);
// Use ref to store current prompt to avoid loop restarts
const promptRef = useRef<string>(currentPrompt);
// Update prompt ref when state changes
useEffect(() => {
promptRef.current = currentPrompt;
}, [currentPrompt]);
const handleCaptionUpdate = useCallback((newCaption: string) => {
setCaption(newCaption);
setError(null);
}, []);
const handleError = useCallback((errorMessage: string) => {
setError(errorMessage);
setCaption(`Error: ${errorMessage}`);
}, []);
useCaptioningLoop(videoRef, isLoopRunning, promptRef, handleCaptionUpdate, handleError);
const handlePromptChange = useCallback((prompt: string) => {
setCurrentPrompt(prompt);
setError(null);
}, []);
const handleToggleLoop = useCallback(() => {
setIsLoopRunning((prev) => !prev);
if (error) setError(null);
}, [error]);
return (
<div className="absolute inset-0 text-white">
<div className="relative w-full h-full">
<WebcamCapture isRunning={isLoopRunning} onToggleRunning={handleToggleLoop} error={error} />
{/* Video Scrubber - Only show for video files */}
<VideoScrubber
videoRef={videoRef}
isVisible={sourceType === 'file'}
/>
{/* Draggable Prompt Input - Bottom Left (above scrubber) */}
<DraggableContainer
initialPosition={sourceType === 'file' ? { x: 20, y: window.innerHeight - 200 } : "bottom-left"}
className="z-[150]"
>
<PromptInput onPromptChange={handlePromptChange} />
</DraggableContainer>
{/* Draggable Live Caption - Bottom Right (above scrubber) */}
<DraggableContainer
initialPosition={sourceType === 'file' ? { x: window.innerWidth - 170, y: window.innerHeight - 200 } : "bottom-right"}
className="z-[150]"
>
<LiveCaption caption={caption} isRunning={isLoopRunning} error={error} />
</DraggableContainer>
</div>
</div>
);
}
|