Spaces:
Running
Running
File size: 4,491 Bytes
ff324d9 0a8cce5 ff324d9 0a8cce5 ff324d9 0a8cce5 ff324d9 0a8cce5 ff324d9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | import { useState, useRef, useEffect, useCallback } from "react";
import VideoCapture from "./VideoCapture";
import DraggableContainer from "./DraggableContainer";
import PromptInput from "./PromptInput";
import LiveCaption from "./LiveCaption";
import { useVLMContext } from "../context/useVLMContext";
import { PROMPTS, TIMING } from "../constants";
interface CaptioningViewProps {
videoRef: React.RefObject<HTMLVideoElement | null>;
videoMode?: "camera" | "upload";
}
function useCaptioningLoop(
videoRef: React.RefObject<HTMLVideoElement | null>,
isRunning: boolean,
promptRef: React.RefObject<string>,
onCaptionUpdate: (caption: string) => void,
onError: (error: string) => void,
) {
const { isLoaded, runInference } = useVLMContext();
const abortControllerRef = useRef<AbortController | null>(null);
const onCaptionUpdateRef = useRef(onCaptionUpdate);
const onErrorRef = useRef(onError);
useEffect(() => {
onCaptionUpdateRef.current = onCaptionUpdate;
}, [onCaptionUpdate]);
useEffect(() => {
onErrorRef.current = onError;
}, [onError]);
useEffect(() => {
abortControllerRef.current?.abort();
if (!isRunning || !isLoaded) return;
abortControllerRef.current = new AbortController();
const signal = abortControllerRef.current.signal;
const video = videoRef.current;
const captureLoop = async () => {
while (!signal.aborted) {
if (video && video.readyState >= 2 && !video.paused && video.videoWidth > 0) {
try {
const currentPrompt = promptRef.current || "";
const result = await runInference(video, currentPrompt, onCaptionUpdateRef.current);
if (result && !signal.aborted) onCaptionUpdateRef.current(result);
} catch (error) {
if (!signal.aborted) {
const message = error instanceof Error ? error.message : String(error);
onErrorRef.current(message);
console.error("Error processing frame:", error);
}
}
}
if (signal.aborted) break;
await new Promise((resolve) => setTimeout(resolve, TIMING.FRAME_CAPTURE_DELAY));
}
};
// NB: Wrap with a setTimeout to ensure abort controller can run before starting the loop
// This is necessary for React's strict mode which calls effects twice in development.
setTimeout(captureLoop, 0);
return () => {
abortControllerRef.current?.abort();
};
}, [isRunning, isLoaded, runInference, promptRef, videoRef]);
}
export default function CaptioningView({ videoRef, videoMode = "camera" }: CaptioningViewProps) {
const [caption, setCaption] = useState<string>("");
const [isLoopRunning, setIsLoopRunning] = useState<boolean>(true);
const [currentPrompt, setCurrentPrompt] = useState<string>(PROMPTS.default);
const [error, setError] = useState<string | null>(null);
// Use ref to store current prompt to avoid loop restarts
const promptRef = useRef<string>(currentPrompt);
// Update prompt ref when state changes
useEffect(() => {
promptRef.current = currentPrompt;
}, [currentPrompt]);
const handleCaptionUpdate = useCallback((newCaption: string) => {
setCaption(newCaption);
setError(null);
}, []);
const handleError = useCallback((errorMessage: string) => {
setError(errorMessage);
setCaption(`Error: ${errorMessage}`);
}, []);
useCaptioningLoop(videoRef, isLoopRunning, promptRef, handleCaptionUpdate, handleError);
const handlePromptChange = useCallback((prompt: string) => {
setCurrentPrompt(prompt);
setError(null);
}, []);
const handleToggleLoop = useCallback(() => {
setIsLoopRunning((prev) => !prev);
if (error) setError(null);
}, [error]);
return (
<div className="absolute inset-0 text-white">
<div className="relative w-full h-full">
<VideoCapture
isRunning={isLoopRunning}
onToggleRunning={handleToggleLoop}
error={error}
videoMode={videoMode}
/>
{/* Draggable Prompt Input - Bottom Left */}
<DraggableContainer initialPosition="bottom-left">
<PromptInput onPromptChange={handlePromptChange} />
</DraggableContainer>
{/* Draggable Live Caption - Bottom Right */}
<DraggableContainer initialPosition="bottom-right">
<LiveCaption caption={caption} isRunning={isLoopRunning} error={error} />
</DraggableContainer>
</div>
</div>
);
}
|