| | import { useState, useRef, useEffect, useCallback } from "react"; |
| | import WebcamCapture from "./WebcamCapture"; |
| | import DraggableContainer from "./DraggableContainer"; |
| | import PromptInput from "./PromptInput"; |
| | import LiveCaption from "./LiveCaption"; |
| | import { useVLMContext } from "../context/useVLMContext"; |
| | import { PROMPTS, TIMING } from "../constants"; |
| |
|
| | interface CaptioningViewProps { |
| | videoRef: React.RefObject<HTMLVideoElement | null>; |
| | } |
| |
|
| | function useCaptioningLoop( |
| | videoRef: React.RefObject<HTMLVideoElement | null>, |
| | isRunning: boolean, |
| | promptRef: React.RefObject<string>, |
| | onCaptionUpdate: (caption: string) => void, |
| | onError: (error: string) => void, |
| | ) { |
| | const { isLoaded, runInference } = useVLMContext(); |
| | const abortControllerRef = useRef<AbortController | null>(null); |
| | const onCaptionUpdateRef = useRef(onCaptionUpdate); |
| | const onErrorRef = useRef(onError); |
| |
|
| | useEffect(() => { |
| | onCaptionUpdateRef.current = onCaptionUpdate; |
| | }, [onCaptionUpdate]); |
| |
|
| | useEffect(() => { |
| | onErrorRef.current = onError; |
| | }, [onError]); |
| |
|
| | useEffect(() => { |
| | abortControllerRef.current?.abort(); |
| | if (!isRunning || !isLoaded) return; |
| |
|
| | abortControllerRef.current = new AbortController(); |
| | const signal = abortControllerRef.current.signal; |
| | const video = videoRef.current; |
| | const captureLoop = async () => { |
| | while (!signal.aborted) { |
| | if (video && video.readyState >= 2 && !video.paused && video.videoWidth > 0) { |
| | try { |
| | const currentPrompt = promptRef.current || ""; |
| | const result = await runInference(video, currentPrompt, onCaptionUpdateRef.current); |
| | if (result && !signal.aborted) onCaptionUpdateRef.current(result); |
| | } catch (error) { |
| | if (!signal.aborted) { |
| | const message = error instanceof Error ? error.message : String(error); |
| | onErrorRef.current(message); |
| | console.error("Error processing frame:", error); |
| | } |
| | } |
| | } |
| | if (signal.aborted) break; |
| | await new Promise((resolve) => setTimeout(resolve, TIMING.FRAME_CAPTURE_DELAY)); |
| | } |
| | }; |
| |
|
| | |
| | |
| | setTimeout(captureLoop, 0); |
| |
|
| | return () => { |
| | abortControllerRef.current?.abort(); |
| | }; |
| | }, [isRunning, isLoaded, runInference, promptRef, videoRef]); |
| | } |
| |
|
| | export default function CaptioningView({ videoRef }: CaptioningViewProps) { |
| | const [caption, setCaption] = useState<string>(""); |
| | const [isLoopRunning, setIsLoopRunning] = useState<boolean>(true); |
| | const [currentPrompt, setCurrentPrompt] = useState<string>(PROMPTS.default); |
| | const [error, setError] = useState<string | null>(null); |
| |
|
| | |
| | const promptRef = useRef<string>(currentPrompt); |
| |
|
| | |
| | useEffect(() => { |
| | promptRef.current = currentPrompt; |
| | }, [currentPrompt]); |
| |
|
| | const handleCaptionUpdate = useCallback((newCaption: string) => { |
| | setCaption(newCaption); |
| | setError(null); |
| | }, []); |
| |
|
| | const handleError = useCallback((errorMessage: string) => { |
| | setError(errorMessage); |
| | setCaption(`Error: ${errorMessage}`); |
| | }, []); |
| |
|
| | useCaptioningLoop(videoRef, isLoopRunning, promptRef, handleCaptionUpdate, handleError); |
| |
|
| | const handlePromptChange = useCallback((prompt: string) => { |
| | setCurrentPrompt(prompt); |
| | setError(null); |
| | }, []); |
| |
|
| | const handleToggleLoop = useCallback(() => { |
| | setIsLoopRunning((prev) => !prev); |
| | if (error) setError(null); |
| | }, [error]); |
| |
|
| | return ( |
| | <div className="absolute inset-0 text-white"> |
| | <div className="relative w-full h-full"> |
| | <WebcamCapture isRunning={isLoopRunning} onToggleRunning={handleToggleLoop} error={error} /> |
| | |
| | {/* Draggable Prompt Input - Bottom Left */} |
| | <DraggableContainer initialPosition="bottom-left"> |
| | <PromptInput onPromptChange={handlePromptChange} /> |
| | </DraggableContainer> |
| | |
| | {/* Draggable Live Caption - Bottom Right */} |
| | <DraggableContainer initialPosition="bottom-right"> |
| | <LiveCaption caption={caption} isRunning={isLoopRunning} error={error} /> |
| | </DraggableContainer> |
| | </div> |
| | </div> |
| | ); |
| | } |
| |
|