import { useEffect, useRef, useState } from 'react' import { motion, AnimatePresence } from 'framer-motion' import { Settings, Hand } from 'lucide-react' import { useWebcam } from './hooks/useWebcam' import { useMediaPipe } from './hooks/useMediaPipe' import { useWebSocket } from './hooks/useWebSocket' import { WebcamFeed } from './components/WebcamFeed' import { PredictionHUD } from './components/PredictionHUD' import { OnboardingGuide } from './components/OnboardingGuide' import { Calibration } from './components/Calibration' import { ModelSelector } from './components/ModelSelector' import type { ModelMode } from './types' type AppStage = 'onboarding' | 'model-select' | 'calibration' | 'running' function App() { // ── Stage management ───────────────────────────────────────── const showOnboarding = !localStorage.getItem('sanketsetu-onboarded') const [stage, setStage] = useState(showOnboarding ? 'onboarding' : 'model-select') const savedModel = localStorage.getItem('sanketsetu-model-mode') as ModelMode | null const [selectedModel, setSelectedModel] = useState(savedModel ?? 'ensemble') // Default all modes to available so the selector is usable even before the // health check completes or if the backend is temporarily unreachable. const [availableModes, setAvailableModes] = useState>( new Set(['ensemble', 'A', 'B', 'C']) ) const handleOnboardingDone = () => { localStorage.setItem('sanketsetu-onboarded', '1') setStage('model-select') } const handleModelContinue = () => { localStorage.setItem('sanketsetu-model-mode', selectedModel) setStage('calibration') } useEffect(() => { let active = true const healthUrl = `${resolveBackendHttpBase()}/health` const loadAvailability = async () => { try { const res = await fetch(healthUrl) if (!res.ok) return const data = (await res.json()) as { pipelines_available?: string[] } if (!active) return // Only restrict availability when the backend explicitly reports which // pipelines are loaded. If the list is empty (still loading) keep all // modes selectable so the user isn't blocked. const reported = data.pipelines_available ?? [] if (reported.length > 0) { const next = new Set(['ensemble']) for (const mode of reported) { if (mode === 'A' || mode === 'B' || mode === 'C') next.add(mode as ModelMode) } setAvailableModes(next) } } catch { // Keep local defaults when backend health is unavailable. } } loadAvailability() return () => { active = false } }, []) useEffect(() => { if (selectedModel !== 'ensemble' && !availableModes.has(selectedModel)) { setSelectedModel('ensemble') } }, [availableModes, selectedModel]) // ── Webcam ─────────────────────────────────────────────────── const { videoRef, isReady, error, facingMode, switchCamera } = useWebcam() // ── MediaPipe ──────────────────────────────────────────────── const { landmarks, rawLandmarks, isLoading: mpLoading, error: mpError, startDetection, stopDetection } = useMediaPipe() useEffect(() => { if (isReady && videoRef.current && stage === 'running') { startDetection(videoRef.current) } else if (stage !== 'running') { stopDetection() } }, [isReady, stage, startDetection, stopDetection, videoRef]) // Start detecting during calibration too (to detect hand) useEffect(() => { if (isReady && videoRef.current && stage === 'calibration') { startDetection(videoRef.current) } }, [isReady, stage, startDetection, videoRef]) // ── WebSocket ──────────────────────────────────────────────── const { lastPrediction, isConnected, latency, lowBandwidth, sendLandmarks } = useWebSocket() const imageCanvasRef = useRef(null) // Send landmarks on every new frame useEffect(() => { if (stage === 'running' && landmarks) { let imageB64: string | undefined if (selectedModel === 'C' && videoRef.current) { imageB64 = captureVideoFrame(videoRef.current, imageCanvasRef) } sendLandmarks(landmarks, { modelMode: selectedModel, imageB64, }) } }, [landmarks, selectedModel, sendLandmarks, stage, videoRef]) // Was the last prediction recently (within 1.5s)? const lastPredTs = useRef(0) const [recognised, setRecognised] = useState(false) useEffect(() => { if (lastPrediction) { lastPredTs.current = Date.now() setRecognised(true) setTimeout(() => setRecognised(false), 800) } }, [lastPrediction]) return (
{/* ── Onboarding overlay ─────────────────────────────────── */} {stage === 'onboarding' && ( )} {/* ── Model selector overlay ─────────────────────────────── */} {stage === 'model-select' && ( )} {/* ── Calibration overlay ────────────────────────────────── */} {stage === 'calibration' && ( setStage('running')} /> )} {/* ── Reconnecting banner ───────────────────────────────── */} {!isConnected && stage === 'running' && ( Reconnecting to server… )} {/* ── Low-bandwidth banner ─────────────────────────────────── */} {lowBandwidth && isConnected && ( High latency detected — reduced to 5 fps to conserve bandwidth )} {/* ── Header ─────────────────────────────────────────────── */}

SanketSetu | સંકેત-સેતુ

{mpLoading && Loading AI…} {mpLoading && AI…} {mpError && {mpError}}
{/* ── Main content ───────────────────────────────────────── */}
{/* Webcam panel */} {/* HUD panel */}
) } function captureVideoFrame( video: HTMLVideoElement, canvasRef: { current: HTMLCanvasElement | null }, ): string | undefined { if (!video.videoWidth || !video.videoHeight) return undefined if (!canvasRef.current) { canvasRef.current = document.createElement('canvas') } const canvas = canvasRef.current canvas.width = 128 canvas.height = 128 const ctx = canvas.getContext('2d') if (!ctx) return undefined // Center-crop to square before resizing to model input size. const side = Math.min(video.videoWidth, video.videoHeight) const sx = (video.videoWidth - side) / 2 const sy = (video.videoHeight - side) / 2 ctx.drawImage(video, sx, sy, side, side, 0, 0, 128, 128) return canvas.toDataURL('image/jpeg', 0.85).replace(/^data:image\/jpeg;base64,/, '') } function resolveBackendHttpBase(): string { const envWs = import.meta.env.VITE_WS_URL as string | undefined if (envWs) { return envWs .replace(/^wss:\/\//i, 'https://') .replace(/^ws:\/\//i, 'http://') } if (import.meta.env.DEV) return 'http://localhost:8000' return window.location.origin } export default App