import { useEffect, useMemo, useState } from 'react'; import { VideoInput } from './components/VideoInput'; import { ProcessingPipeline } from './components/ProcessingPipeline'; import { KeypointsDisplay } from './components/KeypointsDisplay'; import { TranslationOutput } from './components/TranslationOutput'; import { Sidebar, type SidebarSection } from './components/Sidebar'; type PredictionResult = { label: string; score: number; elapsed_ms: number; uid?: string; }; type ResultRecord = PredictionResult & { id: string; source: 'upload' | 'camera'; filename: string; created_at: string; }; const RECORDING_MS = 2000; const MAX_HISTORY = 200; const HISTORY_STORAGE_KEY = 'lumisign_history_v1'; const SAVED_STORAGE_KEY = 'lumisign_saved_v1'; function createRecordId() { if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') { return crypto.randomUUID(); } return `${Date.now()}-${Math.random().toString(16).slice(2)}`; } function loadStoredRecords(key: string): ResultRecord[] { try { const raw = localStorage.getItem(key); if (!raw) return []; const parsed = JSON.parse(raw) as unknown; if (!Array.isArray(parsed)) return []; return parsed.filter((entry): entry is ResultRecord => { return ( entry !== null && typeof entry === 'object' && typeof (entry as { id?: unknown }).id === 'string' && typeof (entry as { label?: unknown }).label === 'string' && typeof (entry as { score?: unknown }).score === 'number' && typeof (entry as { elapsed_ms?: unknown }).elapsed_ms === 'number' ); }); } catch { return []; } } function formatDateTime(iso: string) { const date = new Date(iso); if (Number.isNaN(date.getTime())) return iso; return date.toLocaleString(); } function escapeCsvValue(value: string | number | boolean) { const text = String(value); if (text.includes('"') || text.includes(',') || text.includes('\n')) { return `"${text.replace(/"/g, '""')}"`; } return text; } export default function App() { const [activeSection, setActiveSection] = useState('new'); const [isProcessing, setIsProcessing] = useState(false); const [currentStep, setCurrentStep] = useState(0); const [videoFile, setVideoFile] = useState(null); const [prediction, setPrediction] = useState(null); const [error, setError] = useState(null); const [cameraStream, setCameraStream] = useState(null); const [previewUrl, setPreviewUrl] = useState(null); const [history, setHistory] = useState(() => loadStoredRecords(HISTORY_STORAGE_KEY)); const [saved, setSaved] = useState(() => loadStoredRecords(SAVED_STORAGE_KEY)); const [latestRecord, setLatestRecord] = useState(null); const [notice, setNotice] = useState(null); useEffect(() => { if (!videoFile) { setPreviewUrl(null); return; } const url = URL.createObjectURL(videoFile); setPreviewUrl(url); return () => URL.revokeObjectURL(url); }, [videoFile]); useEffect(() => { return () => { cameraStream?.getTracks().forEach((track) => track.stop()); }; }, [cameraStream]); useEffect(() => { localStorage.setItem(HISTORY_STORAGE_KEY, JSON.stringify(history)); }, [history]); useEffect(() => { localStorage.setItem(SAVED_STORAGE_KEY, JSON.stringify(saved)); }, [saved]); useEffect(() => { if (!notice) return; const timeout = window.setTimeout(() => setNotice(null), 3000); return () => window.clearTimeout(timeout); }, [notice]); const stopCamera = () => { cameraStream?.getTracks().forEach((track) => track.stop()); setCameraStream(null); }; const handleToggleCamera = async () => { if (cameraStream) { stopCamera(); return; } try { const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false, }); setCameraStream(stream); setActiveSection('new'); setVideoFile(null); setPrediction(null); setError(null); setCurrentStep(0); } catch (err) { const message = err instanceof Error ? err.message : 'Unable to access camera.'; setError(message); } }; const handleVideoUpload = (file: File) => { if (cameraStream) { stopCamera(); } setActiveSection('new'); setVideoFile(file); setPrediction(null); setError(null); setCurrentStep(0); }; const recordClip = (stream: MediaStream, durationMs: number) => { return new Promise((resolve, reject) => { const options: MediaRecorderOptions = {}; if (MediaRecorder.isTypeSupported('video/webm;codecs=vp9')) { options.mimeType = 'video/webm;codecs=vp9'; } else if (MediaRecorder.isTypeSupported('video/webm;codecs=vp8')) { options.mimeType = 'video/webm;codecs=vp8'; } else if (MediaRecorder.isTypeSupported('video/webm')) { options.mimeType = 'video/webm'; } const recorder = new MediaRecorder(stream, options); const chunks: BlobPart[] = []; recorder.ondataavailable = (event) => { if (event.data.size > 0) { chunks.push(event.data); } }; recorder.onerror = () => reject(new Error('Recording failed.')); recorder.onstop = () => { const blob = new Blob(chunks, { type: recorder.mimeType || 'video/webm' }); resolve(blob); }; recorder.start(); window.setTimeout(() => { if (recorder.state !== 'inactive') { recorder.stop(); } }, durationMs); }); }; const requestPrediction = async (blob: Blob, filename: string) => { const form = new FormData(); form.append('file', blob, filename); const resp = await fetch('/predict', { method: 'POST', body: form }); const responseText = await resp.text(); let data: unknown = null; if (responseText) { try { data = JSON.parse(responseText); } catch { data = null; } } if (!resp.ok) { const errorMessage = data && typeof data === 'object' && 'error' in data ? String((data as { error?: unknown }).error ?? '') : responseText; throw new Error(errorMessage || `Request failed (${resp.status})`); } if (!data || typeof data !== 'object') { throw new Error('Backend returned an empty response.'); } return data as PredictionResult; }; const handleCopyPrediction = async () => { const record = latestRecord; if (!record) { setNotice('No prediction available to copy.'); return; } const text = [ `Label: ${record.label}`, `Confidence: ${(record.score * 100).toFixed(1)}%`, `Processing Time: ${record.elapsed_ms} ms`, `Source: ${record.source}`, `Timestamp: ${formatDateTime(record.created_at)}`, ].join('\n'); try { await navigator.clipboard.writeText(text); setNotice('Prediction copied to clipboard.'); } catch { setNotice('Failed to copy. Please allow clipboard access.'); } }; const handleSaveCurrentResult = () => { if (!latestRecord) { setNotice('No prediction available to save.'); return; } setSaved((prev) => { if (prev.some((item) => item.id === latestRecord.id)) { return prev; } return [latestRecord, ...prev]; }); setNotice('Result saved.'); }; const hasSavedCurrent = latestRecord ? saved.some((item) => item.id === latestRecord.id) : false; const handleExportResults = () => { const merged = new Map(); for (const item of history) { merged.set(item.id, { ...item, saved: false }); } for (const item of saved) { const existing = merged.get(item.id); if (existing) { merged.set(item.id, { ...existing, saved: true }); } else { merged.set(item.id, { ...item, saved: true }); } } const rows = Array.from(merged.values()).sort((a, b) => b.created_at.localeCompare(a.created_at), ); if (rows.length === 0) { setNotice('No results to export yet.'); return; } const headers = ['id', 'label', 'confidence', 'elapsed_ms', 'source', 'filename', 'created_at', 'saved']; const csvLines = [ headers.join(','), ...rows.map((item) => [ item.id, item.label, item.score.toFixed(6), item.elapsed_ms, item.source, item.filename, item.created_at, item.saved, ] .map(escapeCsvValue) .join(','), ), ]; const blob = new Blob([csvLines.join('\n')], { type: 'text/csv;charset=utf-8' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); const stamp = new Date().toISOString().replace(/[:.]/g, '-'); link.href = url; link.download = `lumisign-results-${stamp}.csv`; link.click(); URL.revokeObjectURL(url); setNotice('Export complete.'); }; const handleStartTranslation = async () => { if (isProcessing) return; if (!videoFile && !cameraStream) { setError('Please upload a video or enable the camera.'); return; } setIsProcessing(true); setCurrentStep(1); setPrediction(null); setError(null); try { let blob: Blob; let filename: string; let source: ResultRecord['source']; if (videoFile) { blob = videoFile; filename = videoFile.name; source = 'upload'; } else if (cameraStream) { setCurrentStep(2); blob = await recordClip(cameraStream, RECORDING_MS); filename = 'recording.webm'; source = 'camera'; } else { setIsProcessing(false); return; } setCurrentStep(3); const result = await requestPrediction(blob, filename); setPrediction(result); const record: ResultRecord = { ...result, id: createRecordId(), source, filename, created_at: new Date().toISOString(), }; setLatestRecord(record); setHistory((prev) => [record, ...prev].slice(0, MAX_HISTORY)); setCurrentStep(4); setActiveSection('new'); } catch (err) { const message = err instanceof Error ? err.message : 'Prediction failed.'; setError(message); setCurrentStep(0); } finally { setIsProcessing(false); } }; const handleReset = () => { setIsProcessing(false); setCurrentStep(0); setVideoFile(null); setPrediction(null); setError(null); stopCamera(); }; const analytics = useMemo(() => { const total = history.length; const averageConfidence = total > 0 ? history.reduce((sum, item) => sum + item.score, 0) / total : 0; const labelCounts = new Map(); for (const item of history) { labelCounts.set(item.label, (labelCounts.get(item.label) ?? 0) + 1); } const topLabels = Array.from(labelCounts.entries()) .sort((a, b) => b[1] - a[1]) .slice(0, 5); return { total, saved: saved.length, averageConfidence, topLabels, }; }, [history, saved.length]); const renderRecordList = (records: ResultRecord[], emptyText: string) => { if (records.length === 0) { return
{emptyText}
; } return (
{records.slice(0, 20).map((item) => ( ))}
); }; const renderActiveSection = () => { switch (activeSection) { case 'history': return (

History

{renderRecordList(history, 'No translation history yet.')}
); case 'saved': return (

Saved Results

{renderRecordList(saved, 'No saved results yet.')}
); case 'analytics': return (

Analytics

Total Predictions
{analytics.total}
Saved Results
{analytics.saved}
Average Confidence
{(analytics.averageConfidence * 100).toFixed(1)}%
Top Predicted Labels
{analytics.topLabels.length === 0 ? (
Run a few translations to see trends.
) : (
{analytics.topLabels.map(([label, count]) => (
{label}: {count}
))}
)}
); case 'documentation': return (

Documentation

1. Start backend (`uvicorn`) and frontend (`npm run dev`).
2. Upload a sign video or record with camera.
3. Click Start Translation to send it to `/predict`.
4. Save useful outputs or export CSV for analysis.
MODEL_DATASET=Dataset MODEL_TYPE=transformer MODEL_TRANSFORMER_SIZE=large
); case 'help': return (

Help Center

1. If translation fails, confirm backend is reachable on the configured port.
2. If camera fails, allow browser camera permission and retry.
3. If no output appears, check backend logs for model or media errors.
4. Use Export Results to share reproducible prediction history.
); case 'settings': return (

Settings

Language: English
); case 'new': default: return null; } }; return (
{/* Header */}

Indian Sign Language Translator

Real-time ISL to English translation

Language: English
{notice && (
{notice}
)} {renderActiveSection()} {/* Video Input Section */} {/* Processing Pipeline */} {/* Keypoints Display */} {/* Translation Output */}
); }