LumiSign / ui /src /app /App.tsx
anthony01's picture
update: enhance UI components and functionality, add model configuration, and update server proxy
ea15c76
Raw
History Blame Contribute Delete
20 kB
import { useEffect, useMemo, useState } from 'react';
import { VideoInput } from './components/VideoInput';
import { ProcessingPipeline } from './components/ProcessingPipeline';
import { KeypointsDisplay } from './components/KeypointsDisplay';
import { TranslationOutput } from './components/TranslationOutput';
import { Sidebar, type SidebarSection } from './components/Sidebar';
type PredictionResult = {
label: string;
score: number;
elapsed_ms: number;
uid?: string;
};
type ResultRecord = PredictionResult & {
id: string;
source: 'upload' | 'camera';
filename: string;
created_at: string;
};
const RECORDING_MS = 2000;
const MAX_HISTORY = 200;
const HISTORY_STORAGE_KEY = 'lumisign_history_v1';
const SAVED_STORAGE_KEY = 'lumisign_saved_v1';
function createRecordId() {
if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {
return crypto.randomUUID();
}
return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
}
function loadStoredRecords(key: string): ResultRecord[] {
try {
const raw = localStorage.getItem(key);
if (!raw) return [];
const parsed = JSON.parse(raw) as unknown;
if (!Array.isArray(parsed)) return [];
return parsed.filter((entry): entry is ResultRecord => {
return (
entry !== null &&
typeof entry === 'object' &&
typeof (entry as { id?: unknown }).id === 'string' &&
typeof (entry as { label?: unknown }).label === 'string' &&
typeof (entry as { score?: unknown }).score === 'number' &&
typeof (entry as { elapsed_ms?: unknown }).elapsed_ms === 'number'
);
});
} catch {
return [];
}
}
function formatDateTime(iso: string) {
const date = new Date(iso);
if (Number.isNaN(date.getTime())) return iso;
return date.toLocaleString();
}
function escapeCsvValue(value: string | number | boolean) {
const text = String(value);
if (text.includes('"') || text.includes(',') || text.includes('\n')) {
return `"${text.replace(/"/g, '""')}"`;
}
return text;
}
export default function App() {
const [activeSection, setActiveSection] = useState<SidebarSection>('new');
const [isProcessing, setIsProcessing] = useState(false);
const [currentStep, setCurrentStep] = useState(0);
const [videoFile, setVideoFile] = useState<File | null>(null);
const [prediction, setPrediction] = useState<PredictionResult | null>(null);
const [error, setError] = useState<string | null>(null);
const [cameraStream, setCameraStream] = useState<MediaStream | null>(null);
const [previewUrl, setPreviewUrl] = useState<string | null>(null);
const [history, setHistory] = useState<ResultRecord[]>(() => loadStoredRecords(HISTORY_STORAGE_KEY));
const [saved, setSaved] = useState<ResultRecord[]>(() => loadStoredRecords(SAVED_STORAGE_KEY));
const [latestRecord, setLatestRecord] = useState<ResultRecord | null>(null);
const [notice, setNotice] = useState<string | null>(null);
useEffect(() => {
if (!videoFile) {
setPreviewUrl(null);
return;
}
const url = URL.createObjectURL(videoFile);
setPreviewUrl(url);
return () => URL.revokeObjectURL(url);
}, [videoFile]);
useEffect(() => {
return () => {
cameraStream?.getTracks().forEach((track) => track.stop());
};
}, [cameraStream]);
useEffect(() => {
localStorage.setItem(HISTORY_STORAGE_KEY, JSON.stringify(history));
}, [history]);
useEffect(() => {
localStorage.setItem(SAVED_STORAGE_KEY, JSON.stringify(saved));
}, [saved]);
useEffect(() => {
if (!notice) return;
const timeout = window.setTimeout(() => setNotice(null), 3000);
return () => window.clearTimeout(timeout);
}, [notice]);
const stopCamera = () => {
cameraStream?.getTracks().forEach((track) => track.stop());
setCameraStream(null);
};
const handleToggleCamera = async () => {
if (cameraStream) {
stopCamera();
return;
}
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: true,
audio: false,
});
setCameraStream(stream);
setActiveSection('new');
setVideoFile(null);
setPrediction(null);
setError(null);
setCurrentStep(0);
} catch (err) {
const message = err instanceof Error ? err.message : 'Unable to access camera.';
setError(message);
}
};
const handleVideoUpload = (file: File) => {
if (cameraStream) {
stopCamera();
}
setActiveSection('new');
setVideoFile(file);
setPrediction(null);
setError(null);
setCurrentStep(0);
};
const recordClip = (stream: MediaStream, durationMs: number) => {
return new Promise<Blob>((resolve, reject) => {
const options: MediaRecorderOptions = {};
if (MediaRecorder.isTypeSupported('video/webm;codecs=vp9')) {
options.mimeType = 'video/webm;codecs=vp9';
} else if (MediaRecorder.isTypeSupported('video/webm;codecs=vp8')) {
options.mimeType = 'video/webm;codecs=vp8';
} else if (MediaRecorder.isTypeSupported('video/webm')) {
options.mimeType = 'video/webm';
}
const recorder = new MediaRecorder(stream, options);
const chunks: BlobPart[] = [];
recorder.ondataavailable = (event) => {
if (event.data.size > 0) {
chunks.push(event.data);
}
};
recorder.onerror = () => reject(new Error('Recording failed.'));
recorder.onstop = () => {
const blob = new Blob(chunks, { type: recorder.mimeType || 'video/webm' });
resolve(blob);
};
recorder.start();
window.setTimeout(() => {
if (recorder.state !== 'inactive') {
recorder.stop();
}
}, durationMs);
});
};
const requestPrediction = async (blob: Blob, filename: string) => {
const form = new FormData();
form.append('file', blob, filename);
const resp = await fetch('/predict', { method: 'POST', body: form });
const responseText = await resp.text();
let data: unknown = null;
if (responseText) {
try {
data = JSON.parse(responseText);
} catch {
data = null;
}
}
if (!resp.ok) {
const errorMessage =
data && typeof data === 'object' && 'error' in data
? String((data as { error?: unknown }).error ?? '')
: responseText;
throw new Error(errorMessage || `Request failed (${resp.status})`);
}
if (!data || typeof data !== 'object') {
throw new Error('Backend returned an empty response.');
}
return data as PredictionResult;
};
const handleCopyPrediction = async () => {
const record = latestRecord;
if (!record) {
setNotice('No prediction available to copy.');
return;
}
const text = [
`Label: ${record.label}`,
`Confidence: ${(record.score * 100).toFixed(1)}%`,
`Processing Time: ${record.elapsed_ms} ms`,
`Source: ${record.source}`,
`Timestamp: ${formatDateTime(record.created_at)}`,
].join('\n');
try {
await navigator.clipboard.writeText(text);
setNotice('Prediction copied to clipboard.');
} catch {
setNotice('Failed to copy. Please allow clipboard access.');
}
};
const handleSaveCurrentResult = () => {
if (!latestRecord) {
setNotice('No prediction available to save.');
return;
}
setSaved((prev) => {
if (prev.some((item) => item.id === latestRecord.id)) {
return prev;
}
return [latestRecord, ...prev];
});
setNotice('Result saved.');
};
const hasSavedCurrent = latestRecord ? saved.some((item) => item.id === latestRecord.id) : false;
const handleExportResults = () => {
const merged = new Map<string, ResultRecord & { saved: boolean }>();
for (const item of history) {
merged.set(item.id, { ...item, saved: false });
}
for (const item of saved) {
const existing = merged.get(item.id);
if (existing) {
merged.set(item.id, { ...existing, saved: true });
} else {
merged.set(item.id, { ...item, saved: true });
}
}
const rows = Array.from(merged.values()).sort((a, b) =>
b.created_at.localeCompare(a.created_at),
);
if (rows.length === 0) {
setNotice('No results to export yet.');
return;
}
const headers = ['id', 'label', 'confidence', 'elapsed_ms', 'source', 'filename', 'created_at', 'saved'];
const csvLines = [
headers.join(','),
...rows.map((item) =>
[
item.id,
item.label,
item.score.toFixed(6),
item.elapsed_ms,
item.source,
item.filename,
item.created_at,
item.saved,
]
.map(escapeCsvValue)
.join(','),
),
];
const blob = new Blob([csvLines.join('\n')], { type: 'text/csv;charset=utf-8' });
const url = URL.createObjectURL(blob);
const link = document.createElement('a');
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
link.href = url;
link.download = `lumisign-results-${stamp}.csv`;
link.click();
URL.revokeObjectURL(url);
setNotice('Export complete.');
};
const handleStartTranslation = async () => {
if (isProcessing) return;
if (!videoFile && !cameraStream) {
setError('Please upload a video or enable the camera.');
return;
}
setIsProcessing(true);
setCurrentStep(1);
setPrediction(null);
setError(null);
try {
let blob: Blob;
let filename: string;
let source: ResultRecord['source'];
if (videoFile) {
blob = videoFile;
filename = videoFile.name;
source = 'upload';
} else if (cameraStream) {
setCurrentStep(2);
blob = await recordClip(cameraStream, RECORDING_MS);
filename = 'recording.webm';
source = 'camera';
} else {
setIsProcessing(false);
return;
}
setCurrentStep(3);
const result = await requestPrediction(blob, filename);
setPrediction(result);
const record: ResultRecord = {
...result,
id: createRecordId(),
source,
filename,
created_at: new Date().toISOString(),
};
setLatestRecord(record);
setHistory((prev) => [record, ...prev].slice(0, MAX_HISTORY));
setCurrentStep(4);
setActiveSection('new');
} catch (err) {
const message = err instanceof Error ? err.message : 'Prediction failed.';
setError(message);
setCurrentStep(0);
} finally {
setIsProcessing(false);
}
};
const handleReset = () => {
setIsProcessing(false);
setCurrentStep(0);
setVideoFile(null);
setPrediction(null);
setError(null);
stopCamera();
};
const analytics = useMemo(() => {
const total = history.length;
const averageConfidence =
total > 0 ? history.reduce((sum, item) => sum + item.score, 0) / total : 0;
const labelCounts = new Map<string, number>();
for (const item of history) {
labelCounts.set(item.label, (labelCounts.get(item.label) ?? 0) + 1);
}
const topLabels = Array.from(labelCounts.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, 5);
return {
total,
saved: saved.length,
averageConfidence,
topLabels,
};
}, [history, saved.length]);
const renderRecordList = (records: ResultRecord[], emptyText: string) => {
if (records.length === 0) {
return <div className="text-sm text-gray-500">{emptyText}</div>;
}
return (
<div className="space-y-2">
{records.slice(0, 20).map((item) => (
<button
key={item.id}
onClick={() => {
setPrediction(item);
setLatestRecord(item);
setCurrentStep(4);
setActiveSection('new');
}}
className="w-full rounded-lg border border-gray-200 bg-white px-3 py-2 text-left hover:bg-gray-50"
>
<div className="font-medium text-sm">{item.label}</div>
<div className="text-xs text-gray-500">
{(item.score * 100).toFixed(1)}% confidence • {item.elapsed_ms} ms •{' '}
{formatDateTime(item.created_at)}
</div>
</button>
))}
</div>
);
};
const renderActiveSection = () => {
switch (activeSection) {
case 'history':
return (
<div className="mb-6 rounded-lg border border-gray-200 bg-white p-6">
<div className="mb-3 flex items-center justify-between">
<h2 className="text-lg">History</h2>
<button
onClick={() => setHistory([])}
className="rounded border border-gray-300 px-3 py-1.5 text-xs hover:bg-gray-50"
>
Clear History
</button>
</div>
{renderRecordList(history, 'No translation history yet.')}
</div>
);
case 'saved':
return (
<div className="mb-6 rounded-lg border border-gray-200 bg-white p-6">
<div className="mb-3 flex items-center justify-between">
<h2 className="text-lg">Saved Results</h2>
<button
onClick={() => setSaved([])}
className="rounded border border-gray-300 px-3 py-1.5 text-xs hover:bg-gray-50"
>
Clear Saved
</button>
</div>
{renderRecordList(saved, 'No saved results yet.')}
</div>
);
case 'analytics':
return (
<div className="mb-6 rounded-lg border border-gray-200 bg-white p-6">
<h2 className="mb-3 text-lg">Analytics</h2>
<div className="grid grid-cols-1 gap-3 md:grid-cols-3">
<div className="rounded-lg border border-gray-200 p-3">
<div className="text-xs text-gray-500">Total Predictions</div>
<div className="text-xl">{analytics.total}</div>
</div>
<div className="rounded-lg border border-gray-200 p-3">
<div className="text-xs text-gray-500">Saved Results</div>
<div className="text-xl">{analytics.saved}</div>
</div>
<div className="rounded-lg border border-gray-200 p-3">
<div className="text-xs text-gray-500">Average Confidence</div>
<div className="text-xl">{(analytics.averageConfidence * 100).toFixed(1)}%</div>
</div>
</div>
<div className="mt-4 rounded-lg border border-gray-200 p-3">
<div className="mb-2 text-sm text-gray-600">Top Predicted Labels</div>
{analytics.topLabels.length === 0 ? (
<div className="text-sm text-gray-500">Run a few translations to see trends.</div>
) : (
<div className="space-y-1">
{analytics.topLabels.map(([label, count]) => (
<div key={label} className="text-sm">
{label}: {count}
</div>
))}
</div>
)}
</div>
</div>
);
case 'documentation':
return (
<div className="mb-6 rounded-lg border border-gray-200 bg-white p-6 text-sm text-gray-700">
<h2 className="mb-3 text-lg text-black">Documentation</h2>
<div>1. Start backend (`uvicorn`) and frontend (`npm run dev`).</div>
<div>2. Upload a sign video or record with camera.</div>
<div>3. Click Start Translation to send it to `/predict`.</div>
<div>4. Save useful outputs or export CSV for analysis.</div>
<div className="mt-3 rounded bg-gray-50 p-3 font-mono text-xs text-gray-600">
MODEL_DATASET=Dataset MODEL_TYPE=transformer MODEL_TRANSFORMER_SIZE=large
</div>
</div>
);
case 'help':
return (
<div className="mb-6 rounded-lg border border-gray-200 bg-white p-6 text-sm text-gray-700">
<h2 className="mb-3 text-lg text-black">Help Center</h2>
<div>1. If translation fails, confirm backend is reachable on the configured port.</div>
<div>2. If camera fails, allow browser camera permission and retry.</div>
<div>3. If no output appears, check backend logs for model or media errors.</div>
<div>4. Use Export Results to share reproducible prediction history.</div>
</div>
);
case 'settings':
return (
<div className="mb-6 rounded-lg border border-gray-200 bg-white p-6">
<h2 className="mb-4 text-lg">Settings</h2>
<div className="mb-4 text-sm text-gray-700">Language: English</div>
<div className="flex gap-2">
<button
onClick={() => setHistory([])}
className="rounded border border-gray-300 px-3 py-2 text-xs hover:bg-gray-50"
>
Clear History
</button>
<button
onClick={() => setSaved([])}
className="rounded border border-gray-300 px-3 py-2 text-xs hover:bg-gray-50"
>
Clear Saved
</button>
</div>
</div>
);
case 'new':
default:
return null;
}
};
return (
<div className="flex h-screen bg-gray-50">
<Sidebar
activeSection={activeSection}
historyCount={history.length}
savedCount={saved.length}
onReset={handleReset}
onSelectSection={setActiveSection}
/>
<main className="flex-1 overflow-auto">
<div className="mx-auto max-w-7xl p-8">
{/* Header */}
<div className="mb-8">
<div className="flex items-center justify-between">
<div>
<h1 className="mb-2 text-3xl">Indian Sign Language Translator</h1>
<p className="text-gray-600">Real-time ISL to English translation</p>
</div>
<div className="flex gap-3">
<div className="rounded-lg border border-gray-300 bg-white px-4 py-2 text-sm text-gray-700">
Language: English
</div>
<button
onClick={handleExportResults}
className="rounded-lg bg-black px-4 py-2 text-white hover:bg-gray-800"
>
Export Results
</button>
</div>
</div>
</div>
{notice && (
<div className="mb-6 rounded-lg border border-emerald-200 bg-emerald-50 px-4 py-3 text-sm text-emerald-700">
{notice}
</div>
)}
{renderActiveSection()}
{/* Video Input Section */}
<VideoInput
onVideoUpload={handleVideoUpload}
videoFile={videoFile}
isProcessing={isProcessing}
onToggleCamera={handleToggleCamera}
cameraStream={cameraStream}
previewUrl={previewUrl}
/>
{/* Processing Pipeline */}
<ProcessingPipeline currentStep={currentStep} isProcessing={isProcessing} />
{/* Keypoints Display */}
<KeypointsDisplay />
{/* Translation Output */}
<TranslationOutput
currentStep={currentStep}
isProcessing={isProcessing}
onStartTranslation={handleStartTranslation}
onReset={handleReset}
onCopy={handleCopyPrediction}
onSaveResult={handleSaveCurrentResult}
canStart={Boolean(videoFile || cameraStream)}
prediction={prediction}
hasSavedCurrent={hasSavedCurrent}
error={error}
/>
</div>
</main>
</div>
);
}