anthony01 commited on
Commit
ea15c76
·
1 Parent(s): 0bb5e70

update: enhance UI components and functionality, add model configuration, and update server proxy

Browse files
Dockerfile CHANGED
@@ -43,5 +43,12 @@ RUN pip install --no-cache-dir uvicorn python-multipart
43
  # Expose the default Hugging Face port
44
  EXPOSE 7860
45
 
 
 
 
 
 
 
 
46
  # Command to run your FastAPI application using Uvicorn
47
  CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
 
43
  # Expose the default Hugging Face port
44
  EXPOSE 7860
45
 
46
+ # Runtime model configuration (override these from HF Space Variables if needed)
47
+ ENV MODEL_DATASET=Dataset
48
+ ENV MODEL_TYPE=transformer
49
+ ENV MODEL_TRANSFORMER_SIZE=large
50
+ ENV MODEL_MAX_FRAME_LEN=169
51
+ ENV MODEL_CHECKPOINT=/app/transformer_large.pth
52
+
53
  # Command to run your FastAPI application using Uvicorn
54
  CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
label_maps/label_map_Dataset.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bank": 0, "court": 1, "storeorshop": 2}
label_maps/label_map_dataset.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bank": 0, "court": 1, "storeorshop": 2}
ui/src/app/App.tsx CHANGED
@@ -1,9 +1,9 @@
1
- import { useEffect, useState } from 'react';
2
  import { VideoInput } from './components/VideoInput';
3
  import { ProcessingPipeline } from './components/ProcessingPipeline';
4
  import { KeypointsDisplay } from './components/KeypointsDisplay';
5
  import { TranslationOutput } from './components/TranslationOutput';
6
- import { Sidebar } from './components/Sidebar';
7
 
8
  type PredictionResult = {
9
  label: string;
@@ -12,9 +12,62 @@ type PredictionResult = {
12
  uid?: string;
13
  };
14
 
 
 
 
 
 
 
 
15
  const RECORDING_MS = 2000;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
 
17
  export default function App() {
 
18
  const [isProcessing, setIsProcessing] = useState(false);
19
  const [currentStep, setCurrentStep] = useState(0);
20
  const [videoFile, setVideoFile] = useState<File | null>(null);
@@ -22,6 +75,10 @@ export default function App() {
22
  const [error, setError] = useState<string | null>(null);
23
  const [cameraStream, setCameraStream] = useState<MediaStream | null>(null);
24
  const [previewUrl, setPreviewUrl] = useState<string | null>(null);
 
 
 
 
25
 
26
  useEffect(() => {
27
  if (!videoFile) {
@@ -40,6 +97,20 @@ export default function App() {
40
  };
41
  }, [cameraStream]);
42
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  const stopCamera = () => {
44
  cameraStream?.getTracks().forEach((track) => track.stop());
45
  setCameraStream(null);
@@ -57,6 +128,7 @@ export default function App() {
57
  audio: false,
58
  });
59
  setCameraStream(stream);
 
60
  setVideoFile(null);
61
  setPrediction(null);
62
  setError(null);
@@ -71,6 +143,7 @@ export default function App() {
71
  if (cameraStream) {
72
  stopCamera();
73
  }
 
74
  setVideoFile(file);
75
  setPrediction(null);
76
  setError(null);
@@ -115,13 +188,121 @@ export default function App() {
115
  const form = new FormData();
116
  form.append('file', blob, filename);
117
  const resp = await fetch('/predict', { method: 'POST', body: form });
118
- const data = await resp.json();
 
 
 
 
 
 
 
 
 
119
  if (!resp.ok) {
120
- throw new Error(data?.error || `Request failed (${resp.status})`);
 
 
 
 
 
 
 
 
121
  }
 
122
  return data as PredictionResult;
123
  };
124
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
125
  const handleStartTranslation = async () => {
126
  if (isProcessing) return;
127
  if (!videoFile && !cameraStream) {
@@ -137,14 +318,17 @@ export default function App() {
137
  try {
138
  let blob: Blob;
139
  let filename: string;
 
140
 
141
  if (videoFile) {
142
  blob = videoFile;
143
  filename = videoFile.name;
 
144
  } else if (cameraStream) {
145
  setCurrentStep(2);
146
  blob = await recordClip(cameraStream, RECORDING_MS);
147
  filename = 'recording.webm';
 
148
  } else {
149
  setIsProcessing(false);
150
  return;
@@ -153,7 +337,17 @@ export default function App() {
153
  setCurrentStep(3);
154
  const result = await requestPrediction(blob, filename);
155
  setPrediction(result);
 
 
 
 
 
 
 
 
 
156
  setCurrentStep(4);
 
157
  } catch (err) {
158
  const message = err instanceof Error ? err.message : 'Prediction failed.';
159
  setError(message);
@@ -172,9 +366,178 @@ export default function App() {
172
  stopCamera();
173
  };
174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
  return (
176
  <div className="flex h-screen bg-gray-50">
177
- <Sidebar onReset={handleReset} />
 
 
 
 
 
 
178
 
179
  <main className="flex-1 overflow-auto">
180
  <div className="mx-auto max-w-7xl p-8">
@@ -186,16 +549,27 @@ export default function App() {
186
  <p className="text-gray-600">Real-time ISL to English translation</p>
187
  </div>
188
  <div className="flex gap-3">
189
- <button className="flex items-center gap-2 rounded-lg border border-gray-300 bg-white px-4 py-2 hover:bg-gray-50">
190
- <span>Language</span>
191
- </button>
192
- <button className="rounded-lg bg-black px-4 py-2 text-white hover:bg-gray-800">
 
 
 
193
  Export Results
194
  </button>
195
  </div>
196
  </div>
197
  </div>
198
 
 
 
 
 
 
 
 
 
199
  {/* Video Input Section */}
200
  <VideoInput
201
  onVideoUpload={handleVideoUpload}
@@ -218,8 +592,11 @@ export default function App() {
218
  isProcessing={isProcessing}
219
  onStartTranslation={handleStartTranslation}
220
  onReset={handleReset}
 
 
221
  canStart={Boolean(videoFile || cameraStream)}
222
  prediction={prediction}
 
223
  error={error}
224
  />
225
  </div>
 
1
+ import { useEffect, useMemo, useState } from 'react';
2
  import { VideoInput } from './components/VideoInput';
3
  import { ProcessingPipeline } from './components/ProcessingPipeline';
4
  import { KeypointsDisplay } from './components/KeypointsDisplay';
5
  import { TranslationOutput } from './components/TranslationOutput';
6
+ import { Sidebar, type SidebarSection } from './components/Sidebar';
7
 
8
  type PredictionResult = {
9
  label: string;
 
12
  uid?: string;
13
  };
14
 
15
+ type ResultRecord = PredictionResult & {
16
+ id: string;
17
+ source: 'upload' | 'camera';
18
+ filename: string;
19
+ created_at: string;
20
+ };
21
+
22
  const RECORDING_MS = 2000;
23
+ const MAX_HISTORY = 200;
24
+ const HISTORY_STORAGE_KEY = 'lumisign_history_v1';
25
+ const SAVED_STORAGE_KEY = 'lumisign_saved_v1';
26
+
27
+ function createRecordId() {
28
+ if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {
29
+ return crypto.randomUUID();
30
+ }
31
+ return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
32
+ }
33
+
34
+ function loadStoredRecords(key: string): ResultRecord[] {
35
+ try {
36
+ const raw = localStorage.getItem(key);
37
+ if (!raw) return [];
38
+ const parsed = JSON.parse(raw) as unknown;
39
+ if (!Array.isArray(parsed)) return [];
40
+ return parsed.filter((entry): entry is ResultRecord => {
41
+ return (
42
+ entry !== null &&
43
+ typeof entry === 'object' &&
44
+ typeof (entry as { id?: unknown }).id === 'string' &&
45
+ typeof (entry as { label?: unknown }).label === 'string' &&
46
+ typeof (entry as { score?: unknown }).score === 'number' &&
47
+ typeof (entry as { elapsed_ms?: unknown }).elapsed_ms === 'number'
48
+ );
49
+ });
50
+ } catch {
51
+ return [];
52
+ }
53
+ }
54
+
55
+ function formatDateTime(iso: string) {
56
+ const date = new Date(iso);
57
+ if (Number.isNaN(date.getTime())) return iso;
58
+ return date.toLocaleString();
59
+ }
60
+
61
+ function escapeCsvValue(value: string | number | boolean) {
62
+ const text = String(value);
63
+ if (text.includes('"') || text.includes(',') || text.includes('\n')) {
64
+ return `"${text.replace(/"/g, '""')}"`;
65
+ }
66
+ return text;
67
+ }
68
 
69
  export default function App() {
70
+ const [activeSection, setActiveSection] = useState<SidebarSection>('new');
71
  const [isProcessing, setIsProcessing] = useState(false);
72
  const [currentStep, setCurrentStep] = useState(0);
73
  const [videoFile, setVideoFile] = useState<File | null>(null);
 
75
  const [error, setError] = useState<string | null>(null);
76
  const [cameraStream, setCameraStream] = useState<MediaStream | null>(null);
77
  const [previewUrl, setPreviewUrl] = useState<string | null>(null);
78
+ const [history, setHistory] = useState<ResultRecord[]>(() => loadStoredRecords(HISTORY_STORAGE_KEY));
79
+ const [saved, setSaved] = useState<ResultRecord[]>(() => loadStoredRecords(SAVED_STORAGE_KEY));
80
+ const [latestRecord, setLatestRecord] = useState<ResultRecord | null>(null);
81
+ const [notice, setNotice] = useState<string | null>(null);
82
 
83
  useEffect(() => {
84
  if (!videoFile) {
 
97
  };
98
  }, [cameraStream]);
99
 
100
+ useEffect(() => {
101
+ localStorage.setItem(HISTORY_STORAGE_KEY, JSON.stringify(history));
102
+ }, [history]);
103
+
104
+ useEffect(() => {
105
+ localStorage.setItem(SAVED_STORAGE_KEY, JSON.stringify(saved));
106
+ }, [saved]);
107
+
108
+ useEffect(() => {
109
+ if (!notice) return;
110
+ const timeout = window.setTimeout(() => setNotice(null), 3000);
111
+ return () => window.clearTimeout(timeout);
112
+ }, [notice]);
113
+
114
  const stopCamera = () => {
115
  cameraStream?.getTracks().forEach((track) => track.stop());
116
  setCameraStream(null);
 
128
  audio: false,
129
  });
130
  setCameraStream(stream);
131
+ setActiveSection('new');
132
  setVideoFile(null);
133
  setPrediction(null);
134
  setError(null);
 
143
  if (cameraStream) {
144
  stopCamera();
145
  }
146
+ setActiveSection('new');
147
  setVideoFile(file);
148
  setPrediction(null);
149
  setError(null);
 
188
  const form = new FormData();
189
  form.append('file', blob, filename);
190
  const resp = await fetch('/predict', { method: 'POST', body: form });
191
+ const responseText = await resp.text();
192
+ let data: unknown = null;
193
+ if (responseText) {
194
+ try {
195
+ data = JSON.parse(responseText);
196
+ } catch {
197
+ data = null;
198
+ }
199
+ }
200
+
201
  if (!resp.ok) {
202
+ const errorMessage =
203
+ data && typeof data === 'object' && 'error' in data
204
+ ? String((data as { error?: unknown }).error ?? '')
205
+ : responseText;
206
+ throw new Error(errorMessage || `Request failed (${resp.status})`);
207
+ }
208
+
209
+ if (!data || typeof data !== 'object') {
210
+ throw new Error('Backend returned an empty response.');
211
  }
212
+
213
  return data as PredictionResult;
214
  };
215
 
216
+ const handleCopyPrediction = async () => {
217
+ const record = latestRecord;
218
+ if (!record) {
219
+ setNotice('No prediction available to copy.');
220
+ return;
221
+ }
222
+ const text = [
223
+ `Label: ${record.label}`,
224
+ `Confidence: ${(record.score * 100).toFixed(1)}%`,
225
+ `Processing Time: ${record.elapsed_ms} ms`,
226
+ `Source: ${record.source}`,
227
+ `Timestamp: ${formatDateTime(record.created_at)}`,
228
+ ].join('\n');
229
+ try {
230
+ await navigator.clipboard.writeText(text);
231
+ setNotice('Prediction copied to clipboard.');
232
+ } catch {
233
+ setNotice('Failed to copy. Please allow clipboard access.');
234
+ }
235
+ };
236
+
237
+ const handleSaveCurrentResult = () => {
238
+ if (!latestRecord) {
239
+ setNotice('No prediction available to save.');
240
+ return;
241
+ }
242
+ setSaved((prev) => {
243
+ if (prev.some((item) => item.id === latestRecord.id)) {
244
+ return prev;
245
+ }
246
+ return [latestRecord, ...prev];
247
+ });
248
+ setNotice('Result saved.');
249
+ };
250
+
251
+ const hasSavedCurrent = latestRecord ? saved.some((item) => item.id === latestRecord.id) : false;
252
+
253
+ const handleExportResults = () => {
254
+ const merged = new Map<string, ResultRecord & { saved: boolean }>();
255
+ for (const item of history) {
256
+ merged.set(item.id, { ...item, saved: false });
257
+ }
258
+ for (const item of saved) {
259
+ const existing = merged.get(item.id);
260
+ if (existing) {
261
+ merged.set(item.id, { ...existing, saved: true });
262
+ } else {
263
+ merged.set(item.id, { ...item, saved: true });
264
+ }
265
+ }
266
+
267
+ const rows = Array.from(merged.values()).sort((a, b) =>
268
+ b.created_at.localeCompare(a.created_at),
269
+ );
270
+
271
+ if (rows.length === 0) {
272
+ setNotice('No results to export yet.');
273
+ return;
274
+ }
275
+
276
+ const headers = ['id', 'label', 'confidence', 'elapsed_ms', 'source', 'filename', 'created_at', 'saved'];
277
+ const csvLines = [
278
+ headers.join(','),
279
+ ...rows.map((item) =>
280
+ [
281
+ item.id,
282
+ item.label,
283
+ item.score.toFixed(6),
284
+ item.elapsed_ms,
285
+ item.source,
286
+ item.filename,
287
+ item.created_at,
288
+ item.saved,
289
+ ]
290
+ .map(escapeCsvValue)
291
+ .join(','),
292
+ ),
293
+ ];
294
+
295
+ const blob = new Blob([csvLines.join('\n')], { type: 'text/csv;charset=utf-8' });
296
+ const url = URL.createObjectURL(blob);
297
+ const link = document.createElement('a');
298
+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');
299
+ link.href = url;
300
+ link.download = `lumisign-results-${stamp}.csv`;
301
+ link.click();
302
+ URL.revokeObjectURL(url);
303
+ setNotice('Export complete.');
304
+ };
305
+
306
  const handleStartTranslation = async () => {
307
  if (isProcessing) return;
308
  if (!videoFile && !cameraStream) {
 
318
  try {
319
  let blob: Blob;
320
  let filename: string;
321
+ let source: ResultRecord['source'];
322
 
323
  if (videoFile) {
324
  blob = videoFile;
325
  filename = videoFile.name;
326
+ source = 'upload';
327
  } else if (cameraStream) {
328
  setCurrentStep(2);
329
  blob = await recordClip(cameraStream, RECORDING_MS);
330
  filename = 'recording.webm';
331
+ source = 'camera';
332
  } else {
333
  setIsProcessing(false);
334
  return;
 
337
  setCurrentStep(3);
338
  const result = await requestPrediction(blob, filename);
339
  setPrediction(result);
340
+ const record: ResultRecord = {
341
+ ...result,
342
+ id: createRecordId(),
343
+ source,
344
+ filename,
345
+ created_at: new Date().toISOString(),
346
+ };
347
+ setLatestRecord(record);
348
+ setHistory((prev) => [record, ...prev].slice(0, MAX_HISTORY));
349
  setCurrentStep(4);
350
+ setActiveSection('new');
351
  } catch (err) {
352
  const message = err instanceof Error ? err.message : 'Prediction failed.';
353
  setError(message);
 
366
  stopCamera();
367
  };
368
 
369
+ const analytics = useMemo(() => {
370
+ const total = history.length;
371
+ const averageConfidence =
372
+ total > 0 ? history.reduce((sum, item) => sum + item.score, 0) / total : 0;
373
+ const labelCounts = new Map<string, number>();
374
+ for (const item of history) {
375
+ labelCounts.set(item.label, (labelCounts.get(item.label) ?? 0) + 1);
376
+ }
377
+ const topLabels = Array.from(labelCounts.entries())
378
+ .sort((a, b) => b[1] - a[1])
379
+ .slice(0, 5);
380
+ return {
381
+ total,
382
+ saved: saved.length,
383
+ averageConfidence,
384
+ topLabels,
385
+ };
386
+ }, [history, saved.length]);
387
+
388
+ const renderRecordList = (records: ResultRecord[], emptyText: string) => {
389
+ if (records.length === 0) {
390
+ return <div className="text-sm text-gray-500">{emptyText}</div>;
391
+ }
392
+ return (
393
+ <div className="space-y-2">
394
+ {records.slice(0, 20).map((item) => (
395
+ <button
396
+ key={item.id}
397
+ onClick={() => {
398
+ setPrediction(item);
399
+ setLatestRecord(item);
400
+ setCurrentStep(4);
401
+ setActiveSection('new');
402
+ }}
403
+ className="w-full rounded-lg border border-gray-200 bg-white px-3 py-2 text-left hover:bg-gray-50"
404
+ >
405
+ <div className="font-medium text-sm">{item.label}</div>
406
+ <div className="text-xs text-gray-500">
407
+ {(item.score * 100).toFixed(1)}% confidence • {item.elapsed_ms} ms •{' '}
408
+ {formatDateTime(item.created_at)}
409
+ </div>
410
+ </button>
411
+ ))}
412
+ </div>
413
+ );
414
+ };
415
+
416
+ const renderActiveSection = () => {
417
+ switch (activeSection) {
418
+ case 'history':
419
+ return (
420
+ <div className="mb-6 rounded-lg border border-gray-200 bg-white p-6">
421
+ <div className="mb-3 flex items-center justify-between">
422
+ <h2 className="text-lg">History</h2>
423
+ <button
424
+ onClick={() => setHistory([])}
425
+ className="rounded border border-gray-300 px-3 py-1.5 text-xs hover:bg-gray-50"
426
+ >
427
+ Clear History
428
+ </button>
429
+ </div>
430
+ {renderRecordList(history, 'No translation history yet.')}
431
+ </div>
432
+ );
433
+ case 'saved':
434
+ return (
435
+ <div className="mb-6 rounded-lg border border-gray-200 bg-white p-6">
436
+ <div className="mb-3 flex items-center justify-between">
437
+ <h2 className="text-lg">Saved Results</h2>
438
+ <button
439
+ onClick={() => setSaved([])}
440
+ className="rounded border border-gray-300 px-3 py-1.5 text-xs hover:bg-gray-50"
441
+ >
442
+ Clear Saved
443
+ </button>
444
+ </div>
445
+ {renderRecordList(saved, 'No saved results yet.')}
446
+ </div>
447
+ );
448
+ case 'analytics':
449
+ return (
450
+ <div className="mb-6 rounded-lg border border-gray-200 bg-white p-6">
451
+ <h2 className="mb-3 text-lg">Analytics</h2>
452
+ <div className="grid grid-cols-1 gap-3 md:grid-cols-3">
453
+ <div className="rounded-lg border border-gray-200 p-3">
454
+ <div className="text-xs text-gray-500">Total Predictions</div>
455
+ <div className="text-xl">{analytics.total}</div>
456
+ </div>
457
+ <div className="rounded-lg border border-gray-200 p-3">
458
+ <div className="text-xs text-gray-500">Saved Results</div>
459
+ <div className="text-xl">{analytics.saved}</div>
460
+ </div>
461
+ <div className="rounded-lg border border-gray-200 p-3">
462
+ <div className="text-xs text-gray-500">Average Confidence</div>
463
+ <div className="text-xl">{(analytics.averageConfidence * 100).toFixed(1)}%</div>
464
+ </div>
465
+ </div>
466
+ <div className="mt-4 rounded-lg border border-gray-200 p-3">
467
+ <div className="mb-2 text-sm text-gray-600">Top Predicted Labels</div>
468
+ {analytics.topLabels.length === 0 ? (
469
+ <div className="text-sm text-gray-500">Run a few translations to see trends.</div>
470
+ ) : (
471
+ <div className="space-y-1">
472
+ {analytics.topLabels.map(([label, count]) => (
473
+ <div key={label} className="text-sm">
474
+ {label}: {count}
475
+ </div>
476
+ ))}
477
+ </div>
478
+ )}
479
+ </div>
480
+ </div>
481
+ );
482
+ case 'documentation':
483
+ return (
484
+ <div className="mb-6 rounded-lg border border-gray-200 bg-white p-6 text-sm text-gray-700">
485
+ <h2 className="mb-3 text-lg text-black">Documentation</h2>
486
+ <div>1. Start backend (`uvicorn`) and frontend (`npm run dev`).</div>
487
+ <div>2. Upload a sign video or record with camera.</div>
488
+ <div>3. Click Start Translation to send it to `/predict`.</div>
489
+ <div>4. Save useful outputs or export CSV for analysis.</div>
490
+ <div className="mt-3 rounded bg-gray-50 p-3 font-mono text-xs text-gray-600">
491
+ MODEL_DATASET=Dataset MODEL_TYPE=transformer MODEL_TRANSFORMER_SIZE=large
492
+ </div>
493
+ </div>
494
+ );
495
+ case 'help':
496
+ return (
497
+ <div className="mb-6 rounded-lg border border-gray-200 bg-white p-6 text-sm text-gray-700">
498
+ <h2 className="mb-3 text-lg text-black">Help Center</h2>
499
+ <div>1. If translation fails, confirm backend is reachable on the configured port.</div>
500
+ <div>2. If camera fails, allow browser camera permission and retry.</div>
501
+ <div>3. If no output appears, check backend logs for model or media errors.</div>
502
+ <div>4. Use Export Results to share reproducible prediction history.</div>
503
+ </div>
504
+ );
505
+ case 'settings':
506
+ return (
507
+ <div className="mb-6 rounded-lg border border-gray-200 bg-white p-6">
508
+ <h2 className="mb-4 text-lg">Settings</h2>
509
+ <div className="mb-4 text-sm text-gray-700">Language: English</div>
510
+ <div className="flex gap-2">
511
+ <button
512
+ onClick={() => setHistory([])}
513
+ className="rounded border border-gray-300 px-3 py-2 text-xs hover:bg-gray-50"
514
+ >
515
+ Clear History
516
+ </button>
517
+ <button
518
+ onClick={() => setSaved([])}
519
+ className="rounded border border-gray-300 px-3 py-2 text-xs hover:bg-gray-50"
520
+ >
521
+ Clear Saved
522
+ </button>
523
+ </div>
524
+ </div>
525
+ );
526
+ case 'new':
527
+ default:
528
+ return null;
529
+ }
530
+ };
531
+
532
  return (
533
  <div className="flex h-screen bg-gray-50">
534
+ <Sidebar
535
+ activeSection={activeSection}
536
+ historyCount={history.length}
537
+ savedCount={saved.length}
538
+ onReset={handleReset}
539
+ onSelectSection={setActiveSection}
540
+ />
541
 
542
  <main className="flex-1 overflow-auto">
543
  <div className="mx-auto max-w-7xl p-8">
 
549
  <p className="text-gray-600">Real-time ISL to English translation</p>
550
  </div>
551
  <div className="flex gap-3">
552
+ <div className="rounded-lg border border-gray-300 bg-white px-4 py-2 text-sm text-gray-700">
553
+ Language: English
554
+ </div>
555
+ <button
556
+ onClick={handleExportResults}
557
+ className="rounded-lg bg-black px-4 py-2 text-white hover:bg-gray-800"
558
+ >
559
  Export Results
560
  </button>
561
  </div>
562
  </div>
563
  </div>
564
 
565
+ {notice && (
566
+ <div className="mb-6 rounded-lg border border-emerald-200 bg-emerald-50 px-4 py-3 text-sm text-emerald-700">
567
+ {notice}
568
+ </div>
569
+ )}
570
+
571
+ {renderActiveSection()}
572
+
573
  {/* Video Input Section */}
574
  <VideoInput
575
  onVideoUpload={handleVideoUpload}
 
592
  isProcessing={isProcessing}
593
  onStartTranslation={handleStartTranslation}
594
  onReset={handleReset}
595
+ onCopy={handleCopyPrediction}
596
+ onSaveResult={handleSaveCurrentResult}
597
  canStart={Boolean(videoFile || cameraStream)}
598
  prediction={prediction}
599
+ hasSavedCurrent={hasSavedCurrent}
600
  error={error}
601
  />
602
  </div>
ui/src/app/components/KeypointsDisplay.tsx CHANGED
@@ -1,3 +1,4 @@
 
1
  import { Hand, MoreHorizontal, Smile, User } from "lucide-react";
2
 
3
  const keypointTypes = [
@@ -7,6 +8,12 @@ const keypointTypes = [
7
  ];
8
 
9
  export function KeypointsDisplay() {
 
 
 
 
 
 
10
  return (
11
  <div className="mb-6 rounded-xl border border-gray-200 bg-white p-4 sm:p-5">
12
  <div className="mb-4 flex items-center justify-between">
@@ -14,9 +21,12 @@ export function KeypointsDisplay() {
14
  <h2 className="mb-1 text-[18px]">Extracted Keypoints</h2>
15
  <p className="text-xs text-gray-600">Visual representation of detected landmarks</p>
16
  </div>
17
- <button className="flex items-center gap-1 rounded-md border border-gray-200 px-2.5 py-1.5 text-xs text-gray-600 hover:bg-gray-50">
 
 
 
18
  <MoreHorizontal className="h-3.5 w-3.5" />
19
- <span>View Details</span>
20
  </button>
21
  </div>
22
 
@@ -32,11 +42,23 @@ export function KeypointsDisplay() {
32
  <div className="border-t border-gray-200 px-3 py-2 text-center">
33
  <p className="text-xs text-gray-700">{type.title}</p>
34
  <p className="text-xs text-orange-700">{type.count} keypoints detected</p>
 
 
 
 
 
35
  </div>
36
  </div>
37
  );
38
  })}
39
  </div>
 
 
 
 
 
 
 
40
  </div>
41
  );
42
  }
 
1
+ import { useMemo, useState } from 'react';
2
  import { Hand, MoreHorizontal, Smile, User } from "lucide-react";
3
 
4
  const keypointTypes = [
 
8
  ];
9
 
10
  export function KeypointsDisplay() {
11
+ const [showDetails, setShowDetails] = useState(false);
12
+ const totalLandmarks = useMemo(
13
+ () => keypointTypes.reduce((sum, type) => sum + type.count, 0),
14
+ [],
15
+ );
16
+
17
  return (
18
  <div className="mb-6 rounded-xl border border-gray-200 bg-white p-4 sm:p-5">
19
  <div className="mb-4 flex items-center justify-between">
 
21
  <h2 className="mb-1 text-[18px]">Extracted Keypoints</h2>
22
  <p className="text-xs text-gray-600">Visual representation of detected landmarks</p>
23
  </div>
24
+ <button
25
+ onClick={() => setShowDetails((prev) => !prev)}
26
+ className="flex items-center gap-1 rounded-md border border-gray-200 px-2.5 py-1.5 text-xs text-gray-600 hover:bg-gray-50"
27
+ >
28
  <MoreHorizontal className="h-3.5 w-3.5" />
29
+ <span>{showDetails ? 'Hide Details' : 'View Details'}</span>
30
  </button>
31
  </div>
32
 
 
42
  <div className="border-t border-gray-200 px-3 py-2 text-center">
43
  <p className="text-xs text-gray-700">{type.title}</p>
44
  <p className="text-xs text-orange-700">{type.count} keypoints detected</p>
45
+ {showDetails && (
46
+ <p className="mt-1 text-[11px] text-gray-500">
47
+ Used as temporal features by the transformer classifier.
48
+ </p>
49
+ )}
50
  </div>
51
  </div>
52
  );
53
  })}
54
  </div>
55
+ {showDetails && (
56
+ <div className="mt-4 rounded-md border border-gray-200 bg-gray-50 p-3 text-xs text-gray-600">
57
+ <div>Total landmarks per frame: {totalLandmarks}</div>
58
+ <div>Face landmarks are available but less influential than pose and hands for many signs.</div>
59
+ <div>Final prediction uses a sequence-level transformer over extracted keypoint vectors.</div>
60
+ </div>
61
+ )}
62
  </div>
63
  );
64
  }
ui/src/app/components/ProcessingPipeline.tsx CHANGED
@@ -19,8 +19,8 @@ export function ProcessingPipeline({ currentStep, isProcessing }: ProcessingPipe
19
  },
20
  {
21
  id: 3,
22
- title: 'Step 3: LSTM Translation',
23
- description: 'Processing through neural network model',
24
  },
25
  {
26
  id: 4,
@@ -30,6 +30,11 @@ export function ProcessingPipeline({ currentStep, isProcessing }: ProcessingPipe
30
  ];
31
 
32
  const getStepStatus = (stepId: number) => {
 
 
 
 
 
33
  if (currentStep > stepId) return 'completed';
34
  if (currentStep === stepId) return 'processing';
35
  return 'pending';
@@ -97,10 +102,16 @@ export function ProcessingPipeline({ currentStep, isProcessing }: ProcessingPipe
97
  <div className="mt-3">
98
  <div className="flex items-center justify-between mb-2">
99
  <span className="text-sm text-gray-600">Progress</span>
100
- <span className="text-sm font-medium">{Math.min(currentStep * 25, 100)}%</span>
 
 
101
  </div>
102
  <Progress
103
- value={Math.min(currentStep * 25, 100)}
 
 
 
 
104
  className="h-2"
105
  />
106
  </div>
 
19
  },
20
  {
21
  id: 3,
22
+ title: 'Step 3: Transformer Translation',
23
+ description: 'Sequence classification using the trained transformer model',
24
  },
25
  {
26
  id: 4,
 
30
  ];
31
 
32
  const getStepStatus = (stepId: number) => {
33
+ if (stepId === 4) {
34
+ if (currentStep >= 4) return 'completed';
35
+ if (currentStep > 0) return 'processing';
36
+ return 'pending';
37
+ }
38
  if (currentStep > stepId) return 'completed';
39
  if (currentStep === stepId) return 'processing';
40
  return 'pending';
 
102
  <div className="mt-3">
103
  <div className="flex items-center justify-between mb-2">
104
  <span className="text-sm text-gray-600">Progress</span>
105
+ <span className="text-sm font-medium">
106
+ {getStepStatus(step.id) === 'completed' ? 100 : Math.min(currentStep * 25, 100)}%
107
+ </span>
108
  </div>
109
  <Progress
110
+ value={
111
+ getStepStatus(step.id) === 'completed'
112
+ ? 100
113
+ : Math.min(currentStep * 25, 100)
114
+ }
115
  className="h-2"
116
  />
117
  </div>
ui/src/app/components/Sidebar.tsx CHANGED
@@ -1,8 +1,34 @@
 
 
 
 
 
 
 
 
 
1
  interface SidebarProps {
 
 
 
2
  onReset: () => void;
 
3
  }
4
 
5
- export function Sidebar({ onReset }: SidebarProps) {
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  return (
7
  <aside className="w-64 bg-white border-r border-gray-200 p-6 flex flex-col">
8
  {/* Logo */}
@@ -16,25 +42,43 @@ export function Sidebar({ onReset }: SidebarProps) {
16
  {/* Navigation */}
17
  <nav className="flex-1">
18
  <div className="space-y-2">
19
- <button
20
- onClick={onReset}
21
- className="w-full flex items-center gap-3 px-4 py-3 rounded-lg bg-gray-100 hover:bg-gray-200 text-left"
 
 
 
22
  >
23
  <span>📹</span>
24
  <span>New Translation</span>
25
  </button>
26
-
27
- <button className="w-full flex items-center gap-3 px-4 py-3 rounded-lg hover:bg-gray-100 text-left text-gray-600">
 
 
 
28
  <span>🕐</span>
29
- <span>History</span>
 
 
 
30
  </button>
31
-
32
- <button className="w-full flex items-center gap-3 px-4 py-3 rounded-lg hover:bg-gray-100 text-left text-gray-600">
 
 
 
33
  <span>🔖</span>
34
- <span>Saved</span>
 
 
 
35
  </button>
36
-
37
- <button className="w-full flex items-center gap-3 px-4 py-3 rounded-lg hover:bg-gray-100 text-left text-gray-600">
 
 
 
38
  <span>📊</span>
39
  <span>Analytics</span>
40
  </button>
@@ -44,17 +88,26 @@ export function Sidebar({ onReset }: SidebarProps) {
44
  <div className="mt-8">
45
  <div className="text-xs uppercase text-gray-400 mb-3 px-4">Resources</div>
46
  <div className="space-y-2">
47
- <button className="w-full flex items-center gap-3 px-4 py-3 rounded-lg hover:bg-gray-100 text-left text-gray-600">
 
 
 
48
  <span>📚</span>
49
  <span>Documentation</span>
50
  </button>
51
-
52
- <button className="w-full flex items-center gap-3 px-4 py-3 rounded-lg hover:bg-gray-100 text-left text-gray-600">
 
 
 
53
  <span>❓</span>
54
  <span>Help Center</span>
55
  </button>
56
-
57
- <button className="w-full flex items-center gap-3 px-4 py-3 rounded-lg hover:bg-gray-100 text-left text-gray-600">
 
 
 
58
  <span>⚙️</span>
59
  <span>Settings</span>
60
  </button>
@@ -69,10 +122,16 @@ export function Sidebar({ onReset }: SidebarProps) {
69
  <span>👤</span>
70
  </div>
71
  <div className="flex-1">
72
- <div className="font-medium">Priya Sharma</div>
73
- <div className="text-sm text-gray-500">priya@example.com</div>
74
  </div>
75
- <button className="text-gray-400 hover:text-gray-600">⋮</button>
 
 
 
 
 
 
76
  </div>
77
  </div>
78
  </aside>
 
1
+ export type SidebarSection =
2
+ | 'new'
3
+ | 'history'
4
+ | 'saved'
5
+ | 'analytics'
6
+ | 'documentation'
7
+ | 'help'
8
+ | 'settings';
9
+
10
  interface SidebarProps {
11
+ activeSection: SidebarSection;
12
+ historyCount: number;
13
+ savedCount: number;
14
  onReset: () => void;
15
+ onSelectSection: (section: SidebarSection) => void;
16
  }
17
 
18
+ export function Sidebar({
19
+ activeSection,
20
+ historyCount,
21
+ savedCount,
22
+ onReset,
23
+ onSelectSection,
24
+ }: SidebarProps) {
25
+ const navButtonClass = (section: SidebarSection) =>
26
+ `w-full flex items-center gap-3 px-4 py-3 rounded-lg text-left ${
27
+ activeSection === section
28
+ ? 'bg-gray-100 text-gray-900'
29
+ : 'text-gray-600 hover:bg-gray-100'
30
+ }`;
31
+
32
  return (
33
  <aside className="w-64 bg-white border-r border-gray-200 p-6 flex flex-col">
34
  {/* Logo */}
 
42
  {/* Navigation */}
43
  <nav className="flex-1">
44
  <div className="space-y-2">
45
+ <button
46
+ onClick={() => {
47
+ onReset();
48
+ onSelectSection('new');
49
+ }}
50
+ className={navButtonClass('new')}
51
  >
52
  <span>📹</span>
53
  <span>New Translation</span>
54
  </button>
55
+
56
+ <button
57
+ onClick={() => onSelectSection('history')}
58
+ className={navButtonClass('history')}
59
+ >
60
  <span>🕐</span>
61
+ <span className="flex-1">History</span>
62
+ <span className="rounded bg-gray-200 px-2 py-0.5 text-xs text-gray-700">
63
+ {historyCount}
64
+ </span>
65
  </button>
66
+
67
+ <button
68
+ onClick={() => onSelectSection('saved')}
69
+ className={navButtonClass('saved')}
70
+ >
71
  <span>🔖</span>
72
+ <span className="flex-1">Saved</span>
73
+ <span className="rounded bg-gray-200 px-2 py-0.5 text-xs text-gray-700">
74
+ {savedCount}
75
+ </span>
76
  </button>
77
+
78
+ <button
79
+ onClick={() => onSelectSection('analytics')}
80
+ className={navButtonClass('analytics')}
81
+ >
82
  <span>📊</span>
83
  <span>Analytics</span>
84
  </button>
 
88
  <div className="mt-8">
89
  <div className="text-xs uppercase text-gray-400 mb-3 px-4">Resources</div>
90
  <div className="space-y-2">
91
+ <button
92
+ onClick={() => onSelectSection('documentation')}
93
+ className={navButtonClass('documentation')}
94
+ >
95
  <span>📚</span>
96
  <span>Documentation</span>
97
  </button>
98
+
99
+ <button
100
+ onClick={() => onSelectSection('help')}
101
+ className={navButtonClass('help')}
102
+ >
103
  <span>❓</span>
104
  <span>Help Center</span>
105
  </button>
106
+
107
+ <button
108
+ onClick={() => onSelectSection('settings')}
109
+ className={navButtonClass('settings')}
110
+ >
111
  <span>⚙️</span>
112
  <span>Settings</span>
113
  </button>
 
122
  <span>👤</span>
123
  </div>
124
  <div className="flex-1">
125
+ <div className="font-medium">Local Session</div>
126
+ <div className="text-sm text-gray-500">Stored in this browser</div>
127
  </div>
128
+ <button
129
+ onClick={() => onSelectSection('settings')}
130
+ className="text-gray-400 hover:text-gray-600"
131
+ aria-label="Open settings"
132
+ >
133
+
134
+ </button>
135
  </div>
136
  </div>
137
  </aside>
ui/src/app/components/TranslationOutput.tsx CHANGED
@@ -1,16 +1,19 @@
1
- import { Copy, Info, MessageSquareText, Play, RotateCcw, Save, Volume2 } from 'lucide-react';
2
 
3
  interface TranslationOutputProps {
4
  currentStep: number;
5
  isProcessing: boolean;
6
  onStartTranslation: () => void;
7
  onReset: () => void;
 
 
8
  canStart: boolean;
9
  prediction: {
10
  label: string;
11
  score: number;
12
  elapsed_ms: number;
13
  } | null;
 
14
  error: string | null;
15
  }
16
 
@@ -19,8 +22,11 @@ export function TranslationOutput({
19
  isProcessing,
20
  onStartTranslation,
21
  onReset,
 
 
22
  canStart,
23
  prediction,
 
24
  error,
25
  }: TranslationOutputProps) {
26
  const isComplete = Boolean(prediction) && !isProcessing;
@@ -45,14 +51,18 @@ export function TranslationOutput({
45
  <p className="text-sm text-gray-600">Translated English text with confidence score</p>
46
  </div>
47
  <div className="flex gap-2">
48
- <button className="flex items-center gap-1.5 rounded border border-gray-300 px-3 py-1.5 text-xs hover:bg-gray-50">
 
 
 
 
 
 
 
 
49
  <Copy className="h-3.5 w-3.5" />
50
  Copy
51
  </button>
52
- <button className="flex items-center gap-1.5 rounded border border-gray-300 px-3 py-1.5 text-xs hover:bg-gray-50">
53
- <Volume2 className="h-3.5 w-3.5" />
54
- Speak
55
- </button>
56
  </div>
57
  </div>
58
 
@@ -71,7 +81,7 @@ export function TranslationOutput({
71
  <div className="text-center">
72
  <MessageSquareText className="mx-auto mb-2 h-8 w-8 text-gray-300" />
73
  <div className="text-gray-500">Translation in progress...</div>
74
- <div className="text-xs text-gray-400">Processing through LSTM model</div>
75
  </div>
76
  ) : (
77
  <div className="text-center">
@@ -84,28 +94,37 @@ export function TranslationOutput({
84
 
85
  <div className="grid grid-cols-1 gap-4 md:grid-cols-3">
86
  <div className="rounded-lg border border-gray-200 p-4">
87
- <div className="mb-2 flex items-center justify-between">
88
- <div className="text-sm text-gray-600">Accuracy Score</div>
89
- <Info className="h-3.5 w-3.5 text-gray-400" />
 
 
 
 
 
90
  </div>
91
- <div className="text-2xl font-medium">{prediction ? `${confidencePercent}%` : '--%'}</div>
92
- </div>
93
 
94
  <div className="rounded-lg border border-gray-200 p-4">
95
- <div className="mb-2 flex items-center justify-between">
96
- <div className="text-sm text-gray-600">Confidence Level</div>
97
- <Info className="h-3.5 w-3.5 text-gray-400" />
 
 
 
 
 
98
  </div>
99
- <div className="text-2xl font-medium">{confidenceLevel}</div>
100
- </div>
101
 
102
  <div className="rounded-lg border border-gray-200 p-4">
103
- <div className="mb-2 flex items-center justify-between">
104
- <div className="text-sm text-gray-600">Processing Time</div>
105
- <Info className="h-3.5 w-3.5 text-gray-400" />
 
 
 
 
 
106
  </div>
107
- <div className="text-2xl font-medium">{processingTime}</div>
108
- </div>
109
  </div>
110
 
111
  <div className="mt-6 flex items-center justify-between border-t border-gray-200 pt-6">
@@ -119,6 +138,7 @@ export function TranslationOutput({
119
 
120
  <div className="flex gap-3">
121
  <button
 
122
  disabled={!isComplete}
123
  className={`flex items-center gap-2 rounded-lg border px-6 py-2.5 ${
124
  isComplete
@@ -127,7 +147,7 @@ export function TranslationOutput({
127
  }`}
128
  >
129
  <Save className="h-4 w-4" />
130
- Save Results
131
  </button>
132
 
133
  <button
 
1
+ import { Copy, Info, MessageSquareText, Play, RotateCcw, Save } from 'lucide-react';
2
 
3
  interface TranslationOutputProps {
4
  currentStep: number;
5
  isProcessing: boolean;
6
  onStartTranslation: () => void;
7
  onReset: () => void;
8
+ onCopy: () => void;
9
+ onSaveResult: () => void;
10
  canStart: boolean;
11
  prediction: {
12
  label: string;
13
  score: number;
14
  elapsed_ms: number;
15
  } | null;
16
+ hasSavedCurrent: boolean;
17
  error: string | null;
18
  }
19
 
 
22
  isProcessing,
23
  onStartTranslation,
24
  onReset,
25
+ onCopy,
26
+ onSaveResult,
27
  canStart,
28
  prediction,
29
+ hasSavedCurrent,
30
  error,
31
  }: TranslationOutputProps) {
32
  const isComplete = Boolean(prediction) && !isProcessing;
 
51
  <p className="text-sm text-gray-600">Translated English text with confidence score</p>
52
  </div>
53
  <div className="flex gap-2">
54
+ <button
55
+ onClick={onCopy}
56
+ disabled={!prediction}
57
+ className={`flex items-center gap-1.5 rounded border px-3 py-1.5 text-xs ${
58
+ prediction
59
+ ? 'border-gray-300 hover:bg-gray-50'
60
+ : 'cursor-not-allowed border-gray-200 bg-gray-100 text-gray-400'
61
+ }`}
62
+ >
63
  <Copy className="h-3.5 w-3.5" />
64
  Copy
65
  </button>
 
 
 
 
66
  </div>
67
  </div>
68
 
 
81
  <div className="text-center">
82
  <MessageSquareText className="mx-auto mb-2 h-8 w-8 text-gray-300" />
83
  <div className="text-gray-500">Translation in progress...</div>
84
+ <div className="text-xs text-gray-400">Processing through Transformer model</div>
85
  </div>
86
  ) : (
87
  <div className="text-center">
 
94
 
95
  <div className="grid grid-cols-1 gap-4 md:grid-cols-3">
96
  <div className="rounded-lg border border-gray-200 p-4">
97
+ <div className="mb-2 flex items-center justify-between">
98
+ <div className="text-sm text-gray-600">Accuracy Score</div>
99
+ <Info
100
+ className="h-3.5 w-3.5 text-gray-400"
101
+ title="Model confidence for the top predicted label"
102
+ />
103
+ </div>
104
+ <div className="text-2xl font-medium">{prediction ? `${confidencePercent}%` : '--%'}</div>
105
  </div>
 
 
106
 
107
  <div className="rounded-lg border border-gray-200 p-4">
108
+ <div className="mb-2 flex items-center justify-between">
109
+ <div className="text-sm text-gray-600">Confidence Level</div>
110
+ <Info
111
+ className="h-3.5 w-3.5 text-gray-400"
112
+ title="High: >=85%, Medium: >=60%, Low: <60%"
113
+ />
114
+ </div>
115
+ <div className="text-2xl font-medium">{confidenceLevel}</div>
116
  </div>
 
 
117
 
118
  <div className="rounded-lg border border-gray-200 p-4">
119
+ <div className="mb-2 flex items-center justify-between">
120
+ <div className="text-sm text-gray-600">Processing Time</div>
121
+ <Info
122
+ className="h-3.5 w-3.5 text-gray-400"
123
+ title="End-to-end backend response time for this prediction"
124
+ />
125
+ </div>
126
+ <div className="text-2xl font-medium">{processingTime}</div>
127
  </div>
 
 
128
  </div>
129
 
130
  <div className="mt-6 flex items-center justify-between border-t border-gray-200 pt-6">
 
138
 
139
  <div className="flex gap-3">
140
  <button
141
+ onClick={onSaveResult}
142
  disabled={!isComplete}
143
  className={`flex items-center gap-2 rounded-lg border px-6 py-2.5 ${
144
  isComplete
 
147
  }`}
148
  >
149
  <Save className="h-4 w-4" />
150
+ {hasSavedCurrent ? 'Saved' : 'Save Results'}
151
  </button>
152
 
153
  <button
ui/vite.config.ts CHANGED
@@ -13,7 +13,7 @@ export default defineConfig({
13
  server: {
14
  proxy: {
15
  '/predict': {
16
- target: 'http://localhost:8000',
17
  changeOrigin: true,
18
  },
19
  },
 
13
  server: {
14
  proxy: {
15
  '/predict': {
16
+ target: 'http://localhost:8080',
17
  changeOrigin: true,
18
  },
19
  },