ApurvaKondekar commited on
Commit
fd5c3ae
Β·
verified Β·
1 Parent(s): 83ae7f7
Files changed (1) hide show
  1. app.py +190 -1533
app.py CHANGED
@@ -5,12 +5,7 @@ import numpy as np
5
  import librosa
6
  import cv2
7
  import re
8
- from transformers import (
9
- Wav2Vec2Processor,
10
- Wav2Vec2Model,
11
- AutoTokenizer,
12
- AutoModel
13
- )
14
  from torchvision import models
15
  import tempfile
16
  import os
@@ -18,391 +13,187 @@ from huggingface_hub import hf_hub_download
18
  import whisper
19
  import subprocess
20
 
21
- # =========================================================
22
- # CONFIGURATION
23
- # =========================================================
24
-
25
  DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
26
-
27
  SAMPLE_RATE = 16000
28
-
29
  TEXT_MAX_LEN = 64
30
-
31
  LABELS = ["angry", "happy", "neutral", "sad"]
32
 
33
- gpu_status = (
34
- "GPU ONLINE"
35
- if torch.cuda.is_available()
36
- else "CPU MODE"
37
- )
38
-
39
- gpu_color = "#00ff88" if torch.cuda.is_available() else "#ff6b35"
40
-
41
- # =========================================================
42
- # LOAD PROCESSORS
43
- # =========================================================
44
-
45
- processor = Wav2Vec2Processor.from_pretrained(
46
- "facebook/wav2vec2-base-960h"
47
- )
48
-
49
- tokenizer = AutoTokenizer.from_pretrained(
50
- "bert-base-uncased"
51
- )
52
-
53
- # =========================================================
54
- # MODEL ARCHITECTURE
55
- # =========================================================
56
 
 
57
  class ResNetVideoEncoder(nn.Module):
58
-
59
  def __init__(self, out_dim=768):
60
-
61
  super().__init__()
62
-
63
  base = models.resnet18(pretrained=False)
64
-
65
- self.backbone = nn.Sequential(
66
- *list(base.children())[:-1]
67
- )
68
-
69
  self.proj = nn.Linear(512, out_dim)
70
 
71
  def forward(self, x):
72
-
73
  B, C, T, H, W = x.shape
74
-
75
  feats = []
76
-
77
  for t in range(T):
78
-
79
  ft = self.backbone(x[:, :, t])
80
-
81
- feats.append(
82
- ft.squeeze(-1).squeeze(-1)
83
- )
84
-
85
  feats = torch.stack(feats, dim=1).mean(1)
86
-
87
  return self.proj(feats)
88
 
89
- # =========================================================
90
-
91
  def mean_pool(x, mask):
92
-
93
  mask = mask[:, :x.size(1)]
94
-
95
  mask = mask.unsqueeze(-1).float()
96
-
97
- return (
98
- (x * mask).sum(1)
99
- / mask.sum(1).clamp(min=1e-6)
100
- )
101
-
102
- # =========================================================
103
 
104
  class HBF(nn.Module):
105
-
106
  def __init__(self, d=768, n_layers=6):
107
-
108
  super().__init__()
109
-
110
- self.proj_a = nn.ModuleList(
111
- [nn.Linear(d, d) for _ in range(n_layers)]
112
- )
113
-
114
- self.proj_t = nn.ModuleList(
115
- [nn.Linear(d, d) for _ in range(n_layers)]
116
- )
117
-
118
- self.proj_v = nn.ModuleList(
119
- [nn.Linear(d, d) for _ in range(n_layers)]
120
- )
121
-
122
- self.fwd1 = nn.ModuleList(
123
- [nn.Linear(3*d, d) for _ in range(n_layers)]
124
- )
125
-
126
- self.fwd2 = nn.ModuleList(
127
- [nn.Linear(d, d) for _ in range(n_layers)]
128
- )
129
-
130
  self.drop = nn.Dropout(0.1)
131
-
132
- self.act1 = nn.GELU()
133
-
134
- self.act2 = nn.Tanh()
135
-
136
  self.n = n_layers
137
 
138
  def forward(self, a, t, v):
139
-
140
  v_prev = None
141
-
142
  for i in range(self.n):
143
-
144
- va = self.act2(
145
- self.drop(self.proj_a[i](a))
146
- )
147
-
148
- vt = self.act2(
149
- self.drop(self.proj_t[i](t))
150
- )
151
-
152
- vv = self.act2(
153
- self.drop(self.proj_v[i](v))
154
- )
155
-
156
- cat = torch.cat(
157
- [va, vt, vv]
158
- if v_prev is None
159
- else [va, vt, v_prev],
160
- -1
161
- )
162
-
163
  x = self.act1(self.fwd1[i](cat))
164
-
165
  v_prev = self.fwd2[i](x)
166
-
167
  return v_prev
168
 
169
- # =========================================================
170
-
171
  class AVVideoModel(nn.Module):
172
-
173
  def __init__(self, num_classes, n_layers=6):
174
-
175
  super().__init__()
176
-
177
- self.a_enc = Wav2Vec2Model.from_pretrained(
178
- "facebook/wav2vec2-base-960h"
179
- )
180
-
181
- self.t_enc = AutoModel.from_pretrained(
182
- "bert-base-uncased"
183
- )
184
-
185
  self.v_enc = ResNetVideoEncoder()
186
-
187
  self.hbf = HBF(n_layers=n_layers)
188
-
189
  self.fc = nn.Linear(768, num_classes)
 
 
 
190
 
191
- def forward(
192
- self,
193
- audio,
194
- audio_mask,
195
- text_ids,
196
- text_mask,
197
- video
198
- ):
199
-
200
- a_out = self.a_enc(
201
- audio,
202
- attention_mask=audio_mask,
203
- return_dict=True
204
- )
205
-
206
- t_out = self.t_enc(
207
- input_ids=text_ids,
208
- attention_mask=text_mask,
209
- return_dict=True
210
- )
211
-
212
- a_pool = mean_pool(
213
- a_out.last_hidden_state,
214
- audio_mask
215
- )
216
-
217
- t_pool = mean_pool(
218
- t_out.last_hidden_state,
219
- text_mask
220
- )
221
-
222
  v_pool = self.v_enc(video)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
 
224
- fused = self.hbf(
225
- a_pool,
226
- t_pool,
227
- v_pool
228
- )
229
-
230
- logits = self.fc(fused)
231
-
232
- return logits
233
-
234
- # =========================================================
235
- # LOAD MODEL
236
- # =========================================================
237
 
238
- model = AVVideoModel(
239
- num_classes=len(LABELS)
240
- ).to(DEVICE)
241
 
 
242
  try:
243
-
244
  model_path = hf_hub_download(
245
- repo_id="ApurvaKondekar/emotion_model",
246
- filename="model_weights.pth"
247
- )
248
-
249
- model.load_state_dict(
250
- torch.load(model_path, map_location=DEVICE)
251
  )
252
-
253
  model.eval()
254
-
255
- print("Model loaded successfully")
256
-
257
  except Exception as e:
258
-
259
- print(f"Model loading failed: {e}")
260
-
261
- # =========================================================
262
- # VIDEO PROCESSING
263
- # =========================================================
264
-
265
- def extract_video_frames(
266
- video_path,
267
- max_frames=8,
268
- resize=(224, 224)
269
- ):
270
-
271
  cap = cv2.VideoCapture(video_path)
272
-
273
  if not cap.isOpened():
274
  return None
275
-
276
- total_frames = int(
277
- cap.get(cv2.CAP_PROP_FRAME_COUNT)
278
- )
279
-
280
- indices = np.linspace(
281
- 0,
282
- total_frames - 1,
283
- max_frames,
284
- dtype=int
285
- )
286
-
287
  frames = []
288
-
289
- for idx in indices:
290
-
291
- cap.set(cv2.CAP_PROP_POS_FRAMES, idx)
292
-
293
  ret, frame = cap.read()
294
-
295
  if not ret:
296
- continue
297
-
298
- frame = cv2.cvtColor(
299
- frame,
300
- cv2.COLOR_BGR2RGB
301
- )
302
-
303
  frame = cv2.resize(frame, resize)
304
-
305
  frames.append(frame)
306
-
307
  cap.release()
308
-
309
  if len(frames) == 0:
310
  return None
311
-
 
312
  while len(frames) < max_frames:
313
  frames.append(frames[-1])
314
-
315
- frames = np.array(frames[:max_frames])
316
-
317
  return frames
318
 
319
- # =========================================================
320
- # AUDIO EXTRACTION
321
- # =========================================================
322
-
323
  def extract_audio_from_video(video_path):
324
-
325
- audio_path = tempfile.NamedTemporaryFile(
326
- delete=False,
327
- suffix=".wav"
328
- ).name
329
-
330
- command = [
331
- "ffmpeg",
332
- "-i", video_path,
333
- "-vn",
334
- "-acodec", "pcm_s16le",
335
- "-ar", str(SAMPLE_RATE),
336
- "-ac", "1",
337
- "-y",
338
- audio_path
339
- ]
340
-
341
- result = subprocess.run(
342
- command,
343
- stdout=subprocess.PIPE,
344
- stderr=subprocess.PIPE
345
- )
346
-
347
- if result.returncode != 0:
348
- error_msg = result.stderr.decode("utf-8", errors="replace")
349
- print(f"[DEBUG] ffmpeg error: {error_msg}")
350
- raise RuntimeError(
351
- f"ffmpeg failed to extract audio: {error_msg[-200:]}"
352
- )
353
-
354
- if not os.path.exists(audio_path) or os.path.getsize(audio_path) == 0:
355
- raise RuntimeError(
356
- "ffmpeg produced no audio output β€” video may have no audio track"
357
- )
358
-
359
- return audio_path
360
-
361
- # =========================================================
362
- # WHISPER
363
- # =========================================================
364
-
365
- whisper_model = whisper.load_model("base")
366
 
367
  def transcribe_audio(audio_path):
 
 
 
 
 
 
 
368
 
369
- result = whisper_model.transcribe(audio_path)
370
-
371
- return result["text"].strip()
372
-
373
- # =========================================================
374
- # PREPROCESSING
375
- # =========================================================
376
-
377
- def preprocess_inputs(
378
- audio_path,
379
- text,
380
- video_path
381
- ):
382
-
383
- wav, _ = librosa.load(
384
- audio_path,
385
- sr=SAMPLE_RATE
386
- )
387
-
388
- audio_inputs = processor(
389
- wav,
390
- sampling_rate=SAMPLE_RATE,
391
- return_tensors="pt"
392
- )
393
 
 
 
 
 
 
 
394
  audio_values = audio_inputs.input_values.to(DEVICE)
395
-
396
- audio_mask = torch.ones_like(
397
- audio_values
398
- ).to(DEVICE)
399
-
400
- text_clean = re.sub(
401
- r"[^a-zA-Z0-9\s]",
402
- "",
403
- text.lower()
404
- )
405
-
406
  text_inputs = tokenizer(
407
  text_clean,
408
  truncation=True,
@@ -410,1244 +201,110 @@ def preprocess_inputs(
410
  max_length=TEXT_MAX_LEN,
411
  return_tensors="pt"
412
  )
413
-
414
  text_ids = text_inputs.input_ids.to(DEVICE)
415
-
416
  text_mask = text_inputs.attention_mask.to(DEVICE)
417
-
 
418
  frames = extract_video_frames(video_path)
419
-
420
- frames_tensor = (
421
- torch.tensor(frames)
422
- .permute(0, 3, 1, 2)
423
- .float() / 255.0
424
- )
425
-
426
- frames_tensor = (
427
- frames_tensor
428
- .unsqueeze(0)
429
- .permute(0, 2, 1, 3, 4)
430
- .to(DEVICE)
431
- )
432
-
433
- return (
434
- audio_values,
435
- audio_mask,
436
- text_ids,
437
- text_mask,
438
- frames_tensor
439
- )
440
-
441
- # =========================================================
442
- # PREDICTION
443
- # =========================================================
444
 
445
  def predict_emotion(video_file):
446
-
 
447
  if video_file is None:
448
-
449
- return (
450
- "⚠ NO INPUT DETECTED β€” UPLOAD A VIDEO FILE TO BEGIN ANALYSIS.",
451
- None,
452
- ""
453
- )
454
-
455
  try:
456
-
457
- # Handle both string paths and dict inputs from Gradio
458
- if isinstance(video_file, dict):
459
- video_path = video_file.get("video", video_file.get("name", None))
460
- else:
461
- video_path = video_file
462
-
463
- if video_path is None or not os.path.exists(video_path):
464
- return (
465
- "⚠ VIDEO FILE NOT FOUND β€” The recorded/uploaded file could not be located.",
466
- None,
467
- ""
468
- )
469
-
470
- print(f"[DEBUG] Processing video: {video_path}")
471
- print(f"[DEBUG] File exists: {os.path.exists(video_path)}")
472
- print(f"[DEBUG] File size: {os.path.getsize(video_path)} bytes")
473
-
474
- audio_path = extract_audio_from_video(
475
- video_path
476
- )
477
-
478
- transcribed_text = transcribe_audio(
479
- audio_path
480
- )
481
-
482
- (
483
- audio,
484
- audio_mask,
485
- text_ids,
486
- text_mask,
487
- video
488
- ) = preprocess_inputs(
489
- audio_path,
490
- transcribed_text,
491
- video_path
492
  )
493
-
 
494
  with torch.no_grad():
495
-
496
- logits = model(
497
- audio,
498
- audio_mask,
499
- text_ids,
500
- text_mask,
501
- video
502
  )
503
-
504
- probs = torch.softmax(
505
- logits,
506
- dim=1
507
- )[0].cpu().numpy()
508
-
509
- result = {
510
- LABELS[i]: float(probs[i])
511
- for i in range(len(LABELS))
512
- }
513
-
514
- predicted_emotion = LABELS[
515
- probs.argmax()
516
- ]
517
-
518
- confidence = float(probs.max())
519
-
520
- result_text = f"""
521
- ## EMOTION DETECTED
522
 
523
- ### `{predicted_emotion.upper()}`
 
524
 
525
- **Neural Confidence:** `{confidence:.2%}`
526
- """
527
-
 
 
 
 
 
 
528
  if os.path.exists(audio_path):
529
  os.remove(audio_path)
530
-
531
- return (
532
- result_text,
533
- result,
534
- transcribed_text
535
- )
536
-
537
  except Exception as e:
 
538
 
539
- return (
540
- f"⚠ SYSTEM ERROR: {str(e)}",
541
- None,
542
- ""
543
- )
544
-
545
- # =========================================================
546
- # FUTURISTIC CYBERPUNK CSS
547
- # =========================================================
548
-
549
- custom_css = """
550
- /* =====================================================
551
- IMPORT FONTS
552
- ===================================================== */
553
- @import url('https://fonts.googleapis.com/css2?family=Orbitron:wght@400;600;700;900&family=Share+Tech+Mono&family=Rajdhani:wght@300;400;600;700&display=swap');
554
-
555
- /* =====================================================
556
- CSS VARIABLES
557
- ===================================================== */
558
- :root {
559
- --neon-cyan: #00f5ff;
560
- --neon-green: #00ff88;
561
- --neon-pink: #ff006e;
562
- --neon-purple: #bf00ff;
563
- --neon-orange: #ff6b35;
564
- --dark-bg: #060e1a;
565
- --dark-panel: #0a1628;
566
- --dark-card: #0d1e35;
567
- --border-glow: rgba(0, 245, 255, 0.35);
568
- --text-primary: #cff4ff;
569
- --text-bright: #ffffff;
570
- --text-dim: #7eb8cc;
571
- --text-muted: #5a9ab0;
572
- --grid-color: rgba(0, 245, 255, 0.05);
573
- }
574
-
575
- /* =====================================================
576
- GLOBAL RESET & BASE
577
- ===================================================== */
578
- * {
579
- box-sizing: border-box;
580
- }
581
-
582
- body, .gradio-container {
583
- background-color: var(--dark-bg) !important;
584
- font-family: 'Rajdhani', sans-serif !important;
585
- color: var(--text-primary) !important;
586
- min-height: 100vh;
587
- position: relative;
588
- overflow-x: hidden;
589
- }
590
-
591
- /* Animated grid background */
592
- body::before {
593
- content: '';
594
- position: fixed;
595
- inset: 0;
596
- background-image:
597
- linear-gradient(var(--grid-color) 1px, transparent 1px),
598
- linear-gradient(90deg, var(--grid-color) 1px, transparent 1px);
599
- background-size: 40px 40px;
600
- z-index: 0;
601
- pointer-events: none;
602
- animation: gridDrift 20s linear infinite;
603
- }
604
-
605
- /* Scanline overlay */
606
- body::after {
607
- content: '';
608
- position: fixed;
609
- inset: 0;
610
- background: repeating-linear-gradient(
611
- 0deg,
612
- transparent,
613
- transparent 2px,
614
- rgba(0, 245, 255, 0.015) 2px,
615
- rgba(0, 245, 255, 0.015) 4px
616
- );
617
- pointer-events: none;
618
- z-index: 1;
619
- animation: scanlines 8s linear infinite;
620
- }
621
-
622
- @keyframes gridDrift {
623
- 0% { background-position: 0 0; }
624
- 100% { background-position: 40px 40px; }
625
- }
626
-
627
- @keyframes scanlines {
628
- 0% { background-position: 0 0; }
629
- 100% { background-position: 0 100px; }
630
- }
631
-
632
- /* =====================================================
633
- CONTAINER
634
- ===================================================== */
635
- .gradio-container {
636
- max-width: 1400px !important;
637
- margin: auto !important;
638
- padding: 30px 24px !important;
639
- position: relative;
640
- z-index: 2;
641
- }
642
-
643
- /* =====================================================
644
- HEADER
645
- ===================================================== */
646
- .cyber-header {
647
- text-align: center;
648
- padding: 40px 20px 20px;
649
- position: relative;
650
- }
651
-
652
- .cyber-title {
653
- font-family: 'Orbitron', monospace !important;
654
- font-size: clamp(28px, 5vw, 58px) !important;
655
- font-weight: 900 !important;
656
- letter-spacing: 6px !important;
657
- text-transform: uppercase !important;
658
- color: transparent !important;
659
- background: linear-gradient(
660
- 90deg,
661
- var(--neon-cyan) 0%,
662
- #ffffff 40%,
663
- var(--neon-purple) 70%,
664
- var(--neon-cyan) 100%
665
- ) !important;
666
- background-size: 200% auto !important;
667
- -webkit-background-clip: text !important;
668
- background-clip: text !important;
669
- animation: titleShimmer 4s linear infinite, fadeSlideDown 0.8s ease both;
670
- text-shadow: none !important;
671
- position: relative;
672
- }
673
-
674
- .cyber-title::after {
675
- content: attr(data-text);
676
- position: absolute;
677
- left: 0; right: 0;
678
- top: 0;
679
- color: var(--neon-cyan);
680
- filter: blur(18px);
681
- opacity: 0.35;
682
- animation: titlePulse 3s ease-in-out infinite;
683
- z-index: -1;
684
- }
685
-
686
- @keyframes titleShimmer {
687
- 0% { background-position: 0% center; }
688
- 100% { background-position: 200% center; }
689
- }
690
-
691
- @keyframes titlePulse {
692
- 0%, 100% { opacity: 0.25; }
693
- 50% { opacity: 0.5; }
694
- }
695
-
696
- @keyframes fadeSlideDown {
697
- from { opacity: 0; transform: translateY(-24px); }
698
- to { opacity: 1; transform: translateY(0); }
699
- }
700
-
701
- .cyber-subtitle {
702
- font-family: 'Share Tech Mono', monospace !important;
703
- font-size: 14px !important;
704
- color: var(--text-dim) !important;
705
- letter-spacing: 3px !important;
706
- text-transform: uppercase !important;
707
- margin-top: 10px !important;
708
- animation: fadeSlideDown 1s ease 0.3s both;
709
- }
710
-
711
- /* Decorative line under title */
712
- .cyber-divider {
713
- display: flex;
714
- align-items: center;
715
- gap: 12px;
716
- margin: 20px auto;
717
- max-width: 600px;
718
- animation: fadeSlideDown 1s ease 0.5s both;
719
- }
720
-
721
- .cyber-divider::before,
722
- .cyber-divider::after {
723
- content: '';
724
- flex: 1;
725
- height: 1px;
726
- background: linear-gradient(90deg, transparent, var(--neon-cyan), transparent);
727
- }
728
-
729
- .cyber-divider-dot {
730
- width: 6px; height: 6px;
731
- background: var(--neon-cyan);
732
- border-radius: 50%;
733
- box-shadow: 0 0 10px var(--neon-cyan);
734
- animation: dotPulse 2s ease-in-out infinite;
735
- }
736
-
737
- @keyframes dotPulse {
738
- 0%, 100% { transform: scale(1); opacity: 1; }
739
- 50% { transform: scale(1.6); opacity: 0.6; }
740
- }
741
-
742
- /* =====================================================
743
- STATUS BADGE
744
- ===================================================== */
745
- .status-bar {
746
- display: flex;
747
- justify-content: center;
748
- margin: 10px 0 24px;
749
- animation: fadeSlideDown 1s ease 0.6s both;
750
- }
751
-
752
- .status-badge {
753
- font-family: 'Share Tech Mono', monospace;
754
- font-size: 12px;
755
- letter-spacing: 2px;
756
- padding: 6px 20px;
757
- border: 1px solid var(--neon-green);
758
- color: var(--neon-green);
759
- background: rgba(0, 255, 136, 0.06);
760
- border-radius: 2px;
761
- position: relative;
762
- overflow: hidden;
763
- text-transform: uppercase;
764
- }
765
-
766
- .status-badge::before {
767
- content: '';
768
- position: absolute;
769
- top: 0; left: -100%;
770
- width: 100%; height: 100%;
771
- background: linear-gradient(90deg, transparent, rgba(0,255,136,0.15), transparent);
772
- animation: scanSweep 3s linear infinite;
773
- }
774
-
775
- @keyframes scanSweep {
776
- 0% { left: -100%; }
777
- 100% { left: 100%; }
778
- }
779
-
780
- /* =====================================================
781
- INFO CARDS (Modalities / Models)
782
- ===================================================== */
783
- .info-grid {
784
- display: grid;
785
- grid-template-columns: 1fr 1fr;
786
- gap: 16px;
787
- margin-bottom: 24px;
788
- animation: fadeSlideDown 1s ease 0.7s both;
789
- }
790
-
791
- .cyber-card {
792
- background: #0d1e35 !important;
793
- border: 1px solid rgba(0, 245, 255, 0.35) !important;
794
- border-radius: 6px !important;
795
- padding: 0 !important;
796
- position: relative;
797
- overflow: hidden;
798
- transition: border-color 0.3s, box-shadow 0.3s;
799
- }
800
-
801
- .cyber-card-purple {
802
- border-color: rgba(191, 0, 255, 0.35) !important;
803
- }
804
-
805
- /* Glowing left bar */
806
- .cyber-card::before {
807
- content: '';
808
- position: absolute;
809
- top: 0; left: 0;
810
- width: 3px; height: 100%;
811
- background: linear-gradient(180deg, #00f5ff, #bf00ff);
812
- box-shadow: 0 0 14px #00f5ff;
813
- z-index: 0;
814
- }
815
-
816
- .cyber-card-purple::before {
817
- background: linear-gradient(180deg, #bf00ff, #00f5ff);
818
- box-shadow: 0 0 14px #bf00ff;
819
- }
820
-
821
- /* Corner glow β€” behind text */
822
- .cyber-card::after {
823
- content: '';
824
- position: absolute;
825
- top: 0; right: 0;
826
- width: 80px; height: 80px;
827
- background: radial-gradient(circle, rgba(0,245,255,0.07) 0%, transparent 70%);
828
- z-index: 0;
829
- }
830
-
831
- /* Content wrapper sits above pseudo-elements */
832
- .card-inner {
833
- position: relative;
834
- z-index: 2;
835
- padding: 22px 22px 22px 28px;
836
- }
837
-
838
- .card-label {
839
- font-family: 'Orbitron', monospace;
840
- font-size: 10px;
841
- font-weight: 700;
842
- letter-spacing: 3px;
843
- text-transform: uppercase;
844
- margin-bottom: 16px;
845
- }
846
-
847
- .card-row {
848
- font-family: 'Share Tech Mono', monospace;
849
- font-size: 14px;
850
- color: #ffffff !important;
851
- line-height: 2.2;
852
- display: flex;
853
- align-items: center;
854
- gap: 10px;
855
- opacity: 1 !important;
856
- }
857
-
858
- .card-dot {
859
- font-size: 12px;
860
- flex-shrink: 0;
861
- }
862
-
863
- .cyber-card:hover {
864
- border-color: rgba(0, 245, 255, 0.6) !important;
865
- box-shadow: 0 0 35px rgba(0, 245, 255, 0.12) !important;
866
- }
867
-
868
- .cyber-card-purple:hover {
869
- border-color: rgba(191, 0, 255, 0.6) !important;
870
- box-shadow: 0 0 35px rgba(191, 0, 255, 0.12) !important;
871
- }
872
-
873
- /* =====================================================
874
- SYSTEM ARCHITECTURE STYLES
875
- ===================================================== */
876
- .arch-grid {
877
- display: grid;
878
- grid-template-columns: 3fr 2fr;
879
- gap: 24px;
880
- padding: 20px 4px 8px;
881
- }
882
-
883
- .arch-section {
884
- display: flex;
885
- flex-direction: column;
886
- gap: 14px;
887
- }
888
-
889
- .arch-title {
890
- font-family: 'Orbitron', monospace;
891
- font-size: 11px;
892
- font-weight: 700;
893
- letter-spacing: 3px;
894
- color: #00f5ff;
895
- text-transform: uppercase;
896
- border-bottom: 1px solid rgba(0,245,255,0.2);
897
- padding-bottom: 8px;
898
- margin-bottom: 4px;
899
- }
900
-
901
- .arch-desc {
902
- font-family: 'Rajdhani', sans-serif;
903
- font-size: 15px;
904
- color: #ffffff !important;
905
- line-height: 1.8;
906
- margin: 0 0 8px;
907
- }
908
-
909
- .arch-row {
910
- display: flex;
911
- align-items: flex-start;
912
- gap: 14px;
913
- }
914
-
915
- .arch-tag {
916
- font-family: 'Orbitron', monospace;
917
- font-size: 9px;
918
- font-weight: 700;
919
- letter-spacing: 1.5px;
920
- border: 1px solid;
921
- border-radius: 2px;
922
- padding: 4px 8px;
923
- white-space: nowrap;
924
- flex-shrink: 0;
925
- margin-top: 2px;
926
- }
927
-
928
- .arch-detail {
929
- font-family: 'Share Tech Mono', monospace;
930
- font-size: 13px;
931
- color: #ffffff !important;
932
- line-height: 1.8;
933
- opacity: 1 !important;
934
- }
935
-
936
- .emotion-chips {
937
- display: grid;
938
- grid-template-columns: 1fr 1fr;
939
- gap: 10px;
940
- margin-top: 4px;
941
- }
942
-
943
- .emotion-chip {
944
- font-family: 'Orbitron', monospace;
945
- font-size: 11px;
946
- font-weight: 700;
947
- letter-spacing: 2px;
948
- border: 1px solid;
949
- border-radius: 4px;
950
- padding: 12px 10px;
951
- text-align: center;
952
- background: rgba(255,255,255,0.02);
953
- transition: all 0.3s;
954
- }
955
-
956
- .emotion-chip:hover {
957
- background: rgba(255,255,255,0.06);
958
- transform: translateY(-2px);
959
- }
960
-
961
- /* =====================================================
962
- HIDE GRADIO "USE API" / BUILT WITH FOOTER
963
- ===================================================== */
964
- .built-with,
965
- a[href*="gradio"],
966
- footer.svelte-1ax1toq,
967
- .svelte-1ax1toq,
968
- div[class*="built"],
969
- .show-api,
970
- button[title*="API"],
971
- .api-btn {
972
- display: none !important;
973
- }
974
-
975
- /* =====================================================
976
- GR PANELS (override Gradio)
977
- ===================================================== */
978
- .gr-group,
979
- .gr-box,
980
- .gr-panel,
981
- .gr-form,
982
- div[class*="panel"],
983
- div[class*="block"] {
984
- background: #0a1628 !important;
985
- border: 1px solid rgba(0, 245, 255, 0.2) !important;
986
- border-radius: 4px !important;
987
- box-shadow: none !important;
988
- backdrop-filter: none !important;
989
- }
990
-
991
- /* =====================================================
992
- MARKDOWN HEADINGS
993
- ===================================================== */
994
- h1, h2, h3, h4 {
995
- font-family: 'Orbitron', monospace !important;
996
- font-weight: 700 !important;
997
- letter-spacing: 2px !important;
998
- text-transform: uppercase !important;
999
- }
1000
-
1001
- h2 {
1002
- font-size: 16px !important;
1003
- color: #00f5ff !important;
1004
- border-bottom: 1px solid rgba(0,245,255,0.3) !important;
1005
- padding-bottom: 8px !important;
1006
- margin-bottom: 16px !important;
1007
- text-shadow: 0 0 12px rgba(0,245,255,0.5) !important;
1008
- }
1009
-
1010
- h3 {
1011
- font-size: 15px !important;
1012
- color: #ffffff !important;
1013
- }
1014
-
1015
- h4 {
1016
- font-size: 13px !important;
1017
- color: #7eb8cc !important;
1018
- }
1019
-
1020
- /* =====================================================
1021
- LABELS / TEXT
1022
- ===================================================== */
1023
- label, .gr-label span, p, li {
1024
- font-family: 'Rajdhani', sans-serif !important;
1025
- color: #ffffff !important;
1026
- font-size: 15px !important;
1027
- font-weight: 600 !important;
1028
- letter-spacing: 1px !important;
1029
- }
1030
-
1031
- /* Gradio component labels */
1032
- .svelte-1ipelgc, span.svelte-1ipelgc,
1033
- div[class*="label"] > span,
1034
- .block > label > span {
1035
- color: var(--neon-cyan) !important;
1036
- font-family: 'Share Tech Mono', monospace !important;
1037
- font-size: 12px !important;
1038
- letter-spacing: 2px !important;
1039
- text-transform: uppercase !important;
1040
- opacity: 1 !important;
1041
- }
1042
-
1043
- /* =====================================================
1044
- ANALYZE BUTTON
1045
- ===================================================== */
1046
- .gr-button,
1047
- button[class*="primary"],
1048
- button {
1049
- font-family: 'Orbitron', monospace !important;
1050
- font-size: 13px !important;
1051
- font-weight: 700 !important;
1052
- letter-spacing: 3px !important;
1053
- text-transform: uppercase !important;
1054
- background: transparent !important;
1055
- border: 1px solid var(--neon-cyan) !important;
1056
- color: var(--neon-cyan) !important;
1057
- border-radius: 3px !important;
1058
- padding: 14px 32px !important;
1059
- position: relative !important;
1060
- overflow: hidden !important;
1061
- transition: all 0.3s ease !important;
1062
- cursor: pointer !important;
1063
- }
1064
-
1065
- .gr-button::before,
1066
- button::before {
1067
- content: '' !important;
1068
- position: absolute !important;
1069
- top: 0; left: -100% !important;
1070
- width: 100%; height: 100% !important;
1071
- background: linear-gradient(
1072
- 90deg,
1073
- transparent,
1074
- rgba(0, 245, 255, 0.2),
1075
- transparent
1076
- ) !important;
1077
- transition: left 0.5s ease !important;
1078
- }
1079
-
1080
- .gr-button:hover::before,
1081
- button:hover::before {
1082
- left: 100% !important;
1083
- }
1084
-
1085
- .gr-button:hover,
1086
- button:hover {
1087
- background: rgba(0, 245, 255, 0.1) !important;
1088
- box-shadow:
1089
- 0 0 20px rgba(0, 245, 255, 0.4),
1090
- 0 0 60px rgba(0, 245, 255, 0.15),
1091
- inset 0 0 20px rgba(0, 245, 255, 0.05) !important;
1092
- transform: translateY(-2px) !important;
1093
- }
1094
-
1095
- .gr-button:active,
1096
- button:active {
1097
- transform: translateY(0px) !important;
1098
- }
1099
-
1100
- /* =====================================================
1101
- INPUT / TEXTAREA
1102
- ===================================================== */
1103
- textarea,
1104
- input[type="text"],
1105
- input {
1106
- font-family: 'Share Tech Mono', monospace !important;
1107
- font-size: 13px !important;
1108
- background: #071020 !important;
1109
- border: 1px solid rgba(0, 245, 255, 0.25) !important;
1110
- border-radius: 3px !important;
1111
- color: #cff4ff !important;
1112
- transition: all 0.3s !important;
1113
- letter-spacing: 0.5px !important;
1114
- }
1115
-
1116
- textarea:focus,
1117
- input:focus {
1118
- border-color: var(--neon-cyan) !important;
1119
- box-shadow: 0 0 15px rgba(0, 245, 255, 0.25) !important;
1120
- outline: none !important;
1121
- }
1122
-
1123
- /* placeholder */
1124
- textarea::placeholder,
1125
- input::placeholder {
1126
- color: rgba(127, 184, 204, 0.45) !important;
1127
- }
1128
-
1129
- /* =====================================================
1130
- VIDEO COMPONENT
1131
- ===================================================== */
1132
- video {
1133
- border-radius: 3px !important;
1134
- border: 1px solid rgba(0, 245, 255, 0.2) !important;
1135
- box-shadow:
1136
- 0 0 30px rgba(0, 245, 255, 0.1),
1137
- inset 0 0 30px rgba(0, 0, 0, 0.5) !important;
1138
- }
1139
-
1140
- /* Upload zone */
1141
- .upload-zone,
1142
- div[class*="upload"] {
1143
- border: 1px dashed rgba(0, 245, 255, 0.3) !important;
1144
- background: rgba(0, 245, 255, 0.03) !important;
1145
- border-radius: 4px !important;
1146
- transition: all 0.3s !important;
1147
- }
1148
-
1149
- .upload-zone:hover,
1150
- div[class*="upload"]:hover {
1151
- border-color: var(--neon-cyan) !important;
1152
- background: rgba(0, 245, 255, 0.07) !important;
1153
- }
1154
-
1155
- /* =====================================================
1156
- REMOVE VIDEO EDIT CONTROLS
1157
- ===================================================== */
1158
-
1159
- /* remove trim button */
1160
- button[aria-label*="Trim"],
1161
- button[title*="Trim"],
1162
- button[aria-label*="trim"],
1163
- button[title*="trim"] {
1164
- display: none !important;
1165
- }
1166
-
1167
- /* remove reset button */
1168
- button[aria-label*="Reset"],
1169
- button[title*="Reset"],
1170
- button[aria-label*="reset"],
1171
- button[title*="reset"] {
1172
- display: none !important;
1173
- }
1174
-
1175
- /* remove edit tools section */
1176
- div[data-testid="video"] .controls,
1177
- div[data-testid="video"] [class*="controls"],
1178
- div[data-testid="video"] [class*="edit"] {
1179
- display: none !important;
1180
- }
1181
-
1182
- /* =====================================================
1183
- LABEL (EMOTION PROBABILITIES)
1184
- ===================================================== */
1185
- .gr-label,
1186
- div[class*="label"] {
1187
- background: var(--dark-panel) !important;
1188
- border: 1px solid rgba(0, 245, 255, 0.12) !important;
1189
- border-radius: 4px !important;
1190
- padding: 12px !important;
1191
- }
1192
-
1193
- /* Label bars */
1194
- div[class*="confidence"] span,
1195
- div[class*="bar"],
1196
- .label-bar,
1197
- [class*="Confidence"] {
1198
- background: linear-gradient(
1199
- 90deg,
1200
- rgba(0, 245, 255, 0.15),
1201
- rgba(0, 245, 255, 0.05)
1202
- ) !important;
1203
- border-left: 2px solid var(--neon-cyan) !important;
1204
- border-radius: 0 !important;
1205
- }
1206
-
1207
- /* =====================================================
1208
- ACCORDION
1209
- ===================================================== */
1210
- .gr-accordion,
1211
- details,
1212
- summary {
1213
- background: #0d1e35 !important;
1214
- border: 1px solid rgba(0,245,255,0.2) !important;
1215
- border-radius: 4px !important;
1216
- font-family: 'Orbitron', monospace !important;
1217
- font-size: 12px !important;
1218
- letter-spacing: 2px !important;
1219
- color: #7eb8cc !important;
1220
- text-transform: uppercase !important;
1221
- }
1222
-
1223
- details summary {
1224
- padding: 14px 20px !important;
1225
- cursor: pointer !important;
1226
- transition: color 0.3s !important;
1227
- color: #a0d4e8 !important;
1228
- }
1229
-
1230
- details summary:hover {
1231
- color: var(--neon-cyan) !important;
1232
- }
1233
-
1234
- details[open] summary {
1235
- color: var(--neon-cyan) !important;
1236
- border-bottom: 1px solid rgba(0,245,255,0.2) !important;
1237
- }
1238
-
1239
- /* =====================================================
1240
- RESULT TEXT (Markdown output)
1241
- ===================================================== */
1242
- .result-markdown code,
1243
- code {
1244
- font-family: 'Share Tech Mono', monospace !important;
1245
- background: rgba(0, 245, 255, 0.08) !important;
1246
- border: 1px solid rgba(0, 245, 255, 0.2) !important;
1247
- border-radius: 2px !important;
1248
- color: var(--neon-cyan) !important;
1249
- padding: 2px 8px !important;
1250
- font-size: 15px !important;
1251
- }
1252
-
1253
- /* =====================================================
1254
- SEPARATOR / HR
1255
- ===================================================== */
1256
- hr {
1257
- border: none !important;
1258
- height: 1px !important;
1259
- background: linear-gradient(
1260
- 90deg,
1261
- transparent,
1262
- rgba(0, 245, 255, 0.3),
1263
- rgba(191, 0, 255, 0.3),
1264
- transparent
1265
- ) !important;
1266
- margin: 24px 0 !important;
1267
- }
1268
-
1269
- /* =====================================================
1270
- FOOTER
1271
- ===================================================== */
1272
- .cyber-footer {
1273
- text-align: center;
1274
- font-family: 'Share Tech Mono', monospace;
1275
- font-size: 11px;
1276
- letter-spacing: 3px;
1277
- color: var(--text-dim);
1278
- margin-top: 40px;
1279
- padding: 20px;
1280
- text-transform: uppercase;
1281
- border-top: 1px solid rgba(0,245,255,0.08);
1282
- position: relative;
1283
- }
1284
-
1285
- .cyber-footer::before {
1286
- content: 'β—ˆ β—ˆ β—ˆ';
1287
- display: block;
1288
- color: rgba(0,245,255,0.2);
1289
- margin-bottom: 10px;
1290
- letter-spacing: 6px;
1291
- font-size: 8px;
1292
- animation: dotPulse 3s ease-in-out infinite;
1293
- }
1294
-
1295
- /* =====================================================
1296
- CORNER DECORATION (applied to main columns)
1297
- ===================================================== */
1298
- .corner-decor {
1299
- position: relative;
1300
- }
1301
-
1302
- .corner-decor::before,
1303
- .corner-decor::after {
1304
- content: '';
1305
- position: absolute;
1306
- width: 12px; height: 12px;
1307
- border-color: var(--neon-cyan);
1308
- border-style: solid;
1309
- opacity: 0.5;
1310
- }
1311
-
1312
- .corner-decor::before {
1313
- top: -1px; left: -1px;
1314
- border-width: 2px 0 0 2px;
1315
- }
1316
-
1317
- .corner-decor::after {
1318
- bottom: -1px; right: -1px;
1319
- border-width: 0 2px 2px 0;
1320
- }
1321
-
1322
- /* =====================================================
1323
- ANIMATED PARTICLES (pure CSS)
1324
- ===================================================== */
1325
- .particle-field {
1326
- position: fixed;
1327
- inset: 0;
1328
- pointer-events: none;
1329
- z-index: 0;
1330
- overflow: hidden;
1331
- }
1332
-
1333
- .particle {
1334
- position: absolute;
1335
- width: 2px; height: 2px;
1336
- background: var(--neon-cyan);
1337
- border-radius: 50%;
1338
- opacity: 0;
1339
- animation: floatParticle linear infinite;
1340
- }
1341
-
1342
- .particle:nth-child(1) { left: 10%; animation-duration: 12s; animation-delay: 0s; width: 1px; height: 1px; }
1343
- .particle:nth-child(2) { left: 20%; animation-duration: 18s; animation-delay: 2s; background: var(--neon-purple); }
1344
- .particle:nth-child(3) { left: 35%; animation-duration: 14s; animation-delay: 4s; }
1345
- .particle:nth-child(4) { left: 50%; animation-duration: 20s; animation-delay: 1s; background: var(--neon-green); width: 1px; height: 1px; }
1346
- .particle:nth-child(5) { left: 65%; animation-duration: 16s; animation-delay: 6s; }
1347
- .particle:nth-child(6) { left: 78%; animation-duration: 13s; animation-delay: 3s; background: var(--neon-pink); }
1348
- .particle:nth-child(7) { left: 88%; animation-duration: 19s; animation-delay: 5s; width: 1px; height: 1px; }
1349
- .particle:nth-child(8) { left: 45%; animation-duration: 15s; animation-delay: 7s; background: var(--neon-purple); }
1350
- .particle:nth-child(9) { left: 55%; animation-duration: 17s; animation-delay: 0.5s; }
1351
- .particle:nth-child(10) { left: 72%; animation-duration: 11s; animation-delay: 8s; background: var(--neon-green); width: 1px; height: 1px; }
1352
- .particle:nth-child(11) { left: 5%; animation-duration: 22s; animation-delay: 2.5s; background: var(--neon-cyan); }
1353
- .particle:nth-child(12) { left: 92%; animation-duration: 14s; animation-delay: 9s; background: var(--neon-pink); width: 1px; height: 1px; }
1354
-
1355
- @keyframes floatParticle {
1356
- 0% { bottom: -10px; opacity: 0; transform: translateX(0); }
1357
- 10% { opacity: 0.6; }
1358
- 90% { opacity: 0.3; }
1359
- 100% { bottom: 105vh; opacity: 0; transform: translateX(30px); }
1360
- }
1361
-
1362
- /* =====================================================
1363
- SCROLLBAR
1364
- ===================================================== */
1365
- ::-webkit-scrollbar { width: 6px; }
1366
- ::-webkit-scrollbar-track { background: var(--dark-bg); }
1367
- ::-webkit-scrollbar-thumb {
1368
- background: linear-gradient(180deg, var(--neon-cyan), var(--neon-purple));
1369
- border-radius: 3px;
1370
- }
1371
-
1372
- /* =====================================================
1373
- RESPONSIVE TWEAKS
1374
- ===================================================== */
1375
- @media (max-width: 768px) {
1376
- .info-grid { grid-template-columns: 1fr; }
1377
- .cyber-title { font-size: 26px !important; letter-spacing: 3px !important; }
1378
- }
1379
-
1380
- /* REMOVE GRADIO FOOTER + SETTINGS */
1381
-
1382
- footer,
1383
- .settings,
1384
- button[title="Settings"],
1385
- button[aria-label="Settings"],
1386
- .gradio-footer,
1387
- div[class*="settings"],
1388
- div[class*="footer"] {
1389
- display: none !important;
1390
- }
1391
-
1392
- /* =====================================================
1393
- CYBERPUNK VIDEO BUTTON FIX
1394
- ===================================================== */
1395
-
1396
- /* upload / record buttons */
1397
- div[data-testid="video"] button * {
1398
- color: #ffffff !important;
1399
- fill: #ffffff !important;
1400
- stroke: #ffffff !important;
1401
- opacity: 1 !important;
1402
- visibility: visible !important;
1403
- font-size: 16px !important;
1404
- font-weight: 700 !important;
1405
- }
1406
-
1407
-
1408
- /* =====================================================
1409
- CYBERPUNK VIDEO BUTTON FIX
1410
- ===================================================== */
1411
-
1412
- /* upload / record buttons */
1413
- div[data-testid="video"] button {
1414
- background: #071020 !important;
1415
- border: 2px solid #00f5ff !important;
1416
- border-radius: 6px !important;
1417
-
1418
- min-width: 130px !important;
1419
- min-height: 40px !important;
1420
-
1421
- display: flex !important;
1422
- align-items: center !important;
1423
- justify-content: center !important;
1424
- gap: 10px !important;
1425
-
1426
- padding: 10px 18px !important;
1427
-
1428
- color: #ffffff !important;
1429
- opacity: 1 !important;
1430
-
1431
- box-shadow: 0 0 12px rgba(0,245,255,0.25) !important;
1432
- }
1433
-
1434
- /* icon size */
1435
- div[data-testid="video"] button svg {
1436
- width: 25px !important;
1437
- height: 25px !important;
1438
-
1439
- stroke: #ffffff !important;
1440
- fill: #ffffff !important;
1441
-
1442
- opacity: 1 !important;
1443
- }
1444
-
1445
- /* text beside icon */
1446
- div[data-testid="video"] button span {
1447
- color: #ffffff !important;
1448
-
1449
- font-size: 16px !important;
1450
- font-weight: 700 !important;
1451
-
1452
- letter-spacing: 1px !important;
1453
-
1454
- opacity: 1 !important;
1455
- visibility: visible !important;
1456
-
1457
- display: inline !important;
1458
- }
1459
-
1460
- /* remove shiny animation overlay */
1461
- div[data-testid="video"] button::before {
1462
- display: none !important;
1463
- }
1464
-
1465
- /* black labels */
1466
- .black-label label,
1467
- .black-label span {
1468
- color: black !important;
1469
- }
1470
- """
1471
-
1472
- # =========================================================
1473
- # UI
1474
- # =========================================================
1475
-
1476
- with gr.Blocks(
1477
- title="Multimodal Emotion Recognition",
1478
- theme=gr.themes.Base(
1479
- primary_hue="cyan",
1480
- neutral_hue="slate",
1481
- font=[gr.themes.GoogleFont("Rajdhani"), "sans-serif"],
1482
- ),
1483
- css=custom_css
1484
- ) as demo:
1485
-
1486
- # ── Floating particles ──────────────────────────────
1487
- gr.HTML("""
1488
- <div class="particle-field">
1489
- <div class="particle"></div>
1490
- <div class="particle"></div>
1491
- <div class="particle"></div>
1492
- <div class="particle"></div>
1493
- <div class="particle"></div>
1494
- <div class="particle"></div>
1495
- <div class="particle"></div>
1496
- <div class="particle"></div>
1497
- <div class="particle"></div>
1498
- <div class="particle"></div>
1499
- <div class="particle"></div>
1500
- <div class="particle"></div>
1501
- </div>
1502
- """)
1503
-
1504
- # ── Header ──────────────────────────────────────────
1505
- gr.HTML("""
1506
- <div class="cyber-header">
1507
- <div class="cyber-title" data-text="MULTIMODAL EMOTION RECOGNITION">
1508
- MULTIMODAL EMOTION RECOGNITION
1509
- </div>
1510
- <div class="cyber-divider">
1511
- <div class="cyber-divider-dot"></div>
1512
- <div class="cyber-divider-dot" style="animation-delay:0.4s"></div>
1513
- <div class="cyber-divider-dot" style="animation-delay:0.8s"></div>
1514
- </div>
1515
- </div>
1516
- """)
1517
-
1518
- # ── Info Cards ──────────────────────────────────────
1519
- gr.HTML("""
1520
- <div class="info-grid">
1521
- <div class="cyber-card">
1522
- <div class="card-inner">
1523
- <div class="card-label" style="color:#00f5ff;">MODALITIES</div>
1524
- <div class="card-row"><span class="card-dot" style="color:#00f5ff;">β—ˆ</span>Audio Waveform Analysis</div>
1525
- <div class="card-row"><span class="card-dot" style="color:#00f5ff;">β—ˆ</span>Speech Transcription</div>
1526
- <div class="card-row"><span class="card-dot" style="color:#00f5ff;">β—ˆ</span>Facial Expression Mapping</div>
1527
- </div>
1528
- </div>
1529
- <div class="cyber-card cyber-card-purple">
1530
- <div class="card-inner">
1531
- <div class="card-label" style="color:#bf00ff;">NEURAL MODELS</div>
1532
- <div class="card-row"><span class="card-dot" style="color:#bf00ff;">β—ˆ</span>Wav2Vec2 Β· BERT Β· ResNet18</div>
1533
- <div class="card-row"><span class="card-dot" style="color:#bf00ff;">β—ˆ</span>Whisper ASR</div>
1534
- <div class="card-row"><span class="card-dot" style="color:#bf00ff;">β—ˆ</span>Hierarchical Bottleneck Fusion (HBF)</div>
1535
- </div>
1536
- </div>
1537
- </div>
1538
- """)
1539
-
1540
- gr.HTML('<hr/>')
1541
-
1542
- # ── Main Analysis Section ───────────────────────────
1543
- with gr.Row(equal_height=True):
1544
-
1545
- # LEFT: Upload
1546
- with gr.Column(scale=1):
1547
-
1548
- gr.Markdown("## INPUT STREAM")
1549
-
1550
- video_input = gr.Video(
1551
- label="Video Input",
1552
- height=360,
1553
- sources=["upload", "webcam"],
1554
- elem_classes="black-label"
1555
- )
1556
-
1557
- predict_btn = gr.Button(
1558
- "ANALYZE EMOTION",
1559
- variant="primary",
1560
- size="lg"
1561
- )
1562
-
1563
- # RIGHT: Results
1564
- with gr.Column(scale=1):
1565
-
1566
- gr.Markdown("## ANALYSIS OUTPUT")
1567
-
1568
- result_text = gr.Markdown(
1569
- value="""
1570
- > `AWAITING INPUT` β€” Upload a video file and trigger analysis to begin neural processing.
1571
- """
1572
- )
1573
-
1574
- result_output = gr.Label(
1575
- label="Emotion Probability Distribution",
1576
- num_top_classes=4,
1577
- elem_classes="black-label"
1578
- )
1579
-
1580
- transcription_output = gr.Textbox(
1581
- label="Transcribed Speech",
1582
- lines=5,
1583
- interactive=False,
1584
- placeholder="Speech transcription will appear here after analysis..."
1585
- )
1586
-
1587
- gr.HTML('<hr/>')
1588
-
1589
- # ── About Accordion ─────────────────────────────────
1590
- with gr.Accordion("SYSTEM ARCHITECTURE", open=False):
1591
-
1592
- gr.HTML("""
1593
- <div class="arch-grid">
1594
- <div class="arch-section">
1595
- <div class="arch-title">NEURAL PIPELINE</div>
1596
- <p class="arch-desc">Three independent encoders fused through a Hybrid Bimodal Fusion (HBF) block across 6 iterative layers.</p>
1597
- <div class="arch-row">
1598
- <div class="arch-tag" style="border-color:#00f5ff;color:#00f5ff;">WAV2VEC2</div>
1599
- <div class="arch-detail">Extracts deep acoustic features from raw audio waveform</div>
1600
- </div>
1601
- <div class="arch-row">
1602
- <div class="arch-tag" style="border-color:#bf00ff;color:#bf00ff;">BERT</div>
1603
- <div class="arch-detail">Understands semantic and emotional tone from transcribed speech</div>
1604
- </div>
1605
- <div class="arch-row">
1606
- <div class="arch-tag" style="border-color:#00ff88;color:#00ff88;">RESNET18</div>
1607
- <div class="arch-detail">Captures facial micro-expressions across sampled video frames</div>
1608
- </div>
1609
- <div class="arch-row">
1610
- <div class="arch-tag" style="border-color:#ff6b35;color:#ff6b35;">HBF</div>
1611
- <div class="arch-detail">Iteratively aligns all three modalities over 6 fusion layers</div>
1612
- </div>
1613
- </div>
1614
- <div class="arch-section">
1615
- <div class="arch-title">DETECTABLE EMOTIONS</div>
1616
- <div class="emotion-chips">
1617
- <div class="emotion-chip" style="border-color:#ff4444;color:#ff4444;box-shadow:0 0 12px rgba(255,68,68,0.2);">⚑ ANGRY</div>
1618
- <div class="emotion-chip" style="border-color:#ffcc00;color:#ffcc00;box-shadow:0 0 12px rgba(255,204,0,0.2);">β—ˆ HAPPY</div>
1619
- <div class="emotion-chip" style="border-color:#00f5ff;color:#00f5ff;box-shadow:0 0 12px rgba(0,245,255,0.2);">β—Ž NEUTRAL</div>
1620
- <div class="emotion-chip" style="border-color:#7b9fff;color:#7b9fff;box-shadow:0 0 12px rgba(123,159,255,0.2);">β—‡ SAD</div>
1621
- </div>
1622
- </div>
1623
- </div>
1624
- """)
1625
-
1626
- # ── Footer ──────────────────────────────────────────
1627
- gr.HTML("""
1628
- <div class="cyber-footer">
1629
- Built on PyTorch &nbsp;Β·&nbsp; HuggingFace Transformers &nbsp;Β·&nbsp; OpenAI Whisper &nbsp;Β·&nbsp; Gradio
1630
- </div>
1631
- """)
1632
 
1633
- # ── Button Action ────────────────────────────────────
 
 
 
 
 
 
 
 
 
 
1634
  predict_btn.click(
1635
  fn=predict_emotion,
1636
  inputs=[video_input],
1637
- outputs=[
1638
- result_text,
1639
- result_output,
1640
- transcription_output
1641
- ],
1642
- show_progress=True
1643
  )
1644
 
1645
- # =========================================================
1646
- # LAUNCH
1647
- # =========================================================
 
 
 
 
 
 
 
1648
 
1649
  if __name__ == "__main__":
1650
-
1651
- demo.queue()
1652
-
1653
  demo.launch()
 
5
  import librosa
6
  import cv2
7
  import re
8
+ from transformers import Wav2Vec2Processor, Wav2Vec2Model, AutoTokenizer, AutoModel
 
 
 
 
 
9
  from torchvision import models
10
  import tempfile
11
  import os
 
13
  import whisper
14
  import subprocess
15
 
16
+ # Configuration
 
 
 
17
  DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
18
  SAMPLE_RATE = 16000
 
19
  TEXT_MAX_LEN = 64
 
20
  LABELS = ["angry", "happy", "neutral", "sad"]
21
 
22
+ # Load processors
23
+ processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-960h")
24
+ tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
 
26
+ # Model Architecture (same as training)
27
  class ResNetVideoEncoder(nn.Module):
 
28
  def __init__(self, out_dim=768):
 
29
  super().__init__()
 
30
  base = models.resnet18(pretrained=False)
31
+ self.backbone = nn.Sequential(*list(base.children())[:-1])
 
 
 
 
32
  self.proj = nn.Linear(512, out_dim)
33
 
34
  def forward(self, x):
 
35
  B, C, T, H, W = x.shape
 
36
  feats = []
 
37
  for t in range(T):
 
38
  ft = self.backbone(x[:, :, t])
39
+ feats.append(ft.squeeze(-1).squeeze(-1))
 
 
 
 
40
  feats = torch.stack(feats, dim=1).mean(1)
 
41
  return self.proj(feats)
42
 
 
 
43
  def mean_pool(x, mask):
 
44
  mask = mask[:, :x.size(1)]
 
45
  mask = mask.unsqueeze(-1).float()
46
+ return (x * mask).sum(1) / mask.sum(1).clamp(min=1e-6)
 
 
 
 
 
 
47
 
48
  class HBF(nn.Module):
 
49
  def __init__(self, d=768, n_layers=6):
 
50
  super().__init__()
51
+ self.proj_a = nn.ModuleList([nn.Linear(d, d) for _ in range(n_layers)])
52
+ self.proj_t = nn.ModuleList([nn.Linear(d, d) for _ in range(n_layers)])
53
+ self.proj_v = nn.ModuleList([nn.Linear(d, d) for _ in range(n_layers)])
54
+ self.fwd1 = nn.ModuleList([nn.Linear(3*d, d) for _ in range(n_layers)])
55
+ self.fwd2 = nn.ModuleList([nn.Linear(d, d) for _ in range(n_layers)])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
  self.drop = nn.Dropout(0.1)
57
+ self.act1, self.act2 = nn.GELU(), nn.Tanh()
 
 
 
 
58
  self.n = n_layers
59
 
60
  def forward(self, a, t, v):
 
61
  v_prev = None
 
62
  for i in range(self.n):
63
+ va = self.act2(self.drop(self.proj_a[i](a)))
64
+ vt = self.act2(self.drop(self.proj_t[i](t)))
65
+ vv = self.act2(self.drop(self.proj_v[i](v)))
66
+ cat = torch.cat([va, vt, vv] if v_prev is None else [va, vt, v_prev], -1)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  x = self.act1(self.fwd1[i](cat))
 
68
  v_prev = self.fwd2[i](x)
 
69
  return v_prev
70
 
 
 
71
  class AVVideoModel(nn.Module):
 
72
  def __init__(self, num_classes, n_layers=6):
 
73
  super().__init__()
74
+ self.a_enc = Wav2Vec2Model.from_pretrained("facebook/wav2vec2-base-960h")
75
+ self.t_enc = AutoModel.from_pretrained("bert-base-uncased")
 
 
 
 
 
 
 
76
  self.v_enc = ResNetVideoEncoder()
 
77
  self.hbf = HBF(n_layers=n_layers)
 
78
  self.fc = nn.Linear(768, num_classes)
79
+ self.fc_audio = nn.Linear(768, num_classes)
80
+ self.fc_text = nn.Linear(768, num_classes)
81
+ self.fc_video = nn.Linear(768, num_classes)
82
 
83
+ def forward(self, audio, audio_mask, text_ids, text_mask, video):
84
+ a_out = self.a_enc(audio, attention_mask=audio_mask, return_dict=True)
85
+ t_out = self.t_enc(input_ids=text_ids, attention_mask=text_mask, return_dict=True)
86
+
87
+ a_pool = mean_pool(a_out.last_hidden_state, audio_mask)
88
+ t_pool = mean_pool(t_out.last_hidden_state, text_mask)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89
  v_pool = self.v_enc(video)
90
+
91
+ a_pool = torch.nan_to_num(a_pool, nan=0.0, posinf=1e4, neginf=-1e4)
92
+ t_pool = torch.nan_to_num(t_pool, nan=0.0, posinf=1e4, neginf=-1e4)
93
+ v_pool = torch.nan_to_num(v_pool, nan=0.0, posinf=1e4, neginf=-1e4)
94
+
95
+ a_logits = self.fc_audio(a_pool)
96
+ t_logits = self.fc_text(t_pool)
97
+ v_logits = self.fc_video(v_pool)
98
+
99
+ fused = self.hbf(a_pool, t_pool, v_pool)
100
+ fused = torch.nan_to_num(fused, nan=0.0, posinf=1e4, neginf=-1e4)
101
+ fused_logits = self.fc(fused)
102
+
103
+ return fused_logits, a_logits, t_logits, v_logits
104
 
105
+ # Load model
 
 
 
 
 
 
 
 
 
 
 
 
106
 
107
+ model = AVVideoModel(num_classes=len(LABELS)).to(DEVICE)
 
 
108
 
109
+ # Download model from Hugging Face Model Hub
110
  try:
 
111
  model_path = hf_hub_download(
112
+ repo_id="ApurvaKondekar/emotion_model", # CHANGE THIS
113
+ filename="model_weights.pth" # YOUR FILE NAME
 
 
 
 
114
  )
115
+ model.load_state_dict(torch.load(model_path, map_location=DEVICE))
116
  model.eval()
117
+ print("βœ… Model loaded from Hugging Face")
 
 
118
  except Exception as e:
119
+ print(f"❌ Failed to load model: {e}")
120
+ # Alternative: Create a dummy model for testing
121
+ # Load trained weights (you'll need to upload this)
122
+ if os.path.exists("model_weights.pth"):
123
+ model.load_state_dict(torch.load("model_weights.pth", map_location=DEVICE))
124
+ model.eval()
125
+ print("βœ… Model loaded successfully")
126
+ else:
127
+ print("⚠️ No model weights found. Using untrained model for demo.")
128
+
129
+ def extract_video_frames(video_path, max_frames=8, resize=(224, 224)):
130
+ """Extract frames from video file"""
 
131
  cap = cv2.VideoCapture(video_path)
 
132
  if not cap.isOpened():
133
  return None
134
+
 
 
 
 
 
 
 
 
 
 
 
135
  frames = []
136
+ while len(frames) < max_frames:
 
 
 
 
137
  ret, frame = cap.read()
 
138
  if not ret:
139
+ break
140
+ frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
 
 
 
 
 
141
  frame = cv2.resize(frame, resize)
 
142
  frames.append(frame)
143
+
144
  cap.release()
145
+
146
  if len(frames) == 0:
147
  return None
148
+
149
+ # Pad if needed
150
  while len(frames) < max_frames:
151
  frames.append(frames[-1])
152
+
153
+ frames = np.array(frames[:max_frames], dtype=np.uint8)
 
154
  return frames
155
 
 
 
 
 
156
  def extract_audio_from_video(video_path):
157
+ """Extract audio from video file using ffmpeg"""
158
+ try:
159
+ audio_path = tempfile.NamedTemporaryFile(delete=False, suffix=".wav").name
160
+ # Use ffmpeg to extract audio
161
+ import subprocess
162
+ command = [
163
+ 'ffmpeg', '-i', video_path,
164
+ '-vn', # No video
165
+ '-acodec', 'pcm_s16le', # Audio codec
166
+ '-ar', str(SAMPLE_RATE), # Sample rate
167
+ '-ac', '1', # Mono
168
+ '-y', # Overwrite
169
+ audio_path
170
+ ]
171
+ subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
172
+ return audio_path
173
+ except Exception as e:
174
+ raise ValueError(f"Could not extract audio from video: {str(e)}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
 
176
  def transcribe_audio(audio_path):
177
+ """Transcribe audio using Whisper"""
178
+ try:
179
+ whisper_model = whisper.load_model("base")
180
+ result = whisper_model.transcribe(audio_path)
181
+ return result["text"].strip()
182
+ except Exception as e:
183
+ raise ValueError(f"Could not transcribe audio: {str(e)}")
184
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
185
 
186
+ def preprocess_inputs(audio_path, text, video_path):
187
+ """Preprocess all three modalities"""
188
+
189
+ # Audio
190
+ wav, _ = librosa.load(audio_path, sr=SAMPLE_RATE)
191
+ audio_inputs = processor(wav, sampling_rate=SAMPLE_RATE, return_tensors="pt")
192
  audio_values = audio_inputs.input_values.to(DEVICE)
193
+ audio_mask = torch.ones_like(audio_values).to(DEVICE)
194
+
195
+ # Text
196
+ text_clean = re.sub(r"[^a-zA-Z0-9\s]", "", text.lower())
 
 
 
 
 
 
 
197
  text_inputs = tokenizer(
198
  text_clean,
199
  truncation=True,
 
201
  max_length=TEXT_MAX_LEN,
202
  return_tensors="pt"
203
  )
 
204
  text_ids = text_inputs.input_ids.to(DEVICE)
 
205
  text_mask = text_inputs.attention_mask.to(DEVICE)
206
+
207
+ # Video
208
  frames = extract_video_frames(video_path)
209
+ if frames is None:
210
+ raise ValueError("Could not extract frames from video")
211
+
212
+ frames_tensor = torch.tensor(frames).permute(0, 3, 1, 2).float() / 255.0
213
+ frames_tensor = frames_tensor.unsqueeze(0).permute(0, 2, 1, 3, 4).to(DEVICE)
214
+
215
+ return audio_values, audio_mask, text_ids, text_mask, frames_tensor
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
216
 
217
  def predict_emotion(video_file):
218
+ """Main prediction function - takes only video input"""
219
+
220
  if video_file is None:
221
+ return "Please provide a video file", None, ""
222
+
 
 
 
 
 
223
  try:
224
+ # Extract audio from video
225
+ audio_path = extract_audio_from_video(video_file)
226
+
227
+ # Transcribe audio
228
+ transcribed_text = transcribe_audio(audio_path)
229
+
230
+ # Preprocess all modalities
231
+ audio, audio_mask, text_ids, text_mask, video = preprocess_inputs(
232
+ audio_path, transcribed_text, video_file
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
233
  )
234
+
235
+ # Inference
236
  with torch.no_grad():
237
+ fused_logits, a_logits, t_logits, v_logits = model(
238
+ audio, audio_mask, text_ids, text_mask, video
 
 
 
 
 
239
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
240
 
241
+ # Get probabilities
242
+ fused_probs = torch.softmax(fused_logits, dim=1)[0].cpu().numpy()
243
 
244
+ # Format results
245
+ result = {LABELS[i]: float(fused_probs[i]) for i in range(len(LABELS))}
246
+
247
+ predicted_emotion = LABELS[fused_probs.argmax()]
248
+ confidence = float(fused_probs.max())
249
+
250
+ result_text = f"🎯 **Predicted Emotion: {predicted_emotion.upper()}**\n\n**Confidence: {confidence:.2%}**"
251
+
252
+ # Clean up temporary audio file
253
  if os.path.exists(audio_path):
254
  os.remove(audio_path)
255
+
256
+ return result_text, result, transcribed_text
257
+
 
 
 
 
258
  except Exception as e:
259
+ return f"Error: {str(e)}", None, ""
260
 
261
+ # Gradio Interface
262
+ with gr.Blocks(title="Multimodal Emotion Recognition", theme=gr.themes.Soft()) as demo:
263
+ gr.Markdown(
264
+ """
265
+ # 🎭 Multimodal Emotion Recognition
266
+
267
+ This system predicts emotions from video by automatically extracting and analyzing:
268
+ - 🎀 **Audio** (extracted from video)
269
+ - πŸ“ **Text** (transcribed from audio using Whisper)
270
+ - πŸŽ₯ **Video** (visual frames)
271
+
272
+ ### How to use:
273
+ 1. Upload a video file (MP4, AVI, MOV, etc.)
274
+ 2. Click "Predict Emotion"
275
+ 3. The system will automatically extract audio, transcribe speech, and analyze all modalities
276
+
277
+ The model will provide emotion predictions based on all three inputs.
278
+ """
279
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
280
 
281
+
282
+ with gr.Row():
283
+ with gr.Column():
284
+ video_input = gr.Video(label="πŸŽ₯ Video Input")
285
+ predict_btn = gr.Button("πŸš€ Predict Emotion", variant="primary", size="lg")
286
+
287
+ with gr.Column():
288
+ result_text = gr.Markdown(label="Result")
289
+ result_output = gr.Label(label="πŸ“Š Prediction Results", num_top_classes=4)
290
+ transcription_output = gr.Textbox(label="πŸ“ Transcribed Text", lines=3, interactive=False)
291
+
292
  predict_btn.click(
293
  fn=predict_emotion,
294
  inputs=[video_input],
295
+ outputs=[result_text, result_output, transcription_output]
 
 
 
 
 
296
  )
297
 
298
+
299
+ gr.Markdown(
300
+ """
301
+ ---
302
+ ### πŸ“Œ Notes:
303
+ - Supported emotions: **Angry, Happy, Neutral, Sad**
304
+ - Model uses Wav2Vec2 (audio), BERT (text), and ResNet18 (video)
305
+ - Best results with clear audio, accurate transcripts, and visible faces
306
+ """
307
+ )
308
 
309
  if __name__ == "__main__":
 
 
 
310
  demo.launch()