change input format

app.py

@@ -10,6 +10,9 @@ from torchvision import models
 import tempfile
 import os
 from huggingface_hub import hf_hub_download
+import whisper
+from moviepy.editor import VideoFileClip
+
 
 # Configuration
 DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -151,6 +154,27 @@ def extract_video_frames(video_path, max_frames=8, resize=(224, 224)):
     frames = np.array(frames[:max_frames], dtype=np.uint8)
     return frames
 
+def extract_audio_from_video(video_path):
+    """Extract audio from video file"""
+    try:
+        video = VideoFileClip(video_path)
+        audio_path = tempfile.NamedTemporaryFile(delete=False, suffix=".wav").name
+        video.audio.write_audiofile(audio_path, fps=SAMPLE_RATE, verbose=False, logger=None)
+        video.close()
+        return audio_path
+    except Exception as e:
+        raise ValueError(f"Could not extract audio from video: {str(e)}")
+
+def transcribe_audio(audio_path):
+    """Transcribe audio using Whisper"""
+    try:
+        whisper_model = whisper.load_model("base")
+        result = whisper_model.transcribe(audio_path)
+        return result["text"].strip()
+    except Exception as e:
+        raise ValueError(f"Could not transcribe audio: {str(e)}")
+
+
 def preprocess_inputs(audio_path, text, video_path):
     """Preprocess all three modalities"""
     
@@ -182,16 +206,22 @@ def preprocess_inputs(audio_path, text, video_path):
     
     return audio_values, audio_mask, text_ids, text_mask, frames_tensor
 
-def predict_emotion(audio_file, text_input, video_file):
-    """Main prediction function"""
+def predict_emotion(video_file):
+    """Main prediction function - takes only video input"""
     
-    if audio_file is None or video_file is None or not text_input.strip():
-        return "Please provide all three inputs: audio, text, and video", None, None, None, None
+    if video_file is None:
+        return "Please provide a video file", None, ""
     
     try:
-        # Preprocess
+        # Extract audio from video
+        audio_path = extract_audio_from_video(video_file)
+        
+        # Transcribe audio
+        transcribed_text = transcribe_audio(audio_path)
+        
+        # Preprocess all modalities
         audio, audio_mask, text_ids, text_mask, video = preprocess_inputs(
-            audio_file, text_input, video_file
+            audio_path, transcribed_text, video_file
         )
         
         # Inference
@@ -202,25 +232,23 @@ def predict_emotion(audio_file, text_input, video_file):
         
         # Get probabilities
         fused_probs = torch.softmax(fused_logits, dim=1)[0].cpu().numpy()
-        audio_probs = torch.softmax(a_logits, dim=1)[0].cpu().numpy()
-        text_probs = torch.softmax(t_logits, dim=1)[0].cpu().numpy()
-        video_probs = torch.softmax(v_logits, dim=1)[0].cpu().numpy()
         
         # Format results
-        fused_result = {LABELS[i]: float(fused_probs[i]) for i in range(len(LABELS))}
-        audio_result = {LABELS[i]: float(audio_probs[i]) for i in range(len(LABELS))}
-        text_result = {LABELS[i]: float(text_probs[i]) for i in range(len(LABELS))}
-        video_result = {LABELS[i]: float(video_probs[i]) for i in range(len(LABELS))}
+        result = {LABELS[i]: float(fused_probs[i]) for i in range(len(LABELS))}
         
         predicted_emotion = LABELS[fused_probs.argmax()]
         confidence = float(fused_probs.max())
         
         result_text = f"🎯 **Predicted Emotion: {predicted_emotion.upper()}**\n\n**Confidence: {confidence:.2%}**"
         
-        return result_text, fused_result, audio_result, text_result, video_result
+        # Clean up temporary audio file
+        if os.path.exists(audio_path):
+            os.remove(audio_path)
+        
+        return result_text, result, transcribed_text
         
     except Exception as e:
-        return f"Error: {str(e)}", None, None, None, None
+        return f"Error: {str(e)}", None, ""
 
 # Gradio Interface
 with gr.Blocks(title="Multimodal Emotion Recognition", theme=gr.themes.Soft()) as demo:
@@ -228,43 +256,37 @@ with gr.Blocks(title="Multimodal Emotion Recognition", theme=gr.themes.Soft()) a
         """
         # 🎭 Multimodal Emotion Recognition
         
-        This system predicts emotions using **Audio**, **Text**, and **Video** inputs simultaneously.
+        This system predicts emotions from video by automatically extracting and analyzing:
+        - 🎤 **Audio** (extracted from video)
+        - 📝 **Text** (transcribed from audio using Whisper)
+        - 🎥 **Video** (visual frames)
         
         ### How to use:
-        1. Upload an audio file (WAV, MP3)
-        2. Enter the transcript or spoken text
-        3. Upload a video file (MP4, AVI)
-        4. Click "Predict Emotion"
+        1. Upload a video file (MP4, AVI, MOV, etc.)
+        2. Click "Predict Emotion"
+        3. The system will automatically extract audio, transcribe speech, and analyze all modalities
         
-        The model will analyze all three modalities and provide predictions.
+        The model will provide emotion predictions based on all three inputs.
         """
     )
+
     
     with gr.Row():
         with gr.Column():
-            audio_input = gr.Audio(type="filepath", label="🎤 Audio Input")
-            text_input = gr.Textbox(
-                label="📝 Text Transcript",
-                placeholder="Enter what was said in the audio/video...",
-                lines=3
-            )
             video_input = gr.Video(label="🎥 Video Input")
             predict_btn = gr.Button("🚀 Predict Emotion", variant="primary", size="lg")
         
         with gr.Column():
             result_text = gr.Markdown(label="Result")
-            
-            with gr.Accordion("📊 Detailed Predictions", open=True):
-                fused_output = gr.Label(label="🔗 Fused Prediction", num_top_classes=4)
-                audio_output = gr.Label(label="🎤 Audio-only Prediction", num_top_classes=4)
-                text_output = gr.Label(label="📝 Text-only Prediction", num_top_classes=4)
-                video_output = gr.Label(label="🎥 Video-only Prediction", num_top_classes=4)
+            result_output = gr.Label(label="📊 Prediction Results", num_top_classes=4)
+            transcription_output = gr.Textbox(label="📝 Transcribed Text", lines=3, interactive=False)
     
     predict_btn.click(
         fn=predict_emotion,
-        inputs=[audio_input, text_input, video_input],
-        outputs=[result_text, fused_output, audio_output, text_output, video_output]
+        inputs=[video_input],
+        outputs=[result_text, result_output, transcription_output]
     )
+
     
     gr.Markdown(
         """

requirements.txt

@@ -7,3 +7,5 @@ numpy
 soundfile
 accelerate
 huggingface_hub
+openai-whisper
+moviepy