Update app.py

app.py

@@ -5,91 +5,91 @@ import cv2
 import os
 import numpy as np
 import tensorflow as tf
-
-emotion_labels = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
-
-def trained_model(model_path):
-
-    input_visual = tf.keras.Input((120, 120, 3, 10), name="input_visual")  # 90 - 120
-    input_audio_cnn = tf.keras.Input((150, 512, 1), name="input_audio_cnn")
-    input_audio_wave = tf.keras.Input((20, 13077), name="input_audio_wave")
-
-    # Visual branch
-    x_v = tf.keras.layers.Conv3D(10, (3, 3, 3), strides=(2, 2, 1), padding='same')(input_visual)
-    x_v = tf.keras.layers.BatchNormalization()(x_v)
-    x_v = tf.keras.layers.ReLU()(x_v)
-    x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
-
-    x_v = tf.keras.layers.Conv3D(40, (3, 3, 3), strides=(2, 2, 1), padding='same')(x_v)
-    x_v = tf.keras.layers.BatchNormalization()(x_v)
-    x_v = tf.keras.layers.ReLU()(x_v)
-    x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
-
-    x_v = tf.keras.layers.Flatten()(x_v)
-
-    x_v = tf.keras.layers.Dropout(0.2)(x_v)
-    x_v = tf.keras.layers.Dense(500)(x_v)
-    x_v = tf.keras.layers.BatchNormalization()(x_v)
-    x_v = tf.keras.layers.ReLU()(x_v)
-
-    # Audio cnn branch
-    x_c = tf.keras.layers.Conv2D(5, (3, 3), strides=(2, 2), padding='same')(input_audio_cnn)
-    x_c = tf.keras.layers.BatchNormalization()(x_c)
-    x_c = tf.keras.layers.ReLU()(x_c)
-    x_c = tf.keras.layers.MaxPooling2D((3, 3))(x_c)
-
-    x_c = tf.keras.layers.Conv2D(30, (3, 3), strides=(2, 2), padding='same')(x_c)
-    x_c = tf.keras.layers.BatchNormalization()(x_c)
-    x_c = tf.keras.layers.ReLU()(x_c)
-    x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
-
-    x_c = tf.keras.layers.Conv2D(100, (3, 3), strides=(1, 1), padding='same')(x_c)
-    x_c = tf.keras.layers.BatchNormalization()(x_c)
-    x_c = tf.keras.layers.ReLU()(x_c)
-    x_c = tf.keras.layers.Conv2D(200, (3, 3), strides=(1, 1), padding='same')(x_c)
-    x_c = tf.keras.layers.BatchNormalization()(x_c)
-    x_c = tf.keras.layers.ReLU()(x_c)
-    x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
-
-    x_c = tf.keras.layers.Flatten()(x_c)
-
-    x_c = tf.keras.layers.Dropout(0.2)(x_c)
-    x_c = tf.keras.layers.Dense(500)(x_c)
-    x_c = tf.keras.layers.BatchNormalization()(x_c)
-    x_c = tf.keras.layers.ReLU()(x_c)
-
-    # Audio wave branch
-    x_w = tf.keras.layers.LSTM(500)(input_audio_wave)
-    x_w = tf.keras.layers.RepeatVector(20)(x_w)
-    x_w = tf.keras.layers.LSTM(500)(x_w)
-
-    x_w = tf.keras.layers.Flatten()(x_w)
-
-    x_w = tf.keras.layers.Dropout(0.2)(x_w)
-    x_w = tf.keras.layers.Dense(500)(x_w)
-    x_w = tf.keras.layers.BatchNormalization()(x_w)
-    x_w = tf.keras.layers.ReLU()(x_w)
-
-    # Audio fusion
-    x_a = x_c + x_w
-    x_a = tf.keras.layers.Dense(500)(x_a)
-    x_a = tf.keras.layers.BatchNormalization()(x_a)
-    x_a = tf.keras.layers.ReLU()(x_a)
-
-    # Fusion
-    x = x_a + x_v
-    x = tf.keras.layers.Dense(500)(x)
-    x = tf.keras.layers.BatchNormalization()(x)
-    x = tf.keras.layers.ReLU()(x)
-
-    # Output
-    x = tf.keras.layers.Dropout(0.1)(x) 
-    x = tf.keras.layers.Dense(6, activation='softmax', name='output_classification')(x)  # 8 - 6
+model = load_model("./model_vui_ve.h5")
+# emotion_labels = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
+
+# def trained_model(model_path):
+
+#     input_visual = tf.keras.Input((120, 120, 3, 10), name="input_visual")  # 90 - 120
+#     input_audio_cnn = tf.keras.Input((150, 512, 1), name="input_audio_cnn")
+#     input_audio_wave = tf.keras.Input((20, 13077), name="input_audio_wave")
+
+#     # Visual branch
+#     x_v = tf.keras.layers.Conv3D(10, (3, 3, 3), strides=(2, 2, 1), padding='same')(input_visual)
+#     x_v = tf.keras.layers.BatchNormalization()(x_v)
+#     x_v = tf.keras.layers.ReLU()(x_v)
+#     x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
+
+#     x_v = tf.keras.layers.Conv3D(40, (3, 3, 3), strides=(2, 2, 1), padding='same')(x_v)
+#     x_v = tf.keras.layers.BatchNormalization()(x_v)
+#     x_v = tf.keras.layers.ReLU()(x_v)
+#     x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
+
+#     x_v = tf.keras.layers.Flatten()(x_v)
+
+#     x_v = tf.keras.layers.Dropout(0.2)(x_v)
+#     x_v = tf.keras.layers.Dense(500)(x_v)
+#     x_v = tf.keras.layers.BatchNormalization()(x_v)
+#     x_v = tf.keras.layers.ReLU()(x_v)
+
+#     # Audio cnn branch
+#     x_c = tf.keras.layers.Conv2D(5, (3, 3), strides=(2, 2), padding='same')(input_audio_cnn)
+#     x_c = tf.keras.layers.BatchNormalization()(x_c)
+#     x_c = tf.keras.layers.ReLU()(x_c)
+#     x_c = tf.keras.layers.MaxPooling2D((3, 3))(x_c)
+
+#     x_c = tf.keras.layers.Conv2D(30, (3, 3), strides=(2, 2), padding='same')(x_c)
+#     x_c = tf.keras.layers.BatchNormalization()(x_c)
+#     x_c = tf.keras.layers.ReLU()(x_c)
+#     x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
+
+#     x_c = tf.keras.layers.Conv2D(100, (3, 3), strides=(1, 1), padding='same')(x_c)
+#     x_c = tf.keras.layers.BatchNormalization()(x_c)
+#     x_c = tf.keras.layers.ReLU()(x_c)
+#     x_c = tf.keras.layers.Conv2D(200, (3, 3), strides=(1, 1), padding='same')(x_c)
+#     x_c = tf.keras.layers.BatchNormalization()(x_c)
+#     x_c = tf.keras.layers.ReLU()(x_c)
+#     x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
+
+#     x_c = tf.keras.layers.Flatten()(x_c)
+
+#     x_c = tf.keras.layers.Dropout(0.2)(x_c)
+#     x_c = tf.keras.layers.Dense(500)(x_c)
+#     x_c = tf.keras.layers.BatchNormalization()(x_c)
+#     x_c = tf.keras.layers.ReLU()(x_c)
+
+#     # Audio wave branch
+#     x_w = tf.keras.layers.LSTM(500)(input_audio_wave)
+#     x_w = tf.keras.layers.RepeatVector(20)(x_w)
+#     x_w = tf.keras.layers.LSTM(500)(x_w)
+
+#     x_w = tf.keras.layers.Flatten()(x_w)
+
+#     x_w = tf.keras.layers.Dropout(0.2)(x_w)
+#     x_w = tf.keras.layers.Dense(500)(x_w)
+#     x_w = tf.keras.layers.BatchNormalization()(x_w)
+#     x_w = tf.keras.layers.ReLU()(x_w)
+
+#     # Audio fusion
+#     x_a = x_c + x_w
+#     x_a = tf.keras.layers.Dense(500)(x_a)
+#     x_a = tf.keras.layers.BatchNormalization()(x_a)
+#     x_a = tf.keras.layers.ReLU()(x_a)
+
+#     # Fusion
+#     x = x_a + x_v
+#     x = tf.keras.layers.Dense(500)(x)
+#     x = tf.keras.layers.BatchNormalization()(x)
+#     x = tf.keras.layers.ReLU()(x)
+
+#     # Output
+#     x = tf.keras.layers.Dropout(0.1)(x) 
+#     x = tf.keras.layers.Dense(6, activation='softmax', name='output_classification')(x)  # 8 - 6
 
     
-    model = model.load(model_path)
+#     model = model.load(model_path)
 
-    return model
+#     return model
 
 def process_video_audio(video_path, audio_path):
 
@@ -136,28 +136,66 @@ def process_video_audio(video_path, audio_path):
     train_audio_cnn = tf.convert_to_tensor(train_audio_cnn, dtype=tf.float16)
 
     return train_visual, train_audio_wave, train_audio_cnn
+    
+def predict_emotion(video_path, audio_path):
+    train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video_path, audio_path)
+    
+    
+    predictions = model.predict({
+        "input_visual": train_visual,
+        "input_audio_cnn": train_audio_cnn,
+        "input_audio_wave": train_audio_wave
+    })
+    
+    predicted_label = np.argmax(predictions)
+    return predicted_label
+
 
+
+predicted_label = predict_emotion(video_path)
+emotion_dict = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
+predicted_emotion = emotion_dict[predicted_label]
+print("Predicted Emotion: ", predicted_emotion)
 # Định nghĩa giao diện Gradio
-def gradio_interface(video, audio):
+
+def predict_emotion_gradio(video, audio):
+    predicted_label = predict_emotion(video, audio)
+    predicted_emotion = emotion_dict[predicted_label]
+    return predicted_emotion
+
+# def gradio_interface(video, audio):
     
-    train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video, audio)
-    model = trained_model("./model_vui_ve.h5")
-    output = model.predict({"input_visual": train_visual, "input_audio_cnn": train_audio_cnn, "input_audio_wave": train_audio_wave})
-    emo_index = tf.math.argmax(output)
+#     train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video, audio)
+#     model = trained_model("./model_vui_ve.h5")
+#     output = model.predict({"input_visual": train_visual, "input_audio_cnn": train_audio_cnn, "input_audio_wave": train_audio_wave})
+#     emo_index = tf.math.argmax(output)
     
-    return emotion_labels[emo_index]
+#     return emotion_labels[emo_index]
 
 iface = gr.Interface(
-    fn=gradio_interface,
+    fn=predict_emotion_gradio, 
     inputs=[
-        gr.Video(),
-        gr.Audio()
+        gr.Video(label="Upload a video"),
+        gr.Audio(label="Upload a audio")
     ],
-    outputs=[
-        gr.Text()
-    ],
-    live=True,
-    title="Video and Audio Processing with Emotion Recognition"
+    outputs=gr.Textbox(label="Predicted Emotion"),
+    title="Emotion Recognition from Video",
+    description="Upload a video and get the predicted emotion."
 )
 
 iface.launch()
+
+# iface = gr.Interface(
+#     fn=gradio_interface,
+#     inputs=[
+#         gr.Video(),
+#         gr.Audio()
+#     ],
+#     outputs=[
+#         gr.Text()
+#     ],
+#     live=True,
+#     title="Video and Audio Processing with Emotion Recognition"
+# )
+
+# iface.launch()