Update app.py

app.py

@@ -9,90 +9,89 @@ import tensorflow as tf
 from tensorflow.keras.models import load_model
 
 
-# emotion_labels = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
-
-# def trained_model(model_path):
-
-#     input_visual = tf.keras.Input((120, 120, 3, 10), name="input_visual")  # 90 - 120
-#     input_audio_cnn = tf.keras.Input((150, 512, 1), name="input_audio_cnn")
-#     input_audio_wave = tf.keras.Input((20, 13077), name="input_audio_wave")
-
-#     # Visual branch
-#     x_v = tf.keras.layers.Conv3D(10, (3, 3, 3), strides=(2, 2, 1), padding='same')(input_visual)
-#     x_v = tf.keras.layers.BatchNormalization()(x_v)
-#     x_v = tf.keras.layers.ReLU()(x_v)
-#     x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
-
-#     x_v = tf.keras.layers.Conv3D(40, (3, 3, 3), strides=(2, 2, 1), padding='same')(x_v)
-#     x_v = tf.keras.layers.BatchNormalization()(x_v)
-#     x_v = tf.keras.layers.ReLU()(x_v)
-#     x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
-
-#     x_v = tf.keras.layers.Flatten()(x_v)
-
-#     x_v = tf.keras.layers.Dropout(0.2)(x_v)
-#     x_v = tf.keras.layers.Dense(500)(x_v)
-#     x_v = tf.keras.layers.BatchNormalization()(x_v)
-#     x_v = tf.keras.layers.ReLU()(x_v)
-
-#     # Audio cnn branch
-#     x_c = tf.keras.layers.Conv2D(5, (3, 3), strides=(2, 2), padding='same')(input_audio_cnn)
-#     x_c = tf.keras.layers.BatchNormalization()(x_c)
-#     x_c = tf.keras.layers.ReLU()(x_c)
-#     x_c = tf.keras.layers.MaxPooling2D((3, 3))(x_c)
-
-#     x_c = tf.keras.layers.Conv2D(30, (3, 3), strides=(2, 2), padding='same')(x_c)
-#     x_c = tf.keras.layers.BatchNormalization()(x_c)
-#     x_c = tf.keras.layers.ReLU()(x_c)
-#     x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
-
-#     x_c = tf.keras.layers.Conv2D(100, (3, 3), strides=(1, 1), padding='same')(x_c)
-#     x_c = tf.keras.layers.BatchNormalization()(x_c)
-#     x_c = tf.keras.layers.ReLU()(x_c)
-#     x_c = tf.keras.layers.Conv2D(200, (3, 3), strides=(1, 1), padding='same')(x_c)
-#     x_c = tf.keras.layers.BatchNormalization()(x_c)
-#     x_c = tf.keras.layers.ReLU()(x_c)
-#     x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
-
-#     x_c = tf.keras.layers.Flatten()(x_c)
-
-#     x_c = tf.keras.layers.Dropout(0.2)(x_c)
-#     x_c = tf.keras.layers.Dense(500)(x_c)
-#     x_c = tf.keras.layers.BatchNormalization()(x_c)
-#     x_c = tf.keras.layers.ReLU()(x_c)
-
-#     # Audio wave branch
-#     x_w = tf.keras.layers.LSTM(500)(input_audio_wave)
-#     x_w = tf.keras.layers.RepeatVector(20)(x_w)
-#     x_w = tf.keras.layers.LSTM(500)(x_w)
-
-#     x_w = tf.keras.layers.Flatten()(x_w)
-
-#     x_w = tf.keras.layers.Dropout(0.2)(x_w)
-#     x_w = tf.keras.layers.Dense(500)(x_w)
-#     x_w = tf.keras.layers.BatchNormalization()(x_w)
-#     x_w = tf.keras.layers.ReLU()(x_w)
-
-#     # Audio fusion
-#     x_a = x_c + x_w
-#     x_a = tf.keras.layers.Dense(500)(x_a)
-#     x_a = tf.keras.layers.BatchNormalization()(x_a)
-#     x_a = tf.keras.layers.ReLU()(x_a)
-
-#     # Fusion
-#     x = x_a + x_v
-#     x = tf.keras.layers.Dense(500)(x)
-#     x = tf.keras.layers.BatchNormalization()(x)
-#     x = tf.keras.layers.ReLU()(x)
-
-#     # Output
-#     x = tf.keras.layers.Dropout(0.1)(x) 
-#     x = tf.keras.layers.Dense(6, activation='softmax', name='output_classification')(x)  # 8 - 6
+
+def trained_model(model_path):
+
+    input_visual = tf.keras.Input((120, 120, 3, 10), name="input_visual")  # 90 - 120
+    input_audio_cnn = tf.keras.Input((150, 512, 1), name="input_audio_cnn")
+    input_audio_wave = tf.keras.Input((20, 13077), name="input_audio_wave")
+
+    # Visual branch
+    x_v = tf.keras.layers.Conv3D(10, (3, 3, 3), strides=(2, 2, 1), padding='same')(input_visual)
+    x_v = tf.keras.layers.BatchNormalization()(x_v)
+    x_v = tf.keras.layers.ReLU()(x_v)
+    x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
+
+    x_v = tf.keras.layers.Conv3D(40, (3, 3, 3), strides=(2, 2, 1), padding='same')(x_v)
+    x_v = tf.keras.layers.BatchNormalization()(x_v)
+    x_v = tf.keras.layers.ReLU()(x_v)
+    x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
+
+    x_v = tf.keras.layers.Flatten()(x_v)
+
+    x_v = tf.keras.layers.Dropout(0.2)(x_v)
+    x_v = tf.keras.layers.Dense(500)(x_v)
+    x_v = tf.keras.layers.BatchNormalization()(x_v)
+    x_v = tf.keras.layers.ReLU()(x_v)
+
+    # Audio cnn branch
+    x_c = tf.keras.layers.Conv2D(5, (3, 3), strides=(2, 2), padding='same')(input_audio_cnn)
+    x_c = tf.keras.layers.BatchNormalization()(x_c)
+    x_c = tf.keras.layers.ReLU()(x_c)
+    x_c = tf.keras.layers.MaxPooling2D((3, 3))(x_c)
+
+    x_c = tf.keras.layers.Conv2D(30, (3, 3), strides=(2, 2), padding='same')(x_c)
+    x_c = tf.keras.layers.BatchNormalization()(x_c)
+    x_c = tf.keras.layers.ReLU()(x_c)
+    x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
+
+    x_c = tf.keras.layers.Conv2D(100, (3, 3), strides=(1, 1), padding='same')(x_c)
+    x_c = tf.keras.layers.BatchNormalization()(x_c)
+    x_c = tf.keras.layers.ReLU()(x_c)
+    x_c = tf.keras.layers.Conv2D(200, (3, 3), strides=(1, 1), padding='same')(x_c)
+    x_c = tf.keras.layers.BatchNormalization()(x_c)
+    x_c = tf.keras.layers.ReLU()(x_c)
+    x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
+
+    x_c = tf.keras.layers.Flatten()(x_c)
+
+    x_c = tf.keras.layers.Dropout(0.2)(x_c)
+    x_c = tf.keras.layers.Dense(500)(x_c)
+    x_c = tf.keras.layers.BatchNormalization()(x_c)
+    x_c = tf.keras.layers.ReLU()(x_c)
+
+    # Audio wave branch
+    x_w = tf.keras.layers.LSTM(500)(input_audio_wave)
+    x_w = tf.keras.layers.RepeatVector(20)(x_w)
+    x_w = tf.keras.layers.LSTM(500)(x_w)
+
+    x_w = tf.keras.layers.Flatten()(x_w)
+
+    x_w = tf.keras.layers.Dropout(0.2)(x_w)
+    x_w = tf.keras.layers.Dense(500)(x_w)
+    x_w = tf.keras.layers.BatchNormalization()(x_w)
+    x_w = tf.keras.layers.ReLU()(x_w)
+
+    # Audio fusion
+    x_a = x_c + x_w
+    x_a = tf.keras.layers.Dense(500)(x_a)
+    x_a = tf.keras.layers.BatchNormalization()(x_a)
+    x_a = tf.keras.layers.ReLU()(x_a)
+
+    # Fusion
+    x = x_a + x_v
+    x = tf.keras.layers.Dense(500)(x)
+    x = tf.keras.layers.BatchNormalization()(x)
+    x = tf.keras.layers.ReLU()(x)
+
+    # Output
+    x = tf.keras.layers.Dropout(0.1)(x) 
+    x = tf.keras.layers.Dense(6, activation='softmax', name='output_classification')(x)  # 8 - 6
 
     
-#     model = model.load(model_path)
+    model = model.load(model_path)
 
-#     return model
+    return model
 
 def process_video_audio(video_path, audio_path):
 
@@ -140,60 +139,60 @@ def process_video_audio(video_path, audio_path):
 
     return train_visual, train_audio_wave, train_audio_cnn
     
-def predict_emotion(video_path, audio_path):
-    train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video_path, audio_path)
+# def predict_emotion(video_path, audio_path):
+#     train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video_path, audio_path)
     
-    model = load_model("./model_vui_ve.h5")
-    predictions = model.predict({
-        "input_visual": train_visual,
-        "input_audio_cnn": train_audio_cnn,
-        "input_audio_wave": train_audio_wave
-    })
+#     model = load_model("./model_vui_ve.h5")
+#     predictions = model.predict({
+#         "input_visual": train_visual,
+#         "input_audio_cnn": train_audio_cnn,
+#         "input_audio_wave": train_audio_wave
+#     })
     
-    predicted_label = np.argmax(predictions)
-    return predicted_label
-
-# Định nghĩa giao diện Gradio
+#     predicted_label = np.argmax(predictions)
+#     return predicted_label
+
+# # Định nghĩa giao diện Gradio
+
+# def predict_emotion_gradio(video, audio):
+#     emotion_dict = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
+#     predicted_label = predict_emotion(video, audio)
+#     predicted_emotion = emotion_dict[predicted_label]
+#     return predicted_emotion
+
+def gradio_interface(video, audio):
+    emotion_labels = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
+    train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video, audio)
+    model = trained_model("./model_vui_ve.h5")
+    output = model.predict({"input_visual": train_visual, "input_audio_cnn": train_audio_cnn, "input_audio_wave": train_audio_wave})
+    emo_index = tf.math.argmax(output)
+    
+    return emotion_labels[emo_index]
 
-def predict_emotion_gradio(video, audio):
-    emotion_dict = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
-    predicted_label = predict_emotion(video, audio)
-    predicted_emotion = emotion_dict[predicted_label]
-    return predicted_emotion
+# iface = gr.Interface(
+#     fn=predict_emotion_gradio, 
+#     inputs=[
+#         gr.Video(label="Upload a video"
+# ),
+#         gr.Audio(label="Upload a audio")
+#     ],
+#     outputs=gr.Textbox(label="Predicted Emotion"),
+#     title="Emotion Recognition from Video",
+#     description="Upload a video and get the predicted emotion."
+# )
 
-# def gradio_interface(video, audio):
-    
-#     train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video, audio)
-#     model = trained_model("./model_vui_ve.h5")
-#     output = model.predict({"input_visual": train_visual, "input_audio_cnn": train_audio_cnn, "input_audio_wave": train_audio_wave})
-#     emo_index = tf.math.argmax(output)
-    
-#     return emotion_labels[emo_index]
+# iface.launch()
 
 iface = gr.Interface(
-    fn=predict_emotion_gradio, 
+    fn=gradio_interface,
     inputs=[
-        gr.Video(label="Upload a video"
-),
-        gr.Audio(label="Upload a audio")
+        gr.Video(),
+        gr.Audio()
     ],
-    outputs=gr.Textbox(label="Predicted Emotion"),
-    title="Emotion Recognition from Video",
-    description="Upload a video and get the predicted emotion."
+    outputs=[
+        gr.Text()
+    ],
+    live=True,
+    title="Video and Audio Processing with Emotion Recognition"
 )
-
 iface.launch()
-
-# iface = gr.Interface(
-#     fn=gradio_interface,
-#     inputs=[
-#         gr.Video(),
-#         gr.Audio()
-#     ],
-#     outputs=[
-#         gr.Text()
-#     ],
-#     live=True,
-#     title="Video and Audio Processing with Emotion Recognition"
-# )
-# iface.launch()