Update app.py

app.py

@@ -10,88 +10,88 @@ from tensorflow.keras.models import load_model
 
 
 
-def trained_model(model_path):
-
-    input_visual = tf.keras.Input((120, 120, 3, 10), name="input_visual")  # 90 - 120
-    input_audio_cnn = tf.keras.Input((150, 512, 1), name="input_audio_cnn")
-    input_audio_wave = tf.keras.Input((20, 13077), name="input_audio_wave")
-
-    # Visual branch
-    x_v = tf.keras.layers.Conv3D(10, (3, 3, 3), strides=(2, 2, 1), padding='same')(input_visual)
-    x_v = tf.keras.layers.BatchNormalization()(x_v)
-    x_v = tf.keras.layers.ReLU()(x_v)
-    x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
-
-    x_v = tf.keras.layers.Conv3D(40, (3, 3, 3), strides=(2, 2, 1), padding='same')(x_v)
-    x_v = tf.keras.layers.BatchNormalization()(x_v)
-    x_v = tf.keras.layers.ReLU()(x_v)
-    x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
-
-    x_v = tf.keras.layers.Flatten()(x_v)
-
-    x_v = tf.keras.layers.Dropout(0.2)(x_v)
-    x_v = tf.keras.layers.Dense(500)(x_v)
-    x_v = tf.keras.layers.BatchNormalization()(x_v)
-    x_v = tf.keras.layers.ReLU()(x_v)
-
-    # Audio cnn branch
-    x_c = tf.keras.layers.Conv2D(5, (3, 3), strides=(2, 2), padding='same')(input_audio_cnn)
-    x_c = tf.keras.layers.BatchNormalization()(x_c)
-    x_c = tf.keras.layers.ReLU()(x_c)
-    x_c = tf.keras.layers.MaxPooling2D((3, 3))(x_c)
-
-    x_c = tf.keras.layers.Conv2D(30, (3, 3), strides=(2, 2), padding='same')(x_c)
-    x_c = tf.keras.layers.BatchNormalization()(x_c)
-    x_c = tf.keras.layers.ReLU()(x_c)
-    x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
-
-    x_c = tf.keras.layers.Conv2D(100, (3, 3), strides=(1, 1), padding='same')(x_c)
-    x_c = tf.keras.layers.BatchNormalization()(x_c)
-    x_c = tf.keras.layers.ReLU()(x_c)
-    x_c = tf.keras.layers.Conv2D(200, (3, 3), strides=(1, 1), padding='same')(x_c)
-    x_c = tf.keras.layers.BatchNormalization()(x_c)
-    x_c = tf.keras.layers.ReLU()(x_c)
-    x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
-
-    x_c = tf.keras.layers.Flatten()(x_c)
-
-    x_c = tf.keras.layers.Dropout(0.2)(x_c)
-    x_c = tf.keras.layers.Dense(500)(x_c)
-    x_c = tf.keras.layers.BatchNormalization()(x_c)
-    x_c = tf.keras.layers.ReLU()(x_c)
-
-    # Audio wave branch
-    x_w = tf.keras.layers.LSTM(500)(input_audio_wave)
-    x_w = tf.keras.layers.RepeatVector(20)(x_w)
-    x_w = tf.keras.layers.LSTM(500)(x_w)
-
-    x_w = tf.keras.layers.Flatten()(x_w)
-
-    x_w = tf.keras.layers.Dropout(0.2)(x_w)
-    x_w = tf.keras.layers.Dense(500)(x_w)
-    x_w = tf.keras.layers.BatchNormalization()(x_w)
-    x_w = tf.keras.layers.ReLU()(x_w)
-
-    # Audio fusion
-    x_a = x_c + x_w
-    x_a = tf.keras.layers.Dense(500)(x_a)
-    x_a = tf.keras.layers.BatchNormalization()(x_a)
-    x_a = tf.keras.layers.ReLU()(x_a)
-
-    # Fusion
-    x = x_a + x_v
-    x = tf.keras.layers.Dense(500)(x)
-    x = tf.keras.layers.BatchNormalization()(x)
-    x = tf.keras.layers.ReLU()(x)
-
-    # Output
-    x = tf.keras.layers.Dropout(0.1)(x) 
-    x = tf.keras.layers.Dense(6, activation='softmax', name='output_classification')(x)  # 8 - 6
+# def trained_model(model_path):
+
+#     input_visual = tf.keras.Input((120, 120, 3, 10), name="input_visual")  # 90 - 120
+#     input_audio_cnn = tf.keras.Input((150, 512, 1), name="input_audio_cnn")
+#     input_audio_wave = tf.keras.Input((20, 13077), name="input_audio_wave")
+
+#     # Visual branch
+#     x_v = tf.keras.layers.Conv3D(10, (3, 3, 3), strides=(2, 2, 1), padding='same')(input_visual)
+#     x_v = tf.keras.layers.BatchNormalization()(x_v)
+#     x_v = tf.keras.layers.ReLU()(x_v)
+#     x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
+
+#     x_v = tf.keras.layers.Conv3D(40, (3, 3, 3), strides=(2, 2, 1), padding='same')(x_v)
+#     x_v = tf.keras.layers.BatchNormalization()(x_v)
+#     x_v = tf.keras.layers.ReLU()(x_v)
+#     x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
+
+#     x_v = tf.keras.layers.Flatten()(x_v)
+
+#     x_v = tf.keras.layers.Dropout(0.2)(x_v)
+#     x_v = tf.keras.layers.Dense(500)(x_v)
+#     x_v = tf.keras.layers.BatchNormalization()(x_v)
+#     x_v = tf.keras.layers.ReLU()(x_v)
+
+#     # Audio cnn branch
+#     x_c = tf.keras.layers.Conv2D(5, (3, 3), strides=(2, 2), padding='same')(input_audio_cnn)
+#     x_c = tf.keras.layers.BatchNormalization()(x_c)
+#     x_c = tf.keras.layers.ReLU()(x_c)
+#     x_c = tf.keras.layers.MaxPooling2D((3, 3))(x_c)
+
+#     x_c = tf.keras.layers.Conv2D(30, (3, 3), strides=(2, 2), padding='same')(x_c)
+#     x_c = tf.keras.layers.BatchNormalization()(x_c)
+#     x_c = tf.keras.layers.ReLU()(x_c)
+#     x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
+
+#     x_c = tf.keras.layers.Conv2D(100, (3, 3), strides=(1, 1), padding='same')(x_c)
+#     x_c = tf.keras.layers.BatchNormalization()(x_c)
+#     x_c = tf.keras.layers.ReLU()(x_c)
+#     x_c = tf.keras.layers.Conv2D(200, (3, 3), strides=(1, 1), padding='same')(x_c)
+#     x_c = tf.keras.layers.BatchNormalization()(x_c)
+#     x_c = tf.keras.layers.ReLU()(x_c)
+#     x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
+
+#     x_c = tf.keras.layers.Flatten()(x_c)
+
+#     x_c = tf.keras.layers.Dropout(0.2)(x_c)
+#     x_c = tf.keras.layers.Dense(500)(x_c)
+#     x_c = tf.keras.layers.BatchNormalization()(x_c)
+#     x_c = tf.keras.layers.ReLU()(x_c)
+
+#     # Audio wave branch
+#     x_w = tf.keras.layers.LSTM(500)(input_audio_wave)
+#     x_w = tf.keras.layers.RepeatVector(20)(x_w)
+#     x_w = tf.keras.layers.LSTM(500)(x_w)
+
+#     x_w = tf.keras.layers.Flatten()(x_w)
+
+#     x_w = tf.keras.layers.Dropout(0.2)(x_w)
+#     x_w = tf.keras.layers.Dense(500)(x_w)
+#     x_w = tf.keras.layers.BatchNormalization()(x_w)
+#     x_w = tf.keras.layers.ReLU()(x_w)
+
+#     # Audio fusion
+#     x_a = x_c + x_w
+#     x_a = tf.keras.layers.Dense(500)(x_a)
+#     x_a = tf.keras.layers.BatchNormalization()(x_a)
+#     x_a = tf.keras.layers.ReLU()(x_a)
+
+#     # Fusion
+#     x = x_a + x_v
+#     x = tf.keras.layers.Dense(500)(x)
+#     x = tf.keras.layers.BatchNormalization()(x)
+#     x = tf.keras.layers.ReLU()(x)
+
+#     # Output
+#     x = tf.keras.layers.Dropout(0.1)(x) 
+#     x = tf.keras.layers.Dense(6, activation='softmax', name='output_classification')(x)  # 8 - 6
 
     
-    model = model.load(model_path)
+#     model = model.load(model_path)
 
-    return model
+#     return model
 
 def process_video_audio(video_path, audio_path):
 
@@ -139,60 +139,60 @@ def process_video_audio(video_path, audio_path):
 
     return train_visual, train_audio_wave, train_audio_cnn
     
-# def predict_emotion(video_path, audio_path):
-#     train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video_path, audio_path)
+def predict_emotion(video_path, audio_path):
+    train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video_path, audio_path)
     
-#     model = load_model("./model_vui_ve.h5")
-#     predictions = model.predict({
-#         "input_visual": train_visual,
-#         "input_audio_cnn": train_audio_cnn,
-#         "input_audio_wave": train_audio_wave
-#     })
+    model = load_model("model_vui_ve.h5")
+    predictions = model.predict({
+        "input_visual": train_visual,
+        "input_audio_cnn": train_audio_cnn,
+        "input_audio_wave": train_audio_wave
+    })
     
-#     predicted_label = np.argmax(predictions)
-#     return predicted_label
-
-# # Định nghĩa giao diện Gradio
-
-# def predict_emotion_gradio(video, audio):
-#     emotion_dict = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
-#     predicted_label = predict_emotion(video, audio)
-#     predicted_emotion = emotion_dict[predicted_label]
-#     return predicted_emotion
-
-def gradio_interface(video, audio):
-    emotion_labels = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
-    train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video, audio)
-    model = trained_model("./model_vui_ve.h5")
-    output = model.predict({"input_visual": train_visual, "input_audio_cnn": train_audio_cnn, "input_audio_wave": train_audio_wave})
-    emo_index = tf.math.argmax(output)
+    predicted_label = np.argmax(predictions)
+    return predicted_label
+
+# Định nghĩa giao diện Gradio
+
+def predict_emotion_gradio(video, audio):
+    emotion_dict = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
+    predicted_label = predict_emotion(video, audio)
+    predicted_emotion = emotion_dict[predicted_label]
+    return predicted_emotion
+
+# def gradio_interface(video, audio):
+#     emotion_labels = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
+#     train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video, audio)
+#     model = trained_model("./model_vui_ve.h5")
+#     output = model.predict({"input_visual": train_visual, "input_audio_cnn": train_audio_cnn, "input_audio_wave": train_audio_wave})
+#     emo_index = tf.math.argmax(output)
     
-    return emotion_labels[emo_index]
-
-# iface = gr.Interface(
-#     fn=predict_emotion_gradio, 
-#     inputs=[
-#         gr.Video(label="Upload a video"
-# ),
-#         gr.Audio(label="Upload a audio")
-#     ],
-#     outputs=gr.Textbox(label="Predicted Emotion"),
-#     title="Emotion Recognition from Video",
-#     description="Upload a video and get the predicted emotion."
-# )
-
-# iface.launch()
+#     return emotion_labels[emo_index]
 
 iface = gr.Interface(
-    fn=gradio_interface,
+    fn=predict_emotion_gradio, 
     inputs=[
-        gr.Video(),
-        gr.Audio()
-    ],
-    outputs=[
-        gr.Text()
+        gr.Video(label="Upload a video"
+),
+        gr.Audio(label="Upload a audio")
     ],
-    live=True,
-    title="Video and Audio Processing with Emotion Recognition"
+    outputs=gr.Textbox(label="Predicted Emotion"),
+    title="Emotion Recognition from Video",
+    description="Upload a video and get the predicted emotion."
 )
+
 iface.launch()
+
+# iface = gr.Interface(
+#     fn=gradio_interface,
+#     inputs=[
+#         gr.Video(),
+#         gr.Audio()
+#     ],
+#     outputs=[
+#         gr.Text()
+#     ],
+#     live=True,
+#     title="Video and Audio Processing with Emotion Recognition"
+# )
+# iface.launch()