Update app.py

app.py

@@ -8,91 +8,7 @@ import numpy as np
 import tensorflow as tf
 from tensorflow.keras.models import load_model
 
-# def trained_model(model_path):
-
-#     input_visual = tf.keras.Input((120, 120, 3, 10), name="input_visual")  # 90 - 120
-#     input_audio_cnn = tf.keras.Input((150, 512, 1), name="input_audio_cnn")
-#     input_audio_wave = tf.keras.Input((20, 13077), name="input_audio_wave")
-
-#     # Visual branch
-#     x_v = tf.keras.layers.Conv3D(10, (3, 3, 3), strides=(2, 2, 1), padding='same')(input_visual)
-#     x_v = tf.keras.layers.BatchNormalization()(x_v)
-#     x_v = tf.keras.layers.ReLU()(x_v)
-#     x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
-
-#     x_v = tf.keras.layers.Conv3D(40, (3, 3, 3), strides=(2, 2, 1), padding='same')(x_v)
-#     x_v = tf.keras.layers.BatchNormalization()(x_v)
-#     x_v = tf.keras.layers.ReLU()(x_v)
-#     x_v = tf.keras.layers.MaxPooling3D((3, 3, 1))(x_v)
-
-#     x_v = tf.keras.layers.Flatten()(x_v)
-
-#     x_v = tf.keras.layers.Dropout(0.2)(x_v)
-#     x_v = tf.keras.layers.Dense(500)(x_v)
-#     x_v = tf.keras.layers.BatchNormalization()(x_v)
-#     x_v = tf.keras.layers.ReLU()(x_v)
-
-#     # Audio cnn branch
-#     x_c = tf.keras.layers.Conv2D(5, (3, 3), strides=(2, 2), padding='same')(input_audio_cnn)
-#     x_c = tf.keras.layers.BatchNormalization()(x_c)
-#     x_c = tf.keras.layers.ReLU()(x_c)
-#     x_c = tf.keras.layers.MaxPooling2D((3, 3))(x_c)
-
-#     x_c = tf.keras.layers.Conv2D(30, (3, 3), strides=(2, 2), padding='same')(x_c)
-#     x_c = tf.keras.layers.BatchNormalization()(x_c)
-#     x_c = tf.keras.layers.ReLU()(x_c)
-#     x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
-
-#     x_c = tf.keras.layers.Conv2D(100, (3, 3), strides=(1, 1), padding='same')(x_c)
-#     x_c = tf.keras.layers.BatchNormalization()(x_c)
-#     x_c = tf.keras.layers.ReLU()(x_c)
-#     x_c = tf.keras.layers.Conv2D(200, (3, 3), strides=(1, 1), padding='same')(x_c)
-#     x_c = tf.keras.layers.BatchNormalization()(x_c)
-#     x_c = tf.keras.layers.ReLU()(x_c)
-#     x_c = tf.keras.layers.MaxPooling2D((2, 2))(x_c)
-
-#     x_c = tf.keras.layers.Flatten()(x_c)
-
-#     x_c = tf.keras.layers.Dropout(0.2)(x_c)
-#     x_c = tf.keras.layers.Dense(500)(x_c)
-#     x_c = tf.keras.layers.BatchNormalization()(x_c)
-#     x_c = tf.keras.layers.ReLU()(x_c)
-
-#     # Audio wave branch
-#     x_w = tf.keras.layers.LSTM(500)(input_audio_wave)
-#     x_w = tf.keras.layers.RepeatVector(20)(x_w)
-#     x_w = tf.keras.layers.LSTM(500)(x_w)
-
-#     x_w = tf.keras.layers.Flatten()(x_w)
-
-#     x_w = tf.keras.layers.Dropout(0.2)(x_w)
-#     x_w = tf.keras.layers.Dense(500)(x_w)
-#     x_w = tf.keras.layers.BatchNormalization()(x_w)
-#     x_w = tf.keras.layers.ReLU()(x_w)
-
-#     # Audio fusion
-#     x_a = x_c + x_w
-#     x_a = tf.keras.layers.Dense(500)(x_a)
-#     x_a = tf.keras.layers.BatchNormalization()(x_a)
-#     x_a = tf.keras.layers.ReLU()(x_a)
-
-#     # Fusion
-#     x = x_a + x_v
-#     x = tf.keras.layers.Dense(500)(x)
-#     x = tf.keras.layers.BatchNormalization()(x)
-#     x = tf.keras.layers.ReLU()(x)
-
-#     # Output
-#     x = tf.keras.layers.Dropout(0.1)(x) 
-#     x = tf.keras.layers.Dense(6, activation='softmax', name='output_classification')(x)  # 8 - 6
-
-    
-#     model = model.load(model_path)
-
-#     return model
-
 def process_video_audio(video_path, audio_path):
-
     wav = pt.tensor(list(audio_path[1]))
     
     train_visual = pt.zeros([1, 120, 120, 3, 10])
@@ -133,16 +49,16 @@ def process_video_audio(video_path, audio_path):
             frame_idx += 1
     cap.release()
     
-    train_visual = tf.convert_to_tensor(train_visual, dtype=tf.float16)
-    train_audio_wave = tf.reshape(tf.convert_to_tensor(train_audio_wave, dtype=tf.float16), (1, 20, 13077))
-    train_audio_cnn = tf.convert_to_tensor(train_audio_cnn, dtype=tf.float16)
+    train_visual = tf.convert_to_tensor(train_visual.numpy(), dtype=tf.float16)
+    train_audio_wave = tf.reshape(tf.convert_to_tensor(train_audio_wave.numpy(), dtype=tf.float16), (1, 20, 13077))
+    train_audio_cnn = tf.convert_to_tensor(train_audio_cnn.numpy(), dtype=tf.float16)
 
     return last_frame, train_visual, train_audio_wave, train_audio_cnn
     
 def predict_emotion(video_path, audio_path):
     last_frame, train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video_path, audio_path)
     
-    model = load_model("model_vui_ve.h5")
+    model = load_model("model_vui_ve.h5", compile=False)
     predictions = model.predict({
         "input_visual": train_visual,
         "input_audio_cnn": train_audio_cnn,
@@ -152,23 +68,12 @@ def predict_emotion(video_path, audio_path):
     predicted_label = np.argmax(predictions)
     return last_frame, predicted_label
 
-# Định nghĩa giao diện Gradio
-
 def predict_emotion_gradio(video_path, audio_path):
     emotion_dict = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
     last_frame, predicted_label = predict_emotion(video_path, audio_path)
     predicted_emotion = emotion_dict[predicted_label]
     return last_frame, predicted_emotion
 
-# def gradio_interface(video, audio):
-#     emotion_labels = {0: 'neutral', 1: 'calm', 2: 'happy', 3: 'sad', 4: 'angry', 5: 'fearful'}
-#     train_visual, train_audio_wave, train_audio_cnn = process_video_audio(video, audio)
-#     model = trained_model("./model_vui_ve.h5")
-#     output = model.predict({"input_visual": train_visual, "input_audio_cnn": train_audio_cnn, "input_audio_wave": train_audio_wave})
-#     emo_index = tf.math.argmax(output)
-    
-#     return emotion_labels[emo_index]
-
 iface = gr.Interface(
     fn=predict_emotion_gradio, 
     inputs=[
@@ -176,24 +81,11 @@ iface = gr.Interface(
         gr.Audio(label="Upload a audio")
     ],
     outputs=[
-        gr.Textbox(label="Predicted Emotion"),
-        gr.Image(label = "image frame last")],
+        gr.Image(label="Last Frame"),
+        gr.Textbox(label="Predicted Emotion")
+    ],
     title="Emotion Recognition from Video",
     description="Upload a video and get the predicted emotion."
 )
 
 iface.launch()
-
-# iface = gr.Interface(
-#     fn=gradio_interface,
-#     inputs=[
-#         gr.Video(),
-#         gr.Audio()
-#     ],
-#     outputs=[
-#         gr.Text()
-#     ],
-#     live=True,
-#     title="Video and Audio Processing with Emotion Recognition"
-# )
-# iface.launch()