fusion f

app.py

@@ -71,82 +71,78 @@ class AgeGenderModel(Wav2Vec2PreTrainedModel):
     
     
     
-# == Fusion = Define Age Wav2Vec2Model's forward to accept already computed CNN7 features from Emotion
-    def _forward(
-        self,
-        extract_features,
-        attention_mask=None):
-        # extract_features : CNN7 fetures of wav2vec2 as they are calc. from CNN7 feature extractor
-
-
-        if attention_mask is not None:
-            # compute reduced attention_mask corresponding to feature vectors
-            attention_mask = self._get_feature_vector_attention_mask(
-                extract_features.shape[1], attention_mask, add_adapter=False
-            )
-
-        hidden_states, extract_features = self.feature_projection(extract_features)
-        hidden_states = self._mask_hidden_states(
-            hidden_states, mask_time_indices=mask_time_indices, attention_mask=attention_mask
+# Fusion = AgeWav2Vec2Model forward() will accept already computed CNN7 features from ExpressioNmodel forward()
+def _forward(
+    self,
+    extract_features,
+    attention_mask=None):
+    # extract_features : CNN7 fetures of wav2vec2 as they are calc. from CNN7 feature extractor
+
+
+    if attention_mask is not None:
+        # compute reduced attention_mask corresponding to feature vectors
+        attention_mask = self._get_feature_vector_attention_mask(
+            extract_features.shape[1], attention_mask, add_adapter=False
         )
 
-        encoder_outputs = self.encoder(
-            hidden_states,
-            attention_mask=attention_mask,
-            output_attentions=output_attentions,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-        )
+    hidden_states, extract_features = self.feature_projection(extract_features)
+    hidden_states = self._mask_hidden_states(
+        hidden_states, mask_time_indices=mask_time_indices, attention_mask=attention_mask
+    )
 
-        hidden_states = encoder_outputs[0]
+    encoder_outputs = self.encoder(
+        hidden_states,
+        attention_mask=attention_mask,
+        output_attentions=output_attentions,
+        output_hidden_states=output_hidden_states,
+        return_dict=return_dict,
+    )
 
-        if self.adapter is not None:
-            raise ValueError
-            hidden_states = self.adapter(hidden_states)
+    hidden_states = encoder_outputs[0]
 
-        return hidden_states
-# ===============================================
+    if self.adapter is not None:
+        raise ValueError
+        hidden_states = self.adapter(hidden_states)
 
+    return hidden_states
 
-# ================== Foward & CNN features
-    def _forward_and_cnn7(
-        self,
-        input_values,
-        attention_mask=None
-        ):
-        
 
-        frozen_cnn7 = self.feature_extractor(input_values)
-        frozen_cnn7 = frozen_cnn7.transpose(1, 2)
+def _forward_and_cnn7(
+    self,
+    input_values,
+    attention_mask=None):
 
-        if attention_mask is not None:
-            # compute reduced attention_mask corresponding to feature vectors
-            attention_mask = self._get_feature_vector_attention_mask(
-                frozen_cnn7.shape[1], attention_mask, add_adapter=False
-            )
+    frozen_cnn7 = self.feature_extractor(input_values)
+    frozen_cnn7 = frozen_cnn7.transpose(1, 2)
 
-        hidden_states, extract_features = self.feature_projection(frozen_cnn7)  # grad=True non frozen
-        hidden_states = self._mask_hidden_states(
-            hidden_states, mask_time_indices=mask_time_indices, attention_mask=attention_mask
+    if attention_mask is not None:
+        # compute reduced attention_mask corresponding to feature vectors
+        attention_mask = self._get_feature_vector_attention_mask(
+            frozen_cnn7.shape[1], attention_mask, add_adapter=False
         )
 
-        encoder_outputs = self.encoder(
-            hidden_states,
-            attention_mask=attention_mask,
-            output_attentions=output_attentions,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-        )
+    hidden_states, extract_features = self.feature_projection(frozen_cnn7)  # grad=True non frozen
+    hidden_states = self._mask_hidden_states(
+        hidden_states, mask_time_indices=mask_time_indices, attention_mask=attention_mask
+    )
+
+    encoder_outputs = self.encoder(
+        hidden_states,
+        attention_mask=attention_mask,
+        output_attentions=output_attentions,
+        output_hidden_states=output_hidden_states,
+        return_dict=return_dict,
+    )
 
-        hidden_states = encoder_outputs[0]
+    hidden_states = encoder_outputs[0]
 
-        if self.adapter is not None:
-            raise ValueError
-            hidden_states = self.adapter(hidden_states)
+    if self.adapter is not None:
+        raise ValueError
+        hidden_states = self.adapter(hidden_states)
 
-        return hidden_states, frozen_cnn7  # feature_projection is trainable thus we are unable to use the projected hidden states from official wav2vev2.forward
+    return hidden_states, frozen_cnn7  # feature_proj is trainable thus we have to access the frozen_cnn7 before projection layer
 
-# =============================
+# Fusion ============================= End
 
 
 class ExpressionHead(nn.Module):