add styles

mimic3_make_harvard_sentences.py

@@ -77,6 +77,21 @@ list_voices = [
 
 
 # ================================================== INTERFACE MODELS
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 LABELS = [
     'arousal', 'dominance', 'valence',
     # 'speech_synthesizer', 'synthetic_singing',
@@ -131,10 +146,77 @@ teacher_cat.forward = types.MethodType(_infer, teacher_cat)
 
 
 
-# Audioset & ADV
+# ===================[:]===================== Dawn
+def _prenorm(x, attention_mask=None):
+    '''mean/var'''
+    if attention_mask is not None:
+        N = attention_mask.sum(1, keepdim=True)  # here attn msk is unprocessed just the original input
+        x -= x.sum(1, keepdim=True) / N
+        var = (x * x).sum(1, keepdim=True) / N
+
+    else:
+        x -= x.mean(1, keepdim=True)  # mean is an onnx operator reducemean saves some ops compared to casting integer N to float and the div
+        var = (x * x).mean(1, keepdim=True)
+    return x / torch.sqrt(var + 1e-7)
+
+from torch import nn
+from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2PreTrainedModel, Wav2Vec2Model
+class RegressionHead(nn.Module):
+        r"""Classification head."""
+
+        def __init__(self, config):
+
+            super().__init__()
+
+            self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+            self.dropout = nn.Dropout(config.final_dropout)
+            self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
+
+        def forward(self, features, **kwargs):
+
+            x = features
+            x = self.dropout(x)
+            x = self.dense(x)
+            x = torch.tanh(x)
+            x = self.dropout(x)
+            x = self.out_proj(x)
+
+            return x
+
+
+class Dawn(Wav2Vec2PreTrainedModel):
+    r"""Speech emotion classifier."""
+
+    def __init__(self, config):
+
+        super().__init__(config)
+
+        self.config = config
+        self.wav2vec2 = Wav2Vec2Model(config)
+        self.classifier = RegressionHead(config)
+        self.init_weights()
+
+    def forward(
+            self,
+            input_values,
+            attention_mask=None,
+    ):
+        x = _prenorm(input_values, attention_mask=attention_mask)
+        outputs = self.wav2vec2(x, attention_mask=attention_mask)
+        hidden_states = outputs[0]
+        hidden_states = torch.mean(hidden_states, dim=1)
+        logits = self.classifier(hidden_states)
+        return logits
+        # return {'hidden_states': hidden_states,
+        #         'logits': logits}
+dawn = Dawn.from_pretrained('audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim').to(config.dev).eval()
+# =======================================
+
+
+
+
+
 
-# audioset_model = audonnx.load(audmodel.load('17c240ec-1.0.0'), device='cuda:0')
-adv_model = audonnx.load(audmodel.load('90398682-2.0.0'), device='cuda:0')
 
 def process_function(x, sampling_rate, idx):
     '''run audioset ct, adv
@@ -154,7 +236,7 @@ def process_function(x, sampling_rate, idx):
     # logits_audioset = audioset_model(x, 16000)['logits_sounds']
     # logits_audioset = logits_audioset[:, [7, 35]]  # speech synthesizer synthetic singing
     # --
-    logits_adv = adv_model(x, 16000)['logits']
+    logits_adv = dawn(torch.from_numpy(x).to(config.dev)).cpu().detach().numpy() #['logits']
     
     cat = np.concatenate([logits_adv,
                         #   _sigmoid(logits_audioset),
@@ -169,7 +251,7 @@ interface = audinterface.Feature(
     # process_func_args={'outputs': 'logits_scene'},
     process_func_applies_sliding_window=False,
     win_dur=7.0,
-    hop_dur=4.0,
+    hop_dur=40.0,
     sampling_rate=16000,
     resample=True,
     verbose=True,
@@ -297,7 +379,7 @@ for _id, _voice in enumerate(list_voices):
         total_audio_mimic3 = []
         total_audio_styletts2 = []
         ix = 0
-        for list_of_10 in harvard_individual_sentences[:1000]:  # 77
+        for list_of_10 in harvard_individual_sentences[:4]:  # 77
             
             text = ' '.join(list_of_10['sentences'])