WavLM

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+SER-Odyssey/Baseline_Model.pdf filter=lfs diff=lfs merge=lfs -text
+SER-Odyssey/Odyssey[[:space:]]2024[[:space:]]-[[:space:]]Speech[[:space:]]Emotion[[:space:]]Recognition[[:space:]]Challenge.[[:space:]]Dataset,[[:space:]]Baseline,[[:space:]]Framework,[[:space:]]and[[:space:]]Results.pdf filter=lfs diff=lfs merge=lfs -text
+SER-Odyssey/SER-WavLM-Multi-Attributes/tensorrt/trt10_ser_fp16.plan filter=lfs diff=lfs merge=lfs -text
+SER-Odyssey/SER-WavLM-Multi-Attributes/tensorrt/trt8_ser_dyn_fp16.plan filter=lfs diff=lfs merge=lfs -text
+wavlm-large-mnn/wavlm_large_fp16.mnn filter=lfs diff=lfs merge=lfs -text
+wavlm-large-mnn/wavlm_large_int8.mnn filter=lfs diff=lfs merge=lfs -text
+WavLM.[[:space:]]Large-Scale[[:space:]]Self-Supervised[[:space:]]Pre-Training[[:space:]]for[[:space:]]Full[[:space:]]Stack[[:space:]]Speech[[:space:]]Processing.pdf filter=lfs diff=lfs merge=lfs -text

SER-Odyssey/Baseline_Model.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2006e79620902e9b411dd8e110f296c9e7d2458110faa8043d900187f203e103
+size 460836

SER-Odyssey/MSP-Podcast_Challenge [JMasr] +48 -24.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b53455da505412b271968d94febb011505ef41201826ba048dc7308306838a04
+size 895217

SER-Odyssey/MSP-Podcast_Challenge.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c86e9700cc05734431656503b7602aa0a5f9b60be4a5a02238e87121324055a7
+size 897745

SER-Odyssey/Odyssey 2024 - Speech Emotion Recognition Challenge. Dataset, Baseline, Framework, and Results.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11f79ccab188b27218b3c5038fbec0ef21e0dca0d08af3d998e77b993d0ed31c
+size 1083858

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Arousal/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Arousal/README.md

@@ -0,0 +1,83 @@
+---
+license: mit
+language:
+- en
+pipeline_tag: audio-classification
+tags:
+- wavlm
+- msp-podcast
+- emotion-recognition
+- audio
+- speech
+- arousal
+- lucas
+- speech-emotion-recognition
+---
+The model was trained on [MSP-Podcast](https://ecs.utdallas.edu/research/researchlabs/msp-lab/MSP-Podcast.html) for the Odyssey 2024 Emotion Recognition competition baseline<br>
+This particular model is the single-task specialized arousal model, which predict arousal in a range of approximately 0...1. 
+
+
+
+# Benchmarks
+CCC based on Test3 and Development sets of the Odyssey Competition
+<table style="width:500px">
+  <tr><th colspan=2 align="center"> Sinle-Task Setup </th></tr>
+  <tr><th colspan=1 align="center">Test 3</th><th colspan=1 align="center">Development</th></tr>
+  <tr>   <td align="center">Aro</td> <td align="center">Aro</td>  </tr>
+  <tr>  <td align="center"> 0.566</td>  <td align="center" >0.651 </td>  </tr>
+</table>
+ 
+
+
+For more details:  [demo](https://huggingface.co/spaces/3loi/WavLM-SER-Multi-Baseline-Odyssey2024), [paper](https://ecs.utdallas.edu/research/researchlabs/msp-lab/publications/Goncalves_2024.pdf), and [GitHub](https://github.com/MSP-UTD/MSP-Podcast_Challenge/tree/main).
+
+
+```
+@InProceedings{Goncalves_2024,
+            author={L. Goncalves and A. N. Salman and A. {Reddy Naini} and L. Moro-Velazquez and T. Thebaud and L. {Paola Garcia} and N. Dehak and B. Sisman and C. Busso},
+            title={Odyssey2024 - Speech Emotion Recognition Challenge: Dataset, Baseline Framework, and Results},
+            booktitle={Odyssey 2024: The Speaker and Language Recognition Workshop)},
+            volume={To appear},
+            year={2024},
+            month={June},
+            address =  {Quebec, Canada},
+}
+```
+
+
+# Usage
+```python
+from transformers import AutoModelForAudioClassification
+import librosa, torch
+
+#load model
+model = AutoModelForAudioClassification.from_pretrained("3loi/SER-Odyssey-Baseline-WavLM-Arousal", trust_remote_code=True)
+
+#get mean/std
+mean = model.config.mean
+std = model.config.std
+
+
+#load an audio file
+audio_path = "/path/to/audio.wav"
+raw_wav, _ = librosa.load(audio_path, sr=model.config.sampling_rate)
+
+#normalize the audio by mean/std
+norm_wav = (raw_wav - mean) / (std+0.000001)
+
+#generate the mask
+mask = torch.ones(1, len(norm_wav))
+
+#batch it (add dim)
+wavs = torch.tensor(norm_wav).unsqueeze(0)
+
+
+#predict
+with torch.no_grad():
+    pred = model(wavs, mask)
+
+print(model.config.id2label) 
+print(pred)
+#{0: 'arousal'}
+#tensor([[0.3670]])
+```

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Arousal/config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "SERModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "pipeline_utils.SERConfig",
+    "AutoModelForAudioClassification": "pipeline_utils.SERModel"
+  },
+  "id2label": {
+    "0": "arousal"
+  },
+  "sampling_rate": 16000,
+  "maxlen": 192000,
+  "mean": -8.278621631819787e-05, 
+  "std": 0.08485510250851999,
+  "classifier_dropout_prob": 0.5,
+  "classifier_hidden_layers": 1,
+  "hidden_size": 1024,
+  "model_type": "ser",
+  "num_attention_heads": 16,
+  "num_classes": 1,
+  "num_hidden_layers": 24,
+  "ssl_type": "microsoft/wavlm-large",
+  "torch_dtype": "float32",
+  "transformers_version": "4.34.0.dev0"
+}

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Arousal/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6513eca66ff2f599b248059ad44c41fef39d61b5cfc4995f777022c42c07106c
+size 1274482316

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Arousal/pipeline_utils.py

@@ -0,0 +1,165 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import AutoModel
+from transformers.modeling_utils import PreTrainedModel ,PretrainedConfig
+
+
+class Pooling(nn.Module):
+    def __init__(self):
+        super().__init__()
+    def compute_length_from_mask(self, mask):
+        """
+        mask: (batch_size, T)
+        Assuming that the sampling rate is 16kHz, the frame shift is 20ms
+        """
+        wav_lens = torch.sum(mask, dim=1) # (batch_size, )
+        feat_lens = torch.div(wav_lens-1, 16000*0.02, rounding_mode="floor") + 1
+        feat_lens = feat_lens.int().tolist()
+        return feat_lens
+        
+    def forward(self, x, mask):
+        raise NotImplementedError
+    
+class MeanPooling(Pooling):
+    def __init__(self):
+        super().__init__()
+    def forward(self, xs, mask):
+        """
+        xs: (batch_size, T, feat_dim)
+        mask: (batch_size, T)
+
+        => output: (batch_size, feat_dim)
+        """
+        feat_lens = self.compute_length_from_mask(mask)
+        pooled_list = []
+        for x, feat_len in zip(xs, feat_lens):
+            pooled = torch.mean(x[:feat_len], dim=0) # (feat_dim, )
+            pooled_list.append(pooled)
+        pooled = torch.stack(pooled_list, dim=0) # (batch_size, feat_dim)
+        return pooled
+    
+
+class AttentiveStatisticsPooling(Pooling):
+    """
+    AttentiveStatisticsPooling
+    Paper: Attentive Statistics Pooling for Deep Speaker Embedding
+    Link: https://arxiv.org/pdf/1803.10963.pdf
+    """
+    def __init__(self, input_size):
+        super().__init__()
+        self._indim = input_size
+        self.sap_linear = nn.Linear(input_size, input_size)
+        self.attention = nn.Parameter(torch.FloatTensor(input_size, 1))
+        torch.nn.init.normal_(self.attention, mean=0, std=1)
+
+    def forward(self, xs, mask):
+        """
+        xs: (batch_size, T, feat_dim)
+        mask: (batch_size, T)
+
+        => output: (batch_size, feat_dim*2)
+        """
+        feat_lens = self.compute_length_from_mask(mask)
+        pooled_list = []
+        for x, feat_len in zip(xs, feat_lens):
+            x = x[:feat_len].unsqueeze(0)
+            h = torch.tanh(self.sap_linear(x))
+            w = torch.matmul(h, self.attention).squeeze(dim=2)
+            w = F.softmax(w, dim=1).view(x.size(0), x.size(1), 1)
+            mu = torch.sum(x * w, dim=1)
+            rh = torch.sqrt((torch.sum((x**2) * w, dim=1) - mu**2).clamp(min=1e-5))
+            x = torch.cat((mu, rh), 1).squeeze(0)
+            pooled_list.append(x)
+        return torch.stack(pooled_list)
+
+
+
+    
+class EmotionRegression(nn.Module):
+    def __init__(self, *args, **kwargs):
+        super(EmotionRegression, self).__init__()
+        input_dim = args[0]
+        hidden_dim = args[1]
+        num_layers = args[2]
+        output_dim = args[3]
+        p = kwargs.get("dropout", 0.5)
+
+        self.fc=nn.ModuleList([
+            nn.Sequential(
+                nn.Linear(input_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.ReLU(), nn.Dropout(p)
+            )
+        ])
+        for lidx in range(num_layers-1):
+            self.fc.append(
+                nn.Sequential(
+                    nn.Linear(hidden_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.ReLU(), nn.Dropout(p)
+                )
+            )
+        self.out = nn.Sequential(
+            nn.Linear(hidden_dim, output_dim)
+        )
+        
+        self.inp_drop = nn.Dropout(p)
+    def get_repr(self, x):
+        h = self.inp_drop(x)
+        for lidx, fc in enumerate(self.fc):
+            h=fc(h)
+        return h
+    
+    def forward(self, x):
+        h=self.get_repr(x)
+        result = self.out(h)
+        return result
+    
+class SERConfig(PretrainedConfig):
+    model_type = "ser"
+
+    def __init__(
+        self,
+        num_classes: int = 1,
+        num_attention_heads = 16,
+        num_hidden_layers = 24,
+        hidden_size = 1024,
+        classifier_hidden_layers = 1,
+        classifier_dropout_prob  = 0.5,
+        ssl_type= "microsoft/wavlm-large",
+        torch_dtype= "float32",
+        **kwargs,
+    ):
+        self.num_classes = num_classes
+        self.num_attention_heads = num_attention_heads
+        self.num_hidden_layers = num_hidden_layers
+        self.hidden_size = hidden_size
+        self.classifier_hidden_layers = classifier_hidden_layers
+        self.classifier_dropout_prob = classifier_dropout_prob
+        self.ssl_type = ssl_type
+        self.torch_dtype = torch_dtype
+        super().__init__(**kwargs)
+        
+class SERModel(PreTrainedModel):
+    config_class = SERConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+        self.ssl_model = AutoModel.from_pretrained(config.ssl_type)
+        self.ssl_model.freeze_feature_encoder()
+        
+        self.pool_model = AttentiveStatisticsPooling(config.hidden_size)
+    
+        self.ser_model = EmotionRegression(config.hidden_size*2, 
+                                               config.hidden_size, 
+                                               config.classifier_hidden_layers, 
+                                               config.num_classes, 
+                                               dropout=config.classifier_dropout_prob)
+        
+        
+    def forward(self, x, mask):
+        ssl = self.ssl_model(x, attention_mask=mask).last_hidden_state
+
+        ssl = self.pool_model(ssl, mask)
+        
+        pred = self.ser_model(ssl)
+        
+        return pred
+    

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Arousal/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f8a77ea0603b9ab91b3ce1d03c165db58d95ebae1c4210ab12dc94459c36b60
+size 1274585617

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Categorical/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Categorical/README.md

@@ -0,0 +1,87 @@
+---
+license: mit
+language:
+- en
+pipeline_tag: audio-classification
+tags:
+- wavlm
+- msp-podcast
+- emotion-recognition
+- audio
+- speech
+- categorical
+- lucas
+- speech-emotion-recognition
+---
+The model was trained on [MSP-Podcast](https://ecs.utdallas.edu/research/researchlabs/msp-lab/MSP-Podcast.html) for the Odyssey 2024 Emotion Recognition competition baseline<br>
+This particular model is the categorical based model which predicts: "Angry", "Sad", "Happy", "Surprise", "Fear", "Disgust", "Contempt" and "Neutral".
+
+
+# Benchmarks
+F1-scores based on Test3 and Development sets of the Odyssey Competition
+<table style="width:500px">
+  <tr><th colspan=8 align="center" >Categorical Setup</th></tr>
+  <tr><th colspan=4 align="center">Test 3</th><th colspan=4 align="center">Development</th></tr>
+  <tr>   <td>F1-Mic.</td> <td>F1-Ma.</td> <td>Prec.</td> <td>Rec.</td>   <td>F1-Mic.</td> <td>F1-Ma.</td> <td>Prec.</td> <td>Rec.</td> </tr>
+  <tr>  <td> 0.327</td> <td>0.311</td> <td>0.332</td> <td>0.325</td>     <td>0.409</td> <td>0.307</td>  <td>0.316</td> <td>0.345</td> </tr>
+</table>
+ 
+
+       
+For more details:  [demo](https://huggingface.co/spaces/3loi/WavLM-SER-Multi-Baseline-Odyssey2024), [paper](https://ecs.utdallas.edu/research/researchlabs/msp-lab/publications/Goncalves_2024.pdf), and [GitHub](https://github.com/MSP-UTD/MSP-Podcast_Challenge/tree/main).
+
+
+```
+@InProceedings{Goncalves_2024,
+            author={L. Goncalves and A. N. Salman and A. {Reddy Naini} and L. Moro-Velazquez and T. Thebaud and L. {Paola Garcia} and N. Dehak and B. Sisman and C. Busso},
+            title={Odyssey2024 - Speech Emotion Recognition Challenge: Dataset, Baseline Framework, and Results},
+            booktitle={Odyssey 2024: The Speaker and Language Recognition Workshop)},
+            volume={To appear},
+            year={2024},
+            month={June},
+            address =  {Quebec, Canada},
+}
+```
+
+
+# Usage
+```python
+from transformers import AutoModelForAudioClassification
+import librosa, torch
+
+#load model
+model = AutoModelForAudioClassification.from_pretrained("3loi/SER-Odyssey-Baseline-WavLM-Categorical-Attributes", trust_remote_code=True)
+
+#get mean/std
+mean = model.config.mean
+std = model.config.std
+
+
+#load an audio file
+audio_path = "/path/to/audio.wav"
+raw_wav, _ = librosa.load(audio_path, sr=model.config.sampling_rate)
+
+#normalize the audio by mean/std
+norm_wav = (raw_wav - mean) / (std+0.000001)
+
+#generate the mask
+mask = torch.ones(1, len(norm_wav))
+
+#batch it (add dim)
+wavs = torch.tensor(norm_wav).unsqueeze(0)
+
+
+#predict
+with torch.no_grad():
+    pred = model(wavs, mask)
+
+print(model.config.id2label)  
+print(pred)
+#{0: 'Angry', 1: 'Sad', 2: 'Happy', 3: 'Surprise', 4: 'Fear', 5: 'Disgust', 6: 'Contempt', 7: 'Neutral'}
+#tensor([[0.0015, 0.3651, 0.0593, 0.0315, 0.0600, 0.0125, 0.0319, 0.4382]])
+
+#convert logits to probability
+probabilities = torch.nn.functional.softmax(pred, dim=1)
+print(probabilities)
+#[[0.0015, 0.3651, 0.0593, 0.0315, 0.0600, 0.0125, 0.0319, 0.4382]]
+```

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Categorical/config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "SERModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "pipeline_utils.SERConfig",
+    "AutoModelForAudioClassification": "pipeline_utils.SERModel"
+  },
+  "id2label": {
+      "0": "Angry",
+      "1": "Sad",
+      "2": "Happy",
+      "3": "Surprise",
+      "4": "Fear",
+      "5": "Disgust",
+      "6": "Contempt",
+      "7": "Neutral"
+  },
+  "sampling_rate": 16000,
+  "classifier_dropout_prob": 0.5,
+  "classifier_hidden_layers": 1,
+  "hidden_size": 1024,
+  "mean": -8.278621631819787e-05,
+  "model_type": "ser",
+  "num_attention_heads": 16,
+  "num_classes": 8,
+  "num_hidden_layers": 24,
+  "ssl_type": "microsoft/wavlm-large",
+  "std": 0.08485510250851999,
+  "torch_dtype": "float32",
+  "transformers_version": "4.34.0.dev0"
+}

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Categorical/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb52f3f472b6a5a824ac238537fa60bf39a73d74b3fa5f4a4473c012cb3d18f4
+size 1274511016

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Categorical/pipeline_utils.py

@@ -0,0 +1,171 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import AutoModel
+from transformers.modeling_utils import PreTrainedModel ,PretrainedConfig
+
+
+class Pooling(nn.Module):
+    def __init__(self):
+        super().__init__()
+    def compute_length_from_mask(self, mask):
+        """
+        mask: (batch_size, T)
+        Assuming that the sampling rate is 16kHz, the frame shift is 20ms
+        """
+        wav_lens = torch.sum(mask, dim=1) # (batch_size, )
+        feat_lens = torch.div(wav_lens-1, 16000*0.02, rounding_mode="floor") + 1
+        feat_lens = feat_lens.int().tolist()
+        return feat_lens
+        
+    def forward(self, x, mask):
+        raise NotImplementedError
+    
+class MeanPooling(Pooling):
+    def __init__(self):
+        super().__init__()
+    def forward(self, xs, mask):
+        """
+        xs: (batch_size, T, feat_dim)
+        mask: (batch_size, T)
+
+        => output: (batch_size, feat_dim)
+        """
+        feat_lens = self.compute_length_from_mask(mask)
+        pooled_list = []
+        for x, feat_len in zip(xs, feat_lens):
+            pooled = torch.mean(x[:feat_len], dim=0) # (feat_dim, )
+            pooled_list.append(pooled)
+        pooled = torch.stack(pooled_list, dim=0) # (batch_size, feat_dim)
+        return pooled
+    
+
+class AttentiveStatisticsPooling(Pooling):
+    """
+    AttentiveStatisticsPooling
+    Paper: Attentive Statistics Pooling for Deep Speaker Embedding
+    Link: https://arxiv.org/pdf/1803.10963.pdf
+    """
+    def __init__(self, input_size):
+        super().__init__()
+        self._indim = input_size
+        self.sap_linear = nn.Linear(input_size, input_size)
+        self.attention = nn.Parameter(torch.FloatTensor(input_size, 1))
+        torch.nn.init.normal_(self.attention, mean=0, std=1)
+
+    def forward(self, xs, mask):
+        """
+        xs: (batch_size, T, feat_dim)
+        mask: (batch_size, T)
+
+        => output: (batch_size, feat_dim*2)
+        """
+        feat_lens = self.compute_length_from_mask(mask)
+        pooled_list = []
+        for x, feat_len in zip(xs, feat_lens):
+            x = x[:feat_len].unsqueeze(0)
+            h = torch.tanh(self.sap_linear(x))
+            w = torch.matmul(h, self.attention).squeeze(dim=2)
+            w = F.softmax(w, dim=1).view(x.size(0), x.size(1), 1)
+            mu = torch.sum(x * w, dim=1)
+            rh = torch.sqrt((torch.sum((x**2) * w, dim=1) - mu**2).clamp(min=1e-5))
+            x = torch.cat((mu, rh), 1).squeeze(0)
+            pooled_list.append(x)
+        return torch.stack(pooled_list)
+
+
+
+    
+class EmotionRegression(nn.Module):
+    def __init__(self, *args, **kwargs):
+        super(EmotionRegression, self).__init__()
+        input_dim = args[0]
+        hidden_dim = args[1]
+        num_layers = args[2]
+        output_dim = args[3]
+        p = kwargs.get("dropout", 0.5)
+
+        self.fc=nn.ModuleList([
+            nn.Sequential(
+                nn.Linear(input_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.ReLU(), nn.Dropout(p)
+            )
+        ])
+        for lidx in range(num_layers-1):
+            self.fc.append(
+                nn.Sequential(
+                    nn.Linear(hidden_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.ReLU(), nn.Dropout(p)
+                )
+            )
+        self.out = nn.Sequential(
+            nn.Linear(hidden_dim, output_dim)
+        )
+        
+        self.inp_drop = nn.Dropout(p)
+    def get_repr(self, x):
+        h = self.inp_drop(x)
+        for lidx, fc in enumerate(self.fc):
+            h=fc(h)
+        return h
+    
+    def forward(self, x):
+        h=self.get_repr(x)
+        result = self.out(h)
+        return result
+
+
+class SERConfig(PretrainedConfig):
+    model_type = "ser"
+
+    def __init__(
+        self,
+        num_classes: int = 8,
+        num_attention_heads = 16,
+        num_hidden_layers = 24,
+        hidden_size = 1024,
+        classifier_hidden_layers = 1,
+        classifier_dropout_prob  = 0.5,
+        ssl_type= "microsoft/wavlm-large",
+        torch_dtype= "float32",
+        mean= -8.278621631819787e-05,
+        std=0.08485510250851999,
+        **kwargs,
+    ):
+        self.num_classes = num_classes
+        self.num_attention_heads = num_attention_heads
+        self.num_hidden_layers = num_hidden_layers
+        self.hidden_size = hidden_size
+        self.classifier_hidden_layers = classifier_hidden_layers
+        self.classifier_dropout_prob = classifier_dropout_prob
+        self.ssl_type = ssl_type
+        self.torch_dtype = torch_dtype
+        
+        self.mean = mean
+        self.std = std
+        super().__init__(**kwargs)
+        
+class SERModel(PreTrainedModel):
+    config_class = SERConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+        self.ssl_model = AutoModel.from_pretrained(config.ssl_type)
+        self.ssl_model.freeze_feature_encoder()
+        
+        self.pool_model = AttentiveStatisticsPooling(config.hidden_size)
+    
+        self.ser_model = EmotionRegression(config.hidden_size*2, 
+                                               config.hidden_size, 
+                                               config.classifier_hidden_layers, 
+                                               config.num_classes, 
+                                               dropout=config.classifier_dropout_prob)
+        
+        
+    def forward(self, x, mask):
+        ssl = self.ssl_model(x, attention_mask=mask).last_hidden_state
+
+        ssl = self.pool_model(ssl, mask)
+        
+        pred = self.ser_model(ssl)
+        
+        return pred
+    

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Categorical/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:446f71c92a67b69977c50b065a0e418c37fa20aba1d2e44ecb1190d97f9c0cbb
+size 1274614289

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Dominance/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Dominance/README.md

@@ -0,0 +1,83 @@
+---
+license: mit
+language:
+- en
+pipeline_tag: audio-classification
+tags:
+- wavlm
+- msp-podcast
+- emotion-recognition
+- audio
+- speech
+- dominance
+- lucas
+- speech-emotion-recognition
+---
+The model was trained on [MSP-Podcast](https://ecs.utdallas.edu/research/researchlabs/msp-lab/MSP-Podcast.html) for the Odyssey 2024 Emotion Recognition competition baseline<br>
+This particular model is the single-task specialized dominance model, which predict dominance in a range of approximately 0...1. 
+
+
+
+# Benchmarks
+CCC based on Test3 and Development sets of the Odyssey Competition
+<table style="width:500px">
+  <tr><th colspan=2 align="center"> Sinle-Task Setup </th></tr>
+  <tr><th colspan=1 align="center">Test 3</th><th colspan=1 align="center">Development</th></tr>
+  <tr>   <td align="center">Dom</td> <td align="center">Dom</td>  </tr>
+  <tr>  <td align="center"> 0.424</td>  <td align="center" >0.584 </td>  </tr>
+</table>
+ 
+
+
+For more details:  [demo](https://huggingface.co/spaces/3loi/WavLM-SER-Multi-Baseline-Odyssey2024), [paper](https://ecs.utdallas.edu/research/researchlabs/msp-lab/publications/Goncalves_2024.pdf) and [GitHub](https://github.com/MSP-UTD/MSP-Podcast_Challenge/tree/main).
+
+
+```
+@InProceedings{Goncalves_2024,
+            author={L. Goncalves and A. N. Salman and A. {Reddy Naini} and L. Moro-Velazquez and T. Thebaud and L. {Paola Garcia} and N. Dehak and B. Sisman and C. Busso},
+            title={Odyssey2024 - Speech Emotion Recognition Challenge: Dataset, Baseline Framework, and Results},
+            booktitle={Odyssey 2024: The Speaker and Language Recognition Workshop)},
+            volume={To appear},
+            year={2024},
+            month={June},
+            address =  {Quebec, Canada},
+}
+```
+
+
+# Usage
+```python
+from transformers import AutoModelForAudioClassification
+import librosa, torch
+
+#load model
+model = AutoModelForAudioClassification.from_pretrained("3loi/SER-Odyssey-Baseline-WavLM-Dominance", trust_remote_code=True)
+
+#get mean/std
+mean = model.config.mean
+std = model.config.std
+
+
+#load an audio file
+audio_path = "/path/to/audio.wav"
+raw_wav, _ = librosa.load(audio_path, sr=model.config.sampling_rate)
+
+#normalize the audio by mean/std
+norm_wav = (raw_wav - mean) / (std+0.000001)
+
+#generate the mask
+mask = torch.ones(1, len(norm_wav))
+
+#batch it (add dim)
+wavs = torch.tensor(norm_wav).unsqueeze(0)
+
+
+#predict
+with torch.no_grad():
+    pred = model(wavs, mask)
+
+print(model.config.id2label) 
+print(pred)
+#{0: 'dominance'}
+#tensor([[0.3670]])
+```

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Dominance/config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "SERModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "pipeline_utils.SERConfig",
+    "AutoModelForAudioClassification": "pipeline_utils.SERModel"
+  },
+  "id2label": {
+    "0": "dominance"
+  },
+  "sampling_rate": 16000,
+  "maxlen": 192000,
+  "mean": -8.278621631819787e-05, 
+  "std": 0.08485510250851999,
+  "classifier_dropout_prob": 0.5,
+  "classifier_hidden_layers": 1,
+  "hidden_size": 1024,
+  "model_type": "ser",
+  "num_attention_heads": 16,
+  "num_classes": 1,
+  "num_hidden_layers": 24,
+  "ssl_type": "microsoft/wavlm-large",
+  "torch_dtype": "float32",
+  "transformers_version": "4.34.0.dev0"
+}

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Dominance/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5279e2387d029fb3c7529830546a876518bc32e264d61a21a593d708c9491e0
+size 1274482316

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Dominance/pipeline_utils.py

@@ -0,0 +1,165 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import AutoModel
+from transformers.modeling_utils import PreTrainedModel ,PretrainedConfig
+
+
+class Pooling(nn.Module):
+    def __init__(self):
+        super().__init__()
+    def compute_length_from_mask(self, mask):
+        """
+        mask: (batch_size, T)
+        Assuming that the sampling rate is 16kHz, the frame shift is 20ms
+        """
+        wav_lens = torch.sum(mask, dim=1) # (batch_size, )
+        feat_lens = torch.div(wav_lens-1, 16000*0.02, rounding_mode="floor") + 1
+        feat_lens = feat_lens.int().tolist()
+        return feat_lens
+        
+    def forward(self, x, mask):
+        raise NotImplementedError
+    
+class MeanPooling(Pooling):
+    def __init__(self):
+        super().__init__()
+    def forward(self, xs, mask):
+        """
+        xs: (batch_size, T, feat_dim)
+        mask: (batch_size, T)
+
+        => output: (batch_size, feat_dim)
+        """
+        feat_lens = self.compute_length_from_mask(mask)
+        pooled_list = []
+        for x, feat_len in zip(xs, feat_lens):
+            pooled = torch.mean(x[:feat_len], dim=0) # (feat_dim, )
+            pooled_list.append(pooled)
+        pooled = torch.stack(pooled_list, dim=0) # (batch_size, feat_dim)
+        return pooled
+    
+
+class AttentiveStatisticsPooling(Pooling):
+    """
+    AttentiveStatisticsPooling
+    Paper: Attentive Statistics Pooling for Deep Speaker Embedding
+    Link: https://arxiv.org/pdf/1803.10963.pdf
+    """
+    def __init__(self, input_size):
+        super().__init__()
+        self._indim = input_size
+        self.sap_linear = nn.Linear(input_size, input_size)
+        self.attention = nn.Parameter(torch.FloatTensor(input_size, 1))
+        torch.nn.init.normal_(self.attention, mean=0, std=1)
+
+    def forward(self, xs, mask):
+        """
+        xs: (batch_size, T, feat_dim)
+        mask: (batch_size, T)
+
+        => output: (batch_size, feat_dim*2)
+        """
+        feat_lens = self.compute_length_from_mask(mask)
+        pooled_list = []
+        for x, feat_len in zip(xs, feat_lens):
+            x = x[:feat_len].unsqueeze(0)
+            h = torch.tanh(self.sap_linear(x))
+            w = torch.matmul(h, self.attention).squeeze(dim=2)
+            w = F.softmax(w, dim=1).view(x.size(0), x.size(1), 1)
+            mu = torch.sum(x * w, dim=1)
+            rh = torch.sqrt((torch.sum((x**2) * w, dim=1) - mu**2).clamp(min=1e-5))
+            x = torch.cat((mu, rh), 1).squeeze(0)
+            pooled_list.append(x)
+        return torch.stack(pooled_list)
+
+
+
+    
+class EmotionRegression(nn.Module):
+    def __init__(self, *args, **kwargs):
+        super(EmotionRegression, self).__init__()
+        input_dim = args[0]
+        hidden_dim = args[1]
+        num_layers = args[2]
+        output_dim = args[3]
+        p = kwargs.get("dropout", 0.5)
+
+        self.fc=nn.ModuleList([
+            nn.Sequential(
+                nn.Linear(input_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.ReLU(), nn.Dropout(p)
+            )
+        ])
+        for lidx in range(num_layers-1):
+            self.fc.append(
+                nn.Sequential(
+                    nn.Linear(hidden_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.ReLU(), nn.Dropout(p)
+                )
+            )
+        self.out = nn.Sequential(
+            nn.Linear(hidden_dim, output_dim)
+        )
+        
+        self.inp_drop = nn.Dropout(p)
+    def get_repr(self, x):
+        h = self.inp_drop(x)
+        for lidx, fc in enumerate(self.fc):
+            h=fc(h)
+        return h
+    
+    def forward(self, x):
+        h=self.get_repr(x)
+        result = self.out(h)
+        return result
+    
+class SERConfig(PretrainedConfig):
+    model_type = "ser"
+
+    def __init__(
+        self,
+        num_classes: int = 1,
+        num_attention_heads = 16,
+        num_hidden_layers = 24,
+        hidden_size = 1024,
+        classifier_hidden_layers = 1,
+        classifier_dropout_prob  = 0.5,
+        ssl_type= "microsoft/wavlm-large",
+        torch_dtype= "float32",
+        **kwargs,
+    ):
+        self.num_classes = num_classes
+        self.num_attention_heads = num_attention_heads
+        self.num_hidden_layers = num_hidden_layers
+        self.hidden_size = hidden_size
+        self.classifier_hidden_layers = classifier_hidden_layers
+        self.classifier_dropout_prob = classifier_dropout_prob
+        self.ssl_type = ssl_type
+        self.torch_dtype = torch_dtype
+        super().__init__(**kwargs)
+        
+class SERModel(PreTrainedModel):
+    config_class = SERConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+        self.ssl_model = AutoModel.from_pretrained(config.ssl_type)
+        self.ssl_model.freeze_feature_encoder()
+        
+        self.pool_model = AttentiveStatisticsPooling(config.hidden_size)
+    
+        self.ser_model = EmotionRegression(config.hidden_size*2, 
+                                               config.hidden_size, 
+                                               config.classifier_hidden_layers, 
+                                               config.num_classes, 
+                                               dropout=config.classifier_dropout_prob)
+        
+        
+    def forward(self, x, mask):
+        ssl = self.ssl_model(x, attention_mask=mask).last_hidden_state
+
+        ssl = self.pool_model(ssl, mask)
+        
+        pred = self.ser_model(ssl)
+        
+        return pred
+    

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Dominance/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6fc0167d183d89114be10df1c4e4f74040b558408efee99a71fcf5205865ef2
+size 1274585617

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Multi-Attributes/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Multi-Attributes/README.md

@@ -0,0 +1,84 @@
+---
+license: mit
+language:
+- en
+pipeline_tag: audio-classification
+tags:
+- wavlm
+- msp-podcast
+- emotion-recognition
+- audio
+- speech
+- valence
+- arousal
+- dominance
+- lucas
+- speech-emotion-recognition
+---
+The model was trained on [MSP-Podcast](https://ecs.utdallas.edu/research/researchlabs/msp-lab/MSP-Podcast.html) for the Odyssey 2024 Emotion Recognition competition baseline<br>
+This particular model is the multi-attributed based model which predict arousal, dominance and valence in a range of approximately 0...1. 
+
+
+# Benchmarks
+CCC based on Test3 and Development sets of the Odyssey Competition
+<table style="width:500px">
+  <tr><th colspan=6 align="center" >Multi-Task Setup</th></tr>
+  <tr><th colspan=3 align="center">Test 3</th><th colspan=3 align="center">Development</th></tr>
+  <tr>   <td>Val</td> <td>Dom</td> <td>Aro</td> <td>Val</td> <td>Dom</td> <td>Aro</td> </tr>
+  <tr>  <td> 0.577</td> <td>0.577</td> <td>0.405</td> <td>0.652</td> <td>0.688</td> <td>0.579</td> </tr>
+</table>
+ 
+
+
+For more details:  [demo](https://huggingface.co/spaces/3loi/WavLM-SER-Multi-Baseline-Odyssey2024), [paper](https://ecs.utdallas.edu/research/researchlabs/msp-lab/publications/Goncalves_2024.pdf), and [GitHub](https://github.com/MSP-UTD/MSP-Podcast_Challenge/tree/main).
+
+
+```
+@InProceedings{Goncalves_2024,
+            author={L. Goncalves and A. N. Salman and A. {Reddy Naini} and L. Moro-Velazquez and T. Thebaud and L. {Paola Garcia} and N. Dehak and B. Sisman and C. Busso},
+            title={Odyssey2024 - Speech Emotion Recognition Challenge: Dataset, Baseline Framework, and Results},
+            booktitle={Odyssey 2024: The Speaker and Language Recognition Workshop)},
+            volume={To appear},
+            year={2024},
+            month={June},
+            address =  {Quebec, Canada},
+}
+```
+
+
+# Usage
+```python
+from transformers import AutoModelForAudioClassification
+import librosa, torch
+
+#load model
+model = AutoModelForAudioClassification.from_pretrained("3loi/SER-Odyssey-Baseline-WavLM-Multi-Attributes", trust_remote_code=True)
+
+#get mean/std
+mean = model.config.mean
+std = model.config.std
+
+
+#load an audio file
+audio_path = "/path/to/audio.wav"
+raw_wav, _ = librosa.load(audio_path, sr=model.config.sampling_rate)
+
+#normalize the audio by mean/std
+norm_wav = (raw_wav - mean) / (std+0.000001)
+
+#generate the mask
+mask = torch.ones(1, len(norm_wav))
+
+#batch it (add dim)
+wavs = torch.tensor(norm_wav).unsqueeze(0)
+
+
+#predict
+with torch.no_grad():
+    pred = model(wavs, mask)
+
+print(model.config.id2label) 
+print(pred)
+#{0: 'arousal', 1: 'dominance', 2: 'valence'}
+#tensor([[0.3670, 0.4553, 0.4240]])
+```

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Multi-Attributes/config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "SERModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "pipeline_utils.SERConfig",
+    "AutoModelForAudioClassification": "pipeline_utils.SERModel"
+  },
+  "id2label": {
+    "0": "arousal",
+    "1": "dominance",
+    "2": "valence"
+  },
+  "sampling_rate": 16000,
+  "maxlen": 192000,
+  "mean": -8.278621631819787e-05, 
+  "std": 0.08485510250851999,
+  "classifier_dropout_prob": 0.5,
+  "classifier_hidden_layers": 1,
+  "hidden_size": 1024,
+  "model_type": "ser",
+  "num_attention_heads": 16,
+  "num_classes": 3,
+  "num_hidden_layers": 24,
+  "ssl_type": "microsoft/wavlm-large",
+  "torch_dtype": "float32",
+  "transformers_version": "4.34.0.dev0"
+}

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Multi-Attributes/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:557ba9b4aa8461a60bc7f5c5bd2e34b4de34d4c8ccfa684c438b6cbdc1893c9d
+size 1274490516

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Multi-Attributes/pipeline_utils.py

@@ -0,0 +1,167 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import AutoModel
+from transformers.modeling_utils import PreTrainedModel ,PretrainedConfig
+
+
+
+
+class Pooling(nn.Module):
+    def __init__(self):
+        super().__init__()
+    def compute_length_from_mask(self, mask):
+        """
+        mask: (batch_size, T)
+        Assuming that the sampling rate is 16kHz, the frame shift is 20ms
+        """
+        wav_lens = torch.sum(mask, dim=1) # (batch_size, )
+        feat_lens = torch.div(wav_lens-1, 16000*0.02, rounding_mode="floor") + 1
+        feat_lens = feat_lens.int().tolist()
+        return feat_lens
+        
+    def forward(self, x, mask):
+        raise NotImplementedError
+    
+class MeanPooling(Pooling):
+    def __init__(self):
+        super().__init__()
+    def forward(self, xs, mask):
+        """
+        xs: (batch_size, T, feat_dim)
+        mask: (batch_size, T)
+
+        => output: (batch_size, feat_dim)
+        """
+        feat_lens = self.compute_length_from_mask(mask)
+        pooled_list = []
+        for x, feat_len in zip(xs, feat_lens):
+            pooled = torch.mean(x[:feat_len], dim=0) # (feat_dim, )
+            pooled_list.append(pooled)
+        pooled = torch.stack(pooled_list, dim=0) # (batch_size, feat_dim)
+        return pooled
+    
+
+class AttentiveStatisticsPooling(Pooling):
+    """
+    AttentiveStatisticsPooling
+    Paper: Attentive Statistics Pooling for Deep Speaker Embedding
+    Link: https://arxiv.org/pdf/1803.10963.pdf
+    """
+    def __init__(self, input_size):
+        super().__init__()
+        self._indim = input_size
+        self.sap_linear = nn.Linear(input_size, input_size)
+        self.attention = nn.Parameter(torch.FloatTensor(input_size, 1))
+        torch.nn.init.normal_(self.attention, mean=0, std=1)
+
+    def forward(self, xs, mask):
+        """
+        xs: (batch_size, T, feat_dim)
+        mask: (batch_size, T)
+
+        => output: (batch_size, feat_dim*2)
+        """
+        feat_lens = self.compute_length_from_mask(mask)
+        pooled_list = []
+        for x, feat_len in zip(xs, feat_lens):
+            x = x[:feat_len].unsqueeze(0)
+            h = torch.tanh(self.sap_linear(x))
+            w = torch.matmul(h, self.attention).squeeze(dim=2)
+            w = F.softmax(w, dim=1).view(x.size(0), x.size(1), 1)
+            mu = torch.sum(x * w, dim=1)
+            rh = torch.sqrt((torch.sum((x**2) * w, dim=1) - mu**2).clamp(min=1e-5))
+            x = torch.cat((mu, rh), 1).squeeze(0)
+            pooled_list.append(x)
+        return torch.stack(pooled_list)
+
+
+
+    
+class EmotionRegression(nn.Module):
+    def __init__(self, *args, **kwargs):
+        super(EmotionRegression, self).__init__()
+        input_dim = args[0]
+        hidden_dim = args[1]
+        num_layers = args[2]
+        output_dim = args[3]
+        p = kwargs.get("dropout", 0.5)
+
+        self.fc=nn.ModuleList([
+            nn.Sequential(
+                nn.Linear(input_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.ReLU(), nn.Dropout(p)
+            )
+        ])
+        for lidx in range(num_layers-1):
+            self.fc.append(
+                nn.Sequential(
+                    nn.Linear(hidden_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.ReLU(), nn.Dropout(p)
+                )
+            )
+        self.out = nn.Sequential(
+            nn.Linear(hidden_dim, output_dim)
+        )
+        
+        self.inp_drop = nn.Dropout(p)
+    def get_repr(self, x):
+        h = self.inp_drop(x)
+        for lidx, fc in enumerate(self.fc):
+            h=fc(h)
+        return h
+    
+    def forward(self, x):
+        h=self.get_repr(x)
+        result = self.out(h)
+        return result
+    
+class SERConfig(PretrainedConfig):
+    model_type = "ser"
+
+    def __init__(
+        self,
+        num_classes: int = 3,
+        num_attention_heads = 16,
+        num_hidden_layers = 24,
+        hidden_size = 1024,
+        classifier_hidden_layers = 1,
+        classifier_dropout_prob  = 0.5,
+        ssl_type= "microsoft/wavlm-large",
+        torch_dtype= "float32",
+        **kwargs,
+    ):
+        self.num_classes = num_classes
+        self.num_attention_heads = num_attention_heads
+        self.num_hidden_layers = num_hidden_layers
+        self.hidden_size = hidden_size
+        self.classifier_hidden_layers = classifier_hidden_layers
+        self.classifier_dropout_prob = classifier_dropout_prob
+        self.ssl_type = ssl_type
+        self.torch_dtype = torch_dtype
+        super().__init__(**kwargs)
+        
+class SERModel(PreTrainedModel):
+    config_class = SERConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+        self.ssl_model = AutoModel.from_pretrained(config.ssl_type)
+        self.ssl_model.freeze_feature_encoder()
+        
+        self.pool_model = AttentiveStatisticsPooling(config.hidden_size)
+    
+        self.ser_model = EmotionRegression(config.hidden_size*2, 
+                                               config.hidden_size, 
+                                               config.classifier_hidden_layers, 
+                                               config.num_classes, 
+                                               dropout=config.classifier_dropout_prob)
+        
+        
+    def forward(self, x, mask):
+        ssl = self.ssl_model(x, attention_mask=mask).last_hidden_state
+
+        ssl = self.pool_model(ssl, mask)
+        
+        pred = self.ser_model(ssl)
+        
+        return pred
+    

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Multi-Attributes/preprocessor_config.json

@@ -0,0 +1,3 @@
+{
+  "mean": 10
+}

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Multi-Attributes/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c34b4fd571efce7b4530a7539f1928213d535f6be19b2324bceca0c08c3e601
+size 1274593809

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Valence/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Valence/README.md

@@ -0,0 +1,83 @@
+---
+license: mit
+language:
+- en
+pipeline_tag: audio-classification
+tags:
+- wavlm
+- msp-podcast
+- emotion-recognition
+- audio
+- speech
+- valence
+- lucas
+- speech-emotion-recognition
+---
+The model was trained on [MSP-Podcast](https://ecs.utdallas.edu/research/researchlabs/msp-lab/MSP-Podcast.html) for the Odyssey 2024 Emotion Recognition competition baseline<br>
+This particular model is the single-task specialized valence model, which predict valence in a range of approximately 0...1. 
+
+
+
+# Benchmarks
+CCC based on Test3 and Development sets of the Odyssey Competition
+<table style="width:500px">
+  <tr><th colspan=2 align="center"> Sinle-Task Setup </th></tr>
+  <tr><th colspan=1 align="center">Test 3</th><th colspan=1 align="center">Development</th></tr>
+  <tr>   <td align="center">Val</td> <td align="center">Val</td>  </tr>
+  <tr>  <td align="center"> 0.607</td>  <td align="center" >0.709 </td>  </tr>
+</table>
+ 
+
+
+For more details:  [demo](https://huggingface.co/spaces/3loi/WavLM-SER-Multi-Baseline-Odyssey2024), [paper](https://ecs.utdallas.edu/research/researchlabs/msp-lab/publications/Goncalves_2024.pdf), and [GitHub](https://github.com/MSP-UTD/MSP-Podcast_Challenge/tree/main).
+
+
+```
+@InProceedings{Goncalves_2024,
+            author={L. Goncalves and A. N. Salman and A. {Reddy Naini} and L. Moro-Velazquez and T. Thebaud and L. {Paola Garcia} and N. Dehak and B. Sisman and C. Busso},
+            title={Odyssey2024 - Speech Emotion Recognition Challenge: Dataset, Baseline Framework, and Results},
+            booktitle={Odyssey 2024: The Speaker and Language Recognition Workshop)},
+            volume={To appear},
+            year={2024},
+            month={June},
+            address =  {Quebec, Canada},
+}
+```
+
+
+# Usage
+```python
+from transformers import AutoModelForAudioClassification
+import librosa, torch
+
+#load model
+model = AutoModelForAudioClassification.from_pretrained("3loi/SER-Odyssey-Baseline-WavLM-Valence", trust_remote_code=True)
+
+#get mean/std
+mean = model.config.mean
+std = model.config.std
+
+
+#load an audio file
+audio_path = "/path/to/audio.wav"
+raw_wav, _ = librosa.load(audio_path, sr=model.config.sampling_rate)
+
+#normalize the audio by mean/std
+norm_wav = (raw_wav - mean) / (std+0.000001)
+
+#generate the mask
+mask = torch.ones(1, len(norm_wav))
+
+#batch it (add dim)
+wavs = torch.tensor(norm_wav).unsqueeze(0)
+
+
+#predict
+with torch.no_grad():
+    pred = model(wavs, mask)
+
+print(model.config.id2label) 
+print(pred)
+#{0: 'valence'}
+#tensor([[0.3670]])
+```

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Valence/config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "SERModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "pipeline_utils.SERConfig",
+    "AutoModelForAudioClassification": "pipeline_utils.SERModel"
+  },
+  "id2label": {
+    "0": "valence"
+  },
+  "sampling_rate": 16000,
+  "maxlen": 192000,
+  "mean": -8.278621631819787e-05, 
+  "std": 0.08485510250851999,
+  "classifier_dropout_prob": 0.5,
+  "classifier_hidden_layers": 1,
+  "hidden_size": 1024,
+  "model_type": "ser",
+  "num_attention_heads": 16,
+  "num_classes": 1,
+  "num_hidden_layers": 24,
+  "ssl_type": "microsoft/wavlm-large",
+  "torch_dtype": "float32",
+  "transformers_version": "4.34.0.dev0"
+}

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Valence/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44449ad4b46e4af5168f29b25055ca67c28ffd44829d11020782c43712bbc8b3
+size 1274482316

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Valence/pipeline_utils.py

@@ -0,0 +1,165 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import AutoModel
+from transformers.modeling_utils import PreTrainedModel ,PretrainedConfig
+
+
+class Pooling(nn.Module):
+    def __init__(self):
+        super().__init__()
+    def compute_length_from_mask(self, mask):
+        """
+        mask: (batch_size, T)
+        Assuming that the sampling rate is 16kHz, the frame shift is 20ms
+        """
+        wav_lens = torch.sum(mask, dim=1) # (batch_size, )
+        feat_lens = torch.div(wav_lens-1, 16000*0.02, rounding_mode="floor") + 1
+        feat_lens = feat_lens.int().tolist()
+        return feat_lens
+        
+    def forward(self, x, mask):
+        raise NotImplementedError
+    
+class MeanPooling(Pooling):
+    def __init__(self):
+        super().__init__()
+    def forward(self, xs, mask):
+        """
+        xs: (batch_size, T, feat_dim)
+        mask: (batch_size, T)
+
+        => output: (batch_size, feat_dim)
+        """
+        feat_lens = self.compute_length_from_mask(mask)
+        pooled_list = []
+        for x, feat_len in zip(xs, feat_lens):
+            pooled = torch.mean(x[:feat_len], dim=0) # (feat_dim, )
+            pooled_list.append(pooled)
+        pooled = torch.stack(pooled_list, dim=0) # (batch_size, feat_dim)
+        return pooled
+    
+
+class AttentiveStatisticsPooling(Pooling):
+    """
+    AttentiveStatisticsPooling
+    Paper: Attentive Statistics Pooling for Deep Speaker Embedding
+    Link: https://arxiv.org/pdf/1803.10963.pdf
+    """
+    def __init__(self, input_size):
+        super().__init__()
+        self._indim = input_size
+        self.sap_linear = nn.Linear(input_size, input_size)
+        self.attention = nn.Parameter(torch.FloatTensor(input_size, 1))
+        torch.nn.init.normal_(self.attention, mean=0, std=1)
+
+    def forward(self, xs, mask):
+        """
+        xs: (batch_size, T, feat_dim)
+        mask: (batch_size, T)
+
+        => output: (batch_size, feat_dim*2)
+        """
+        feat_lens = self.compute_length_from_mask(mask)
+        pooled_list = []
+        for x, feat_len in zip(xs, feat_lens):
+            x = x[:feat_len].unsqueeze(0)
+            h = torch.tanh(self.sap_linear(x))
+            w = torch.matmul(h, self.attention).squeeze(dim=2)
+            w = F.softmax(w, dim=1).view(x.size(0), x.size(1), 1)
+            mu = torch.sum(x * w, dim=1)
+            rh = torch.sqrt((torch.sum((x**2) * w, dim=1) - mu**2).clamp(min=1e-5))
+            x = torch.cat((mu, rh), 1).squeeze(0)
+            pooled_list.append(x)
+        return torch.stack(pooled_list)
+
+
+
+    
+class EmotionRegression(nn.Module):
+    def __init__(self, *args, **kwargs):
+        super(EmotionRegression, self).__init__()
+        input_dim = args[0]
+        hidden_dim = args[1]
+        num_layers = args[2]
+        output_dim = args[3]
+        p = kwargs.get("dropout", 0.5)
+
+        self.fc=nn.ModuleList([
+            nn.Sequential(
+                nn.Linear(input_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.ReLU(), nn.Dropout(p)
+            )
+        ])
+        for lidx in range(num_layers-1):
+            self.fc.append(
+                nn.Sequential(
+                    nn.Linear(hidden_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.ReLU(), nn.Dropout(p)
+                )
+            )
+        self.out = nn.Sequential(
+            nn.Linear(hidden_dim, output_dim)
+        )
+        
+        self.inp_drop = nn.Dropout(p)
+    def get_repr(self, x):
+        h = self.inp_drop(x)
+        for lidx, fc in enumerate(self.fc):
+            h=fc(h)
+        return h
+    
+    def forward(self, x):
+        h=self.get_repr(x)
+        result = self.out(h)
+        return result
+    
+class SERConfig(PretrainedConfig):
+    model_type = "ser"
+
+    def __init__(
+        self,
+        num_classes: int = 1,
+        num_attention_heads = 16,
+        num_hidden_layers = 24,
+        hidden_size = 1024,
+        classifier_hidden_layers = 1,
+        classifier_dropout_prob  = 0.5,
+        ssl_type= "microsoft/wavlm-large",
+        torch_dtype= "float32",
+        **kwargs,
+    ):
+        self.num_classes = num_classes
+        self.num_attention_heads = num_attention_heads
+        self.num_hidden_layers = num_hidden_layers
+        self.hidden_size = hidden_size
+        self.classifier_hidden_layers = classifier_hidden_layers
+        self.classifier_dropout_prob = classifier_dropout_prob
+        self.ssl_type = ssl_type
+        self.torch_dtype = torch_dtype
+        super().__init__(**kwargs)
+        
+class SERModel(PreTrainedModel):
+    config_class = SERConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+        self.ssl_model = AutoModel.from_pretrained(config.ssl_type)
+        self.ssl_model.freeze_feature_encoder()
+        
+        self.pool_model = AttentiveStatisticsPooling(config.hidden_size)
+    
+        self.ser_model = EmotionRegression(config.hidden_size*2, 
+                                               config.hidden_size, 
+                                               config.classifier_hidden_layers, 
+                                               config.num_classes, 
+                                               dropout=config.classifier_dropout_prob)
+        
+        
+    def forward(self, x, mask):
+        ssl = self.ssl_model(x, attention_mask=mask).last_hidden_state
+
+        ssl = self.pool_model(ssl, mask)
+        
+        pred = self.ser_model(ssl)
+        
+        return pred
+    

SER-Odyssey/SER-Odyssey-Baseline-WavLM-Valence/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ca8929ea564b56819ed846e96b1a472df11fa39d63f540108bab62c84b269b8
+size 1274585617

SER-Odyssey/SER-WavLM-Multi-Attributes/.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tensorrt/trt10_ser_fp16.plan filter=lfs diff=lfs merge=lfs -text
+tensorrt/trt8_ser_dyn_fp16.plan filter=lfs diff=lfs merge=lfs -text

SER-Odyssey/SER-WavLM-Multi-Attributes/README.md

@@ -0,0 +1,228 @@
+---
+license: mit
+language:
+- en
+pipeline_tag: audio-classification
+tags:
+- pytorch
+- wavlm
+- msp-podcast
+- emotion-recognition
+- audio
+- speech
+- valence
+- arousal
+- dominance
+- lucas
+- speech-emotion-recognition
+---
+The model is a recreation of [3loi/SER-Odyssey-Baseline-WavLM-Multi-Attributes](https://huggingface.co/3loi/SER-Odyssey-Baseline-WavLM-Multi-Attributes) for direct implementation in torch, with class definition and feed forward method. This model was recreated with the hopes of greater flexibilty of control, training/fine-tuning of model. The model was trained on the same [MSP-Podcast](https://ecs.utdallas.edu/research/researchlabs/msp-lab/MSP-Podcast.html) dataset as the original, but a different smaller subset was used. The subset is evenly distributed across gender and emotion category with hopes that training would improve accuracy of valence and arousal predictions. 
+This model is therefore a multi-attributed based model which predict arousal, dominance and valence. However, unlike the original model, I just kept the original attribute score range of  0...7 (the range the dataset follows). I will provide the evaluations later on. For now I decided to make this repo so that other people could test out my model and see what they think of the inference accuracy themselves, or retrain from scratch, modify etc. My best trained weights s of now are provided in this repo. The class definition for the model is can be found in my [github](https://github.com/PhilipAmadasun/SER-Model-for-dimensional-attribute-prediction#).
+
+# Get class definition
+```
+git clone https://github.com/PhilipAmadasun/SER-Model-for-dimensional-attribute-prediction.git
+```
+
+# Usage
+## Inference Testing
+```python
+import torch
+import torchaudio
+from SER_Model_setup import SERModel 
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+checkpoint_path = "<model.pt file>"
+checkpoint = torch.load(checkpoint_path, map_location=device)
+
+# Create the model architecture and load weights
+model = SERModel()
+model.load_state_dict(checkpoint['model_state_dict'])
+model.to(device)
+model.eval()
+
+audio_path = "<wav file>"
+audio, sr = torchaudio.load(audio_path)
+
+if sr != model.sample_rate:
+    resampler = torchaudio.transforms.Resample(sr, model.sample_rate)
+    audio = resampler(audio)
+#print(audio.shape[0])
+
+if audio.shape[0] > 1:
+    audio = torch.mean(audio, dim=0, keepdim=True)
+
+audio_len = audio.shape[-1]
+
+# Create waveform tensor (shape: [1, audio_len])
+waveform = torch.zeros(1, audio_len, dtype=torch.float32)
+# print(waveform)
+# print()
+# print(f"waveform shape: {waveform.shape}")
+# print()
+waveform[0, :audio_len] = audio
+# print(waveform)
+# print()
+# Create mask as 2D tensor: shape [1, audio_len] with ones in valid region
+mask = torch.ones(1, audio_len, dtype=torch.float32)
+# print(mask)
+# print()
+# print(f"mask shape: {mask.shape}")
+
+# Move waveform and mask to device
+waveform = waveform.to(device)
+mask = mask.to(device)
+
+# Normalize waveform using model's mean and std
+mean = model.mean.to(device)
+std = model.std.to(device)
+waveform = (waveform - mean) / (std + 1e-6)
+
+with torch.no_grad():
+    predictions = model(waveform, mask)  # predictions shape: [1, 3]
+
+# Extract predictions: [0,0] for arousal, [0,1] for valence, [0,2] for dominance
+arousal   = predictions[0, 0].item()
+valence   = predictions[0, 1].item()
+dominance = predictions[0, 2].item()
+
+print(f"Arousal: {arousal:.3f}")
+print(f"Valence: {valence:.3f}")
+print(f"Dominance: {dominance:.3f}")
+```
+## Batch inference
+```python
+import os
+import glob
+import torch
+import torchaudio
+from SER_Model_setup import SERModel  # Adjust if your model code is elsewhere
+
+def load_model_from_checkpoint(checkpoint_path, device='cpu'):
+    """
+    Loads the SERModel and weights from a checkpoint, moves to device, sets eval mode.
+    """
+    checkpoint = torch.load(checkpoint_path, map_location=device)
+    
+    # Create the model architecture
+    model = SERModel()
+    model.load_state_dict(checkpoint['model_state_dict'])
+    
+    model.to(device)
+    model.eval()
+    return model
+
+def batch_inference(model, file_paths, device='cpu', normalize=True):
+    """
+    Perform true batch inference on multiple .wav files in one forward pass.
+    
+    Args:
+        model (SERModel): The loaded SER model in eval mode
+        file_paths (list[str]): List of paths to .wav files
+        device (str or torch.device): 'cpu' or 'cuda'
+        normalize (bool): Whether to normalize waveforms (subtract mean, divide std)
+    
+    Returns:
+        dict: {filename: {"arousal": float, "valence": float, "dominance": float}}
+    """
+
+    # ----------------------------------------
+    # 1) Load & store all waveforms in memory
+    # ----------------------------------------
+    waveforms_list = []
+    lengths = []
+    for fp in file_paths:
+        # Load audio
+        audio, sr = torchaudio.load(fp)
+        
+        # Resample if needed
+        if sr != model.sample_rate:
+            resampler = torchaudio.transforms.Resample(sr, model.sample_rate)
+            audio = resampler(audio)
+        
+        # Convert stereo -> mono if needed
+        if audio.shape[0] > 1:
+            audio = torch.mean(audio, dim=0, keepdim=True)
+        
+        # audio shape => [1, num_samples]
+        lengths.append(audio.shape[-1])
+        waveforms_list.append(audio)
+    
+    # ----------------------------------------
+    # 2) Determine max length
+    # ----------------------------------------
+    max_len = max(lengths)
+    
+    # ----------------------------------------
+    # 3) Pad each waveform to max length & build masks
+    # ----------------------------------------
+    batch_size = len(waveforms_list)
+    batched_waveforms = torch.zeros(batch_size, 1, max_len, dtype=torch.float32)
+    masks = torch.zeros(batch_size, max_len, dtype=torch.float32)
+
+    for i, audio in enumerate(waveforms_list):
+        cur_len = audio.shape[-1]
+        batched_waveforms[i, :, :cur_len] = audio
+        masks[i, :cur_len] = 1.0  # valid portion
+
+    # ----------------------------------------
+    # 4) Move batched data to device BEFORE normalization
+    # ----------------------------------------
+    batched_waveforms = batched_waveforms.to(device)
+    masks = masks.to(device)
+    
+    # ----------------------------------------
+    # 5) Normalize if needed (model.mean, model.std)
+    # ----------------------------------------
+    if normalize:
+        # model.mean and model.std are buffers; ensure they're on the correct device
+        mean = model.mean.to(device)
+        std = model.std.to(device)
+        batched_waveforms = (batched_waveforms - mean) / (std + 1e-6)
+    
+    # ----------------------------------------
+    # 6) Single forward pass
+    # ----------------------------------------
+    with torch.no_grad():
+        predictions = model(batched_waveforms, masks)
+        # predictions shape => [batch_size, 3]
+    
+    # ----------------------------------------
+    # 7) Build result dict
+    # ----------------------------------------
+    results = {}
+    for i, fp in enumerate(file_paths):
+        arousal   = predictions[i, 0].item()
+        valence   = predictions[i, 1].item()
+        dominance = predictions[i, 2].item()
+        filename = os.path.basename(fp)
+        results[filename] = {
+            "arousal": arousal,
+            "valence": valence,
+            "dominance": dominance
+        }
+    
+    return results
+
+if __name__ == "__main__":
+    # -----------------------------------------
+    # Example usage
+    # -----------------------------------------
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    
+    checkpoint_path = "<weights.pt>"
+    model = load_model_from_checkpoint(checkpoint_path, device=device)
+    
+    # Suppose you have a folder of .wav files
+    wav_folder = "<directory containing .wav files>"
+    wav_paths = glob.glob(os.path.join(wav_folder, "*.wav"))
+    
+    # Do a single pass of batch inference
+    all_results = batch_inference(model, wav_paths, device=device, normalize=True)
+    
+    # Print results
+    for fname, preds in all_results.items():
+        print(f"{fname}: Arousal={preds['arousal']:.3f}, "
+              f"Valence={preds['valence']:.3f}, Dominance={preds['dominance']:.3f}")
+```

SER-Odyssey/SER-WavLM-Multi-Attributes/onnx/ReadMe

@@ -0,0 +1 @@
+model in onnx format

SER-Odyssey/SER-WavLM-Multi-Attributes/onnx/ser_dyn.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dad8465907c9dcfaa47628d7e8401a281396c17fe17b3c8b72071279cb6b2cac
+size 1274295745

SER-Odyssey/SER-WavLM-Multi-Attributes/pytorch/best_weights.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:809f5b3ef98835b5ca9dcf9d0efb4bd6cf0a9cc458cfb9443ae07ef71b44f670
+size 1299851786

SER-Odyssey/SER-WavLM-Multi-Attributes/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/uyiosa/SER-WavLM-Multi-Attributes

SER-Odyssey/SER-WavLM-Multi-Attributes/tensorrt/ReadMe

@@ -0,0 +1,2 @@
+trt10 -- compiled with TensorRT version 10
+trt8 -- comiled with TensorRT version 8

SER-Odyssey/SER-WavLM-Multi-Attributes/tensorrt/trt10_ser_fp16.plan

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed792e1cd7a6e6f1d89413b5800da2b7328c40483d8532d8b0bc2e74444e0516
+size 644044452

SER-Odyssey/SER-WavLM-Multi-Attributes/tensorrt/trt8_ser_dyn_fp16.plan

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6e157bb3cc07c0e808a93815a478874a092cd9f172abf83c1822ce9b9f1d55d
+size 643712772

SER-Odyssey/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/3loi/models

WavLM. Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ca8836ebdf8236e610187738217d4c91c5ead13873472e476423f1561e9238e
+size 929604

tiny-random-WavLMForAudioFrameClassification-ONNX/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

tiny-random-WavLMForAudioFrameClassification-ONNX/config.json

@@ -0,0 +1,88 @@
+{
+  "_attn_implementation_autoset": true,
+  "_name_or_path": "hf-internal-testing/tiny-random-WavLMForAudioFrameClassification",
+  "activation_dropout": 0.1,
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": true,
+  "architectures": [
+    "WavLMForAudioFrameClassification"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "codevector_dim": 256,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": false,
+  "conv_dim": [
+    32,
+    32,
+    32
+  ],
+  "conv_kernel": [
+    8,
+    8,
+    8
+  ],
+  "conv_stride": [
+    4,
+    4,
+    4
+  ],
+  "ctc_loss_reduction": "mean",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": false,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_dropout": 0.0,
+  "feat_extract_norm": "group",
+  "feat_proj_dropout": 0.0,
+  "final_dropout": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.1,
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 20,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.1,
+  "mask_feature_length": 10,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.05,
+  "max_bucket_distance": 800,
+  "model_type": "wavlm",
+  "num_adapter_layers": 3,
+  "num_attention_heads": 2,
+  "num_buckets": 320,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 2,
+  "num_conv_pos_embeddings": 16,
+  "num_ctc_classes": 80,
+  "num_feat_extract_layers": 3,
+  "num_hidden_layers": 4,
+  "num_negatives": 100,
+  "output_hidden_size": 16,
+  "pad_token_id": 0,
+  "proj_codevector_dim": 256,
+  "tdnn_dilation": [
+    1,
+    1
+  ],
+  "tdnn_dim": [
+    32,
+    32
+  ],
+  "tdnn_kernel": [
+    3,
+    3
+  ],
+  "transformers_version": "4.48.2",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 32,
+  "xvector_output_dim": 32
+}

tiny-random-WavLMForAudioFrameClassification-ONNX/onnx/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43d68f66c0eb42e09d03c533d705eabde0fd481635fdff874b9d94ae4445b550
+size 276448