Add Transformers load-by-id model files

README.md

@@ -0,0 +1,61 @@
+---
+library_name: transformers
+pipeline_tag: audio-classification
+tags:
+- audio
+- music
+- speech
+- custom-code
+license: mit
+---
+
+# SMAD CRNN
+
+SMAD classifies a 4-second mono audio clip into one of four labels:
+`speech_noise`, `speech_music`, `singing_music`, or `none`.
+
+The model is a small CNN + BiGRU trained from scratch on 80-bin log-mel
+spectrograms. It has 834k parameters and reached 87.95% on the synthetic
+held-out test split used by the project.
+
+## Load by model id
+
+```python
+import torch
+from transformers import AutoFeatureExtractor, AutoModelForAudioClassification
+
+model_id = "duclvQ/smad"
+
+feature_extractor = AutoFeatureExtractor.from_pretrained(
+    model_id,
+    trust_remote_code=True,
+)
+model = AutoModelForAudioClassification.from_pretrained(
+    model_id,
+    trust_remote_code=True,
+).eval()
+
+# `audio` must be mono float32 audio sampled at 16 kHz. For longer files, run
+# this over 4-second windows.
+inputs = feature_extractor(audio, sampling_rate=16000, return_tensors="pt")
+with torch.no_grad():
+    logits = model(**inputs).logits
+    probs = torch.softmax(logits / model.config.temperature, dim=-1)
+
+label_id = int(probs.argmax(-1)[0])
+label = model.config.id2label[label_id]
+confidence = float(probs[0, label_id])
+```
+
+For arbitrary file paths, load/resample with `librosa`:
+
+```python
+import librosa
+
+audio, _ = librosa.load("clip.mp3", sr=16000, mono=True)
+```
+
+## Caveat
+
+The reported score is an upper bound on real-world performance: train and test
+clips come from the same synthetic mixer, so they share its biases.

config.json

@@ -0,0 +1,38 @@
+{
+  "architectures": [
+    "SmadForAudioClassification"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_smad.SmadConfig",
+    "AutoModel": "modeling_smad.SmadForAudioClassification",
+    "AutoModelForAudioClassification": "modeling_smad.SmadForAudioClassification"
+  },
+  "channels": [
+    32,
+    64,
+    128,
+    128
+  ],
+  "dropout": 0.2,
+  "id2label": {
+    "0": "speech_noise",
+    "1": "speech_music",
+    "2": "singing_music",
+    "3": "none"
+  },
+  "label2id": {
+    "speech_noise": 0,
+    "speech_music": 1,
+    "singing_music": 2,
+    "none": 3
+  },
+  "model_type": "smad_crnn",
+  "n_fft": 400,
+  "num_labels": 4,
+  "num_mels": 80,
+  "rnn_hidden": 128,
+  "rnn_type": "gru",
+  "sample_rate": 16000,
+  "segment_seconds": 4.0,
+  "temperature": 0.709507268312107
+}

configuration_smad.py

@@ -0,0 +1,46 @@
+from transformers import PretrainedConfig
+
+
+class SmadConfig(PretrainedConfig):
+    model_type = "smad_crnn"
+
+    def __init__(
+        self,
+        num_mels=80,
+        channels=(32, 64, 128, 128),
+        rnn_hidden=128,
+        dropout=0.2,
+        num_labels=4,
+        rnn_type="gru",
+        temperature=1.0,
+        sample_rate=16000,
+        segment_seconds=4.0,
+        n_fft=400,
+        hop_length=160,
+        label2id=None,
+        id2label=None,
+        **kwargs,
+    ):
+        label2id = label2id or {
+            "speech_noise": 0,
+            "speech_music": 1,
+            "singing_music": 2,
+            "none": 3,
+        }
+        id2label = id2label or {str(v): k for k, v in label2id.items()}
+        super().__init__(
+            num_labels=num_labels,
+            label2id=label2id,
+            id2label=id2label,
+            **kwargs,
+        )
+        self.num_mels = num_mels
+        self.channels = list(channels)
+        self.rnn_hidden = rnn_hidden
+        self.dropout = dropout
+        self.rnn_type = rnn_type
+        self.temperature = temperature
+        self.sample_rate = sample_rate
+        self.segment_seconds = segment_seconds
+        self.n_fft = n_fft
+        self.hop_length = hop_length

feature_extraction_smad.py

@@ -0,0 +1,66 @@
+import numpy as np
+from transformers import BatchFeature, SequenceFeatureExtractor
+
+
+class SmadFeatureExtractor(SequenceFeatureExtractor):
+    model_input_names = ["input_features"]
+
+    def __init__(
+        self,
+        feature_size=80,
+        sampling_rate=16000,
+        padding_value=0.0,
+        segment_seconds=4.0,
+        n_fft=400,
+        hop_length=160,
+        **kwargs,
+    ):
+        super().__init__(
+            feature_size=feature_size,
+            sampling_rate=sampling_rate,
+            padding_value=padding_value,
+            **kwargs,
+        )
+        self.segment_seconds = segment_seconds
+        self.n_fft = n_fft
+        self.hop_length = hop_length
+
+    def waveform_to_mel(self, waveform, sampling_rate=None):
+        import librosa
+
+        sampling_rate = sampling_rate or self.sampling_rate
+        mel = librosa.feature.melspectrogram(
+            y=np.asarray(waveform, dtype=np.float32),
+            sr=sampling_rate,
+            n_fft=self.n_fft,
+            hop_length=self.hop_length,
+            n_mels=self.feature_size,
+            power=2.0,
+        )
+        return librosa.power_to_db(mel).T.astype(np.float32)
+
+    def __call__(self, raw_speech, sampling_rate=None, return_tensors=None, **kwargs):
+        sampling_rate = sampling_rate or self.sampling_rate
+        if sampling_rate != self.sampling_rate:
+            raise ValueError(
+                f"Expected {self.sampling_rate} Hz audio. Resample before calling "
+                f"the feature extractor; received {sampling_rate} Hz."
+            )
+
+        if isinstance(raw_speech, np.ndarray) and raw_speech.ndim == 1:
+            waves = [raw_speech]
+        else:
+            waves = [np.asarray(w, dtype=np.float32) for w in raw_speech]
+
+        target_len = int(round(self.segment_seconds * self.sampling_rate))
+        features = []
+        for wave in waves:
+            if wave.ndim != 1:
+                raise ValueError("Expected mono audio arrays with shape `(samples,)`.")
+            if wave.shape[0] < target_len:
+                wave = np.pad(wave, (0, target_len - wave.shape[0]))
+            elif wave.shape[0] > target_len:
+                wave = wave[:target_len]
+            features.append(self.waveform_to_mel(wave, sampling_rate=sampling_rate))
+
+        return BatchFeature({"input_features": features}, tensor_type=return_tensors)

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63818291e401ca27616cb29ba3a224b780296ac4af87e6d6b70edb9daa40f5bc
+size 3342832

modeling_smad.py

@@ -0,0 +1,110 @@
+import torch
+from torch import nn
+from transformers import PreTrainedModel
+from transformers.modeling_outputs import SequenceClassifierOutput
+
+from .configuration_smad import SmadConfig
+
+
+class ConvBlock(nn.Module):
+    def __init__(self, in_ch, out_ch, pool):
+        super().__init__()
+        self.block = nn.Sequential(
+            nn.Conv2d(in_ch, out_ch, kernel_size=3, padding=1, bias=False),
+            nn.BatchNorm2d(out_ch),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(pool),
+        )
+
+    def forward(self, x):
+        return self.block(x)
+
+
+class TinyAudioCRNN(nn.Module):
+    def __init__(
+        self,
+        n_mels=80,
+        channels=(32, 64, 128, 128),
+        rnn_hidden=128,
+        dropout=0.2,
+        num_classes=4,
+        rnn_type="gru",
+    ):
+        super().__init__()
+        self.register_buffer("feat_mean", torch.zeros(n_mels))
+        self.register_buffer("feat_std", torch.ones(n_mels))
+
+        pools = [(2, 2)] * (len(channels) - 1) + [(2, 1)]
+        blocks, in_ch = [], 1
+        for out_ch, pool in zip(channels, pools):
+            blocks.append(ConvBlock(in_ch, out_ch, pool))
+            in_ch = out_ch
+        self.conv = nn.Sequential(*blocks)
+
+        freq_out = n_mels
+        for freq_pool, _ in pools:
+            freq_out //= freq_pool
+        if freq_out < 1:
+            raise ValueError(f"{len(channels)} conv blocks pool {n_mels} mel bins down to nothing")
+        rnn_in = channels[-1] * freq_out
+
+        self.dropout = nn.Dropout(dropout)
+        rnn_cls = {"gru": nn.GRU, "lstm": nn.LSTM}[rnn_type.lower()]
+        self.rnn = rnn_cls(
+            rnn_in,
+            rnn_hidden,
+            num_layers=1,
+            batch_first=True,
+            bidirectional=True,
+        )
+        self.classifier = nn.Linear(rnn_hidden * 2 * 2, num_classes)
+
+    def forward(self, x):
+        x = (x - self.feat_mean) / self.feat_std
+        x = x.transpose(1, 2).unsqueeze(1)
+        x = self.conv(x)
+        b, c, f, t = x.shape
+        x = x.permute(0, 3, 1, 2).reshape(b, t, c * f)
+        x = self.dropout(x)
+        x, _ = self.rnn(x)
+        pooled = torch.cat([x.mean(dim=1), x.max(dim=1).values], dim=-1)
+        return self.classifier(self.dropout(pooled))
+
+
+class SmadForAudioClassification(PreTrainedModel):
+    config_class = SmadConfig
+    base_model_prefix = "smad"
+    main_input_name = "input_features"
+    all_tied_weights_keys = {}
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.smad = TinyAudioCRNN(
+            n_mels=config.num_mels,
+            channels=tuple(config.channels),
+            rnn_hidden=config.rnn_hidden,
+            dropout=config.dropout,
+            num_classes=config.num_labels,
+            rnn_type=config.rnn_type,
+        )
+
+    def forward(self, input_features=None, labels=None, return_dict=None, **kwargs):
+        if input_features is None:
+            input_features = kwargs.pop("inputs", None)
+        if input_features is None:
+            raise ValueError("Pass log-mel features as `input_features`.")
+
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        logits = self.smad(input_features)
+        loss = None
+        if labels is not None:
+            loss = nn.functional.cross_entropy(logits, labels)
+        if not return_dict:
+            output = (logits,)
+            return ((loss,) + output) if loss is not None else output
+        return SequenceClassifierOutput(loss=loss, logits=logits)
+
+    @torch.no_grad()
+    def predict_proba(self, input_features):
+        logits = self(input_features=input_features).logits
+        return torch.softmax(logits / float(self.config.temperature), dim=-1)

preprocessor_config.json

@@ -0,0 +1,12 @@
+{
+  "auto_map": {
+    "AutoFeatureExtractor": "feature_extraction_smad.SmadFeatureExtractor"
+  },
+  "feature_extractor_type": "SmadFeatureExtractor",
+  "feature_size": 80,
+  "hop_length": 160,
+  "n_fft": 400,
+  "padding_value": 0.0,
+  "sampling_rate": 16000,
+  "segment_seconds": 4.0
+}