Upload folder using huggingface_hub

README.md

@@ -1,3 +1,13 @@
----
-license: bigscience-openrail-m
----
+---
+license: bigscience-openrail-m
+datasets:
+- DORI-SRKW/DORI-ONC
+- DORI-SRKW/DORI-Orcasound
+base_model: openai/whisper-tiny
+pipeline_tag: audio-classification
+tags:
+- biology
+---
+
+This is a marine mammal detector, which classifies absence(0) or presence(1) of marine mammals. While it is trained on Orcasound and Ocean Networks Canada data, it has generalised well to Ocean Observatories Initiative and other nodes.
+

config.json

@@ -0,0 +1,162 @@
+{
+  "_name_or_path": "DORI-SRKW/whisper-tiny-mm",
+  "activation_dropout": 0.0,
+  "activation_function": "gelu",
+  "apply_spec_augment": false,
+  "architectures": [
+    "WhisperForAudioClassification"
+  ],
+  "attention_dropout": 0.0,
+  "begin_suppress_tokens": [
+    220,
+    50257
+  ],
+  "bos_token_id": 50257,
+  "classifier_proj_size": 256,
+  "d_model": 384,
+  "decoder_attention_heads": 6,
+  "decoder_ffn_dim": 1536,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 4,
+  "decoder_start_token_id": 50258,
+  "dropout": 0.0,
+  "encoder_attention_heads": 6,
+  "encoder_ffn_dim": 1536,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 4,
+  "eos_token_id": 50257,
+  "forced_decoder_ids": [
+    [
+      1,
+      50259
+    ],
+    [
+      2,
+      50359
+    ],
+    [
+      3,
+      50363
+    ]
+  ],
+  "id2label": {
+    "0": "absent",
+    "1": "present"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "absent": 0,
+    "present": 1
+  },
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.05,
+  "max_length": 448,
+  "max_source_positions": 1500,
+  "max_target_positions": 448,
+  "median_filter_width": 7,
+  "model_type": "whisper",
+  "num_hidden_layers": 4,
+  "num_mel_bins": 80,
+  "pad_token_id": 50257,
+  "scale_embedding": false,
+  "suppress_tokens": [
+    1,
+    2,
+    7,
+    8,
+    9,
+    10,
+    14,
+    25,
+    26,
+    27,
+    28,
+    29,
+    31,
+    58,
+    59,
+    60,
+    61,
+    62,
+    63,
+    90,
+    91,
+    92,
+    93,
+    359,
+    503,
+    522,
+    542,
+    873,
+    893,
+    902,
+    918,
+    922,
+    931,
+    1350,
+    1853,
+    1982,
+    2460,
+    2627,
+    3246,
+    3253,
+    3268,
+    3536,
+    3846,
+    3961,
+    4183,
+    4667,
+    6585,
+    6647,
+    7273,
+    9061,
+    9383,
+    10428,
+    10929,
+    11938,
+    12033,
+    12331,
+    12562,
+    13793,
+    14157,
+    14635,
+    15265,
+    15618,
+    16553,
+    16604,
+    18362,
+    18956,
+    20075,
+    21675,
+    22520,
+    26130,
+    26161,
+    26435,
+    28279,
+    29464,
+    31650,
+    32302,
+    32470,
+    36865,
+    42863,
+    47425,
+    49870,
+    50254,
+    50258,
+    50358,
+    50359,
+    50360,
+    50361,
+    50362
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.1",
+  "use_cache": true,
+  "use_weighted_layer_sum": false,
+  "vocab_size": 51865
+}

handler.py

@@ -0,0 +1,88 @@
+"""
+handler.py
+Set up the possibility for an inference endpoint on huggingface.
+"""
+from typing import Dict, Any
+import torch
+import torchaudio
+from transformers import WhisperForAudioClassification, WhisperFeatureExtractor
+import numpy as np
+import base64
+
+class EndpointHandler():
+    """
+    This is a wrapper for huggingface models so that they return json objects and consider the same configs as other implementations
+    """
+    def __init__(self, threshold=0.5):
+
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+        model_id = 'DORI-SRKW/whisper-tiny-mm'
+
+        # Load the model
+        try:
+            self.model = WhisperForAudioClassification.from_pretrained(model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True)
+        except:
+            self.model = WhisperForAudioClassification.from_pretrained(model_id, torch_dtype=torch_dtype)
+        self.feature_extractor = WhisperFeatureExtractor.from_pretrained(model_id)
+
+        self.model.eval()
+        self.model.to(self.device)
+        self.threshold = threshold
+
+
+    def __call__(self, data: Dict[str, Any]) -> Dict[str, str]:
+        """
+        Args:
+            data (:obj:):
+                includes the input data and the parameters for the inference.
+        Return:
+            A :obj:`list`:. The object returned should be a list of one list like [[{"label": 0.9939950108528137}]] containing :
+                - "label": A string representing what the label/class is. There can be multiple labels.
+                - "score": A score between 0 and 1 describing how confident the model is for this label/class.
+        """
+
+        # step one, get the sampling rate of the audio
+        audio = data['audio']
+        # we encoded using base64.b64encode(filebytes).decode('utf-8') to pass to api url
+        audio = base64.b64decode(audio.encode('utf-8'))
+
+        
+        fs = data['sampling_rate']
+
+        # split into 15 second intervals
+        audio = np.frombuffer(audio, dtype=np.float32)
+        audio = torch.tensor(audio)
+        audio = audio.reshape(1, -1)
+
+
+        # torchaudio resamples the audio to 32000
+        audio = torchaudio.functional.resample(audio, orig_freq=fs, new_freq=32000)
+
+        # highpass filter 1000 hz
+        audio = torchaudio.functional.highpass_biquad(audio, 32000, 1000, 0.707)
+
+        audio3 = []
+        for i in range(0, len(audio[-1]), 32000*15):
+            audio3.append(audio[:,i:i+32000*15].squeeze().cpu().data.numpy())
+        
+        data = self.feature_extractor(audio3, sampling_rate = 16000, padding='max_length', max_length=32000*15, return_tensors='pt')
+
+        try:
+            data['input_values'] = data['input_values'].squeeze(0)
+        except:
+            # it is called input_features for whisper
+            data['input_features'] = data['input_features'].squeeze(0)
+
+        data = {k: v.to(self.device) for k, v in data.items()}
+        with torch.amp.autocast(device_type=self.device):
+            outputs = []
+            for segment in range(data['input_features'].shape[0]):
+                # iterate through 15 second segments
+                output = self.model(data['input_features'][segment].unsqueeze(0))
+
+
+                outputs.append({'logit': torch.softmax(output.logits, dim=1)[0][1].float().cpu().data.numpy().max(), 'start_time_s': segment*15})
+
+        outputs = {'logit': max([x['logit'] for x in outputs]), 'classification': 'present' if max([x['logit'] for x in outputs]) >= self.threshold else 'absent'}
+        return outputs

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e84017582382993ffd6193969d122788d99dcd2018373d188c38bc16d9fa43d
+size 33237160