Add template files

.gitattributes

@@ -15,3 +15,4 @@
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+asr/model.ckpt filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,10 @@
+# Fine-tuned Model Submission Template
+
+This is a template reprository for the SUPERB benchmark for the _fine-tuned model_ category. In this category, participants are asked to fine-tuned a pretrained model in each of SUPERB's downstream tasks and then store the model weights and hyperparameters in this repo.
+
+There are four steps involved in making a submission:
+
+1. Fine-tune a pretrained model on a downstream task.
+2. Implement the `PreTrainedModel` interface defined in each `model.py` module.
+3. Store the weights and hyperparameters in the task directory
+4. Push all the files to the Hugging Face Hub.

asr/README.md

@@ -0,0 +1,16 @@
+---
+library_name: superb
+benchmark: superb
+task: asr
+type: finetuned_upload
+submission_id: {{submission_id}}
+datasets:
+- superb
+tags:
+- automatic-speech-recognition
+widget:
+- label: Librispeech sample 1
+  src: https://cdn-media.huggingface.co/speech_samples/sample1.flac
+---
+
+# Fine-tuned s3prl model for ASR

asr/args.yaml

@@ -0,0 +1,26 @@
+auto_resume: false
+backend: nccl
+cache_dir: null
+config: ./downstream/asr/config.yaml
+device: cuda
+downstream: asr
+downstream_variant: null
+evaluate_split: test
+expdir: result/downstream/asr-push-to-hub
+expname: asr-push-to-hub
+from_hf_hub: true
+hub: huggingface
+init_ckpt: null
+local_rank: null
+mode: train
+override: config.downstream_expert.datarc.libri_root='/data/lewis/superb/LibriSpeech',,config.downstream_expert.datarc.bucket_file='/data/lewis/superb/LibriSpeech/len_for_bucket',,config.runner.total_steps=2000
+past_exp: null
+seed: 1337
+upstream: osanseviero/hubert_base
+upstream_ckpt: null
+upstream_feature_selection: hidden_states
+upstream_model_config: null
+upstream_model_name: model.pt
+upstream_refresh: false
+upstream_trainable: false
+verbose: false

asr/char.dict

@@ -0,0 +1,28 @@
+| 1980202
+E 1091870
+T 789572
+A 689048
+O 647720
+N 591778
+I 585614
+H 557204
+S 545238
+R 499568
+D 380912
+L 344952
+U 242014
+M 217730
+C 210734
+W 204598
+F 195086
+G 174098
+Y 168548
+P 146722
+B 129608
+V 81496
+K 65070
+' 19660
+X 12530
+J 12062
+Q 8164
+Z 4916

asr/config.yaml

@@ -0,0 +1,84 @@
+downstream_expert:
+  datarc:
+    batch_size: 32
+    bucket_file: /data/lewis/superb/LibriSpeech/len_for_bucket
+    decoder_args:
+      beam: 5
+      beam_threshold: 25
+      criterion: ctc
+      decoder_type: None
+      kenlm_model: /path/to/KenLM
+      lexicon: /path/to/4-gram.arpa
+      lm_weight: 2
+      nbest: 1
+      sil_weight: 0
+      unk_weight: -math.inf
+      word_score: -1
+    dev-clean:
+    - dev-clean
+    dev-other:
+    - dev-other
+    dict_path: ./downstream/asr/char.dict
+    eval_batch_size: 1
+    libri_root: /data/lewis/superb/LibriSpeech
+    num_workers: 12
+    test-clean:
+    - test-clean
+    test-other:
+    - test-other
+    train:
+    - train-clean-100
+    train_batch_size: 32
+    zero_infinity: true
+  modelrc:
+    RNNs:
+      bidirection: true
+      dim:
+      - 1024
+      - 1024
+      dropout:
+      - 0.2
+      - 0.2
+      layer_norm:
+      - false
+      - false
+      module: LSTM
+      proj:
+      - false
+      - false
+      sample_rate:
+      - 1
+      - 1
+      sample_style: concat
+      total_rate: -1
+    Wav2Letter:
+      total_rate: 320
+    project_dim: 1024
+    select: RNNs
+optimizer:
+  lr: 0.0001
+  name: TorchOptim
+  torch_optim_name: Adam
+runner:
+  eval_dataloaders:
+  - dev-clean
+  eval_step: 2000
+  gradient_accumulate_steps: 1
+  gradient_clipping: 1
+  log_step: 100
+  max_keep: 1
+  save_step: 500
+  total_steps: 2000
+specaug:
+  apply_freq_mask: true
+  apply_time_mask: true
+  apply_time_warp: true
+  freq_mask_width_range:
+  - 0
+  - 50
+  num_freq_mask: 4
+  num_time_mask: 2
+  time_mask_width_range:
+  - 0
+  - 40
+  time_warp_window: 5

asr/model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91030dbbf4f9cd4ab4412aec64a219e972cf0049adbe93269bc4e0965586b693
+size 513965711

asr/model.py

@@ -0,0 +1,39 @@
+from s3prl.downstream.runner import Runner
+from typing import Dict
+import torch
+import os
+
+
+class PreTrainedModel(Runner):
+    def __init__(self, path=""):
+        """
+        Initialize downstream model.
+        """
+        ckp_file = os.path.join(path, "model.ckpt")
+        ckp = torch.load(ckp_file, map_location='cpu')
+        ckp["Args"].init_ckpt = ckp_file
+        ckp["Args"].mode = "inference"
+        ckp["Args"].device = "cpu"
+        ckp["Config"]["downstream_expert"]["datarc"]["dict_path"] = os.path.join(path,'char.dict')
+
+        Runner.__init__(self, ckp["Args"], ckp["Config"])
+
+    def __call__(self, inputs)-> Dict[str, str]:
+        """
+        Args:
+            inputs (:obj:`np.array`):
+                The raw waveform of audio received. By default at 16KHz.
+        Return:
+            A :obj:`dict`:. The object return should be liked {"text": "XXX"} containing
+            the detected text from the input audio.
+        """
+        for entry in self.all_entries:
+            entry.model.eval()
+
+        inputs = [torch.FloatTensor(inputs)]
+
+        with torch.no_grad():
+            features = self.upstream.model(inputs)
+            features = self.featurizer.model(inputs, features)
+            preds = self.downstream.model.inference(features, [])
+        return {"text": preds[0]}