Upload new model: PL-BERT-MULTILINGUAL

.gitattributes

@@ -60,3 +60,4 @@ ellie-french-v1/default.wav filter=lfs diff=lfs merge=lfs -text
 ember-french-v1/default.wav filter=lfs diff=lfs merge=lfs -text
 ellie-italian-v1/default.wav filter=lfs diff=lfs merge=lfs -text
 ember-italian-v1/default.wav filter=lfs diff=lfs merge=lfs -text
+PL-BERT-MULTILINGUAL/step_1100000.t7 filter=lfs diff=lfs merge=lfs -text

PL-BERT-MULTILINGUAL/config.yml

@@ -0,0 +1,30 @@
+log_dir: "Checkpoint_all_phonemes"
+mixed_precision: "fp16"
+data_folder: "wikipedia_20220301.en.processed"
+batch_size: 32
+save_interval: 20000
+log_interval: 10
+num_process: 1 # number of GPUs
+num_steps: 2000000
+
+dataset_params:
+    tokenizer: "bert-base-multilingual-cased"
+    token_separator: " " # token used for phoneme separator (space)
+    token_mask: "M" # token used for phoneme mask (M)
+    word_separator: 102 # token used for word separator (<formula>)
+    token_maps: "token_maps.pkl" # token map path
+    
+    max_mel_length: 512 # max phoneme length
+    
+    word_mask_prob: 0.15 # probability to mask the entire word
+    phoneme_mask_prob: 0.1 # probability to mask each phoneme
+    replace_prob: 0.2 # probablity to replace phonemes
+    
+model_params:
+    vocab_size: 178
+    hidden_size: 768
+    num_attention_heads: 12
+    intermediate_size: 2048
+    max_position_embeddings: 512
+    num_hidden_layers: 12
+    dropout: 0.1

PL-BERT-MULTILINGUAL/step_1100000.t7

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e920e029e5226284f6042bef36514323c7fd0ae73c59e0ed4ccecd1f2916fd1
+size 25179228

PL-BERT-MULTILINGUAL/token_maps.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdb2b72e90be652f176c44fe4b101b897167bf1cd669ad76e11e9400928cc848
+size 1858692

PL-BERT-MULTILINGUAL/util.py

@@ -0,0 +1,45 @@
+import os
+import yaml
+import torch
+from transformers import AlbertConfig, AlbertModel
+
+class CustomAlbert(AlbertModel):
+    def forward(self, *args, **kwargs):
+        # Call the original forward method
+        outputs = super().forward(*args, **kwargs)
+
+        # Only return the last_hidden_state
+        return outputs.last_hidden_state
+
+
+def load_plbert(log_dir):
+    config_path = os.path.join(log_dir, "config.yml")
+    plbert_config = yaml.safe_load(open(config_path))
+    
+    albert_base_configuration = AlbertConfig(**plbert_config['model_params'])
+    bert = CustomAlbert(albert_base_configuration)
+
+    files = os.listdir(log_dir)
+    ckpts = []
+    for f in os.listdir(log_dir):
+        if f.startswith("step_"): ckpts.append(f)
+
+    iters = [int(f.split('_')[-1].split('.')[0]) for f in ckpts if os.path.isfile(os.path.join(log_dir, f))]
+    iters = sorted(iters)[-1]
+
+    checkpoint = torch.load(log_dir + "/step_" + str(iters) + ".t7", map_location='cpu')
+    state_dict = checkpoint['net']
+    from collections import OrderedDict
+    new_state_dict = OrderedDict()
+    for k, v in state_dict.items():
+        name = k[7:] # remove `module.`
+        if name.startswith('encoder.'):
+            name = name[8:] # remove `encoder.`
+            new_state_dict[name] = v
+    try:
+        del new_state_dict["embeddings.position_ids"]
+    except KeyError:
+        pass
+    bert.load_state_dict(new_state_dict, strict=False)
+    
+    return bert