upload

flax_wav2vec2/branches_to_use.txt

@@ -0,0 +1,2 @@
+fairseq: https://github.com/patrickvonplaten/fairseq/tree/save_w2v_pretraining_check (see https://github.com/patrickvonplaten/fairseq/pulls)
+transformers: https://github.com/patrickvonplaten/transformers/tree/debug_wav2vec2_pretraining (see https://github.com/huggingface/transformers/pull/12743)

flax_wav2vec2/run_pretraining_loss.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+import datasets
+import fairseq
+import torch
+
+import soundfile as sf
+import sys
+from fairseq.criterions.wav2vec_criterion import Wav2VecCriterionConfig, Wav2vecCriterion
+from fairseq.tasks.audio_pretraining import AudioPretrainingConfig, AudioPretrainingTask
+
+from transformers import Wav2Vec2ForPreTraining, Wav2Vec2FeatureExtractor
+
+hf_path = str(sys.argv[1])
+fairseq_wav2vec2_path = str(sys.argv[2])
+
+model, cfg, task = fairseq.checkpoint_utils.load_model_ensemble_and_task([fairseq_wav2vec2_path])
+
+feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(hf_path, do_normalize=False)
+hf_model = Wav2Vec2ForPreTraining.from_pretrained(hf_path)
+
+model = model[0]
+model.eval()
+
+dummy_speech_data = datasets.load_dataset("patrickvonplaten/librispeech_asr_dummy", "clean", split="validation")
+
+
+def map_to_array(batch):
+    speech_array, _ = sf.read(batch["file"])
+    batch["speech"] = speech_array
+    return batch
+
+
+dummy_speech_data = dummy_speech_data.map(map_to_array, remove_columns=["file"])
+inputs = feature_extractor(dummy_speech_data[:3]["speech"], return_tensors="pt", padding="longest", return_attention_mask=True)
+
+input_values = inputs.input_values
+attention_mask = inputs.attention_mask
+
+audio_cfg = AudioPretrainingConfig(labels="ltr", data="./data")
+task = AudioPretrainingTask.setup_task(audio_cfg)
+criterion = Wav2vecCriterion(Wav2VecCriterionConfig(infonce=True, log_keys=["prob_perplexity", "code_perplexity", "temp"], loss_weights=[0.1, 10]), task)
+
+sample = {
+    "net_input": {
+        "source": input_values,
+        "padding_mask": attention_mask.ne(1),
+    },
+    "id": torch.zeros((1,)),
+}
+
+torch.manual_seed(0)
+loss, sample_size, log, result = criterion(model, sample)
+torch.manual_seed(0)
+hf_result = hf_model(input_values, attention_mask=attention_mask, mask_time_indices=result["mask_indices"], fsq_negs=result["negs"])
+
+print("Loss diff %", 100 * (loss.detach().item() - hf_result.loss.detach().item()) / hf_result.loss.detach())
+
+print("perplexity diff %", 100 * (hf_result.codevector_perplexity.detach().item() - result["prob_perplexity"].detach().item()) / hf_result.codevector_perplexity.detach())

flax_wav2vec2/run_pt_fsq_comp.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+./run_pretraining_loss.py wav2vec2-large-lv60 wav2vec_vox_new.pt

flax_wav2vec2/wav2vec2-large-lv60

@@ -0,0 +1 @@
+Subproject commit 1f3eef2bbbac0a61cae0cf882fd615fc960a062f

flax_wav2vec2/wav2vec_vox_new.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b0748fbd4c725ff62266e3b9544cf948d117bc7fa2dc49528184de547736844
+size 3174007860