add data

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text
+*.tsv filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,21 @@
----
-license: apache-2.0
----
+## 代码
+https://github.com/XiaoshanHsj/speechllm/tree/multispk_lingmeng
+
+看代码的README_lingmeng.md
+
+## 数据：
+英文部分: `./data/Whisper-Sidecar-data-metadata/data_for_wavllm`
+
+德文相关: `./data/de-en-mix`
+
+targetASR (target-talker ASR)用到的reference audio: `./data/reference_enroll_audio/all`
+
+英文部分只有metadata，可以从librispeech生成。德文部分还备份了测试集音频。
+
+## 模型：
+tokenizer: `./llama_model/llama/tokenizer.model`
+
+llama-2-chat: `./llama_model/llama-2-7b-chat/consolidated.00.pth`
+
+训练好的MT-LLM模型目录: `./lingmeng_multispk_multitask_retrain_speechllm_v0.1_llama2_chat_wavlm_weighted_update_lora_32_32_prompt_build_multispk_multitask_de.yaml_16gpu_1accum`
+

data/Whisper-Sidecar-data-metadata/convert_to_wavllm_data_format.py

@@ -0,0 +1,42 @@
+import soundfile as sf
+import json
+from tqdm import tqdm
+
+input_jsonl = "/home/v-lingmeng/datasets/Whisper-Sidecar-data-metadata/data/librispeech3mix_test.jsonl"
+output_tsv = "/home/v-lingmeng/datasets/Whisper-Sidecar-data-metadata/data_for_wavllm/" + input_jsonl.split("/")[-1].replace(".jsonl", "1.tsv")
+print(output_tsv)
+head = "\t".join(["id", "audio", "n_frames", "prompt", "tgt_text", "codec", "with_speech", "language", "speakers", "genders"])
+prompts = ['Transcribe the given audio into text. If multiple speakers are speaking, transcribe the utterances of multiple speakers in the order of their start times, separated by "<sc>".']
+
+
+
+with open("/home/v-lingmeng/datasets/LibriSpeech/SPEAKERS.TXT", "r") as f:
+    speaker_info = f.readlines()
+    speaker_gender = {l.split("|")[0].strip():l.split("|")[1].strip() for l in speaker_info if not l.startswith(";")}
+with open(input_jsonl, "r") as f:
+    lines = f.readlines()
+new_lines = []
+for line in tqdm(lines):
+    line = json.loads(line.strip())
+    audio = line['audio']['path'].replace("./dataset", "/valleblob/v-lingmeng/speech/data")
+    wav_id = audio.split("/")[-1]
+    n_frames = str(sf.read(audio)[0].shape[0])
+    prompt = prompts[0]
+    tgt_text = line["sentence"]
+    codec = "None"
+    with_speech = "True"
+    language = "en"
+    if "speakers" in line:
+        speakers = "|".join(line["speakers"])
+    else:
+        speakers = "|".join([_id.split("-")[0] for _id in wav_id.split("_")])
+    genders = "|".join([speaker_gender[spk] for spk in speakers.split("|")])
+    
+    new_line = "\t".join([wav_id, audio, n_frames, prompt, tgt_text, codec, with_speech, language, speakers, genders])
+    new_lines.append(new_line)
+
+with open(output_tsv, "w") as f:
+    new_lines.insert(0, head)
+    f.write("\n".join(new_lines))
+
+# print(speaker_gender)

data/Whisper-Sidecar-data-metadata/data/aishell1mix2_dev.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:132bd6f6c6e353bca38831088bbe2eae65ce67635b5cc204249bd070f9c56e2a
+size 2460550

data/Whisper-Sidecar-data-metadata/data/aishell1mix2_test.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53b0d3301937ee8affd9a7da43c8c7940dc6f632b1594efa7de4b7faa4524f5b
+size 1483980

data/Whisper-Sidecar-data-metadata/data/aishell1mix2_test20.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53b0d3301937ee8affd9a7da43c8c7940dc6f632b1594efa7de4b7faa4524f5b
+size 1483980

data/Whisper-Sidecar-data-metadata/data/aishell1mix2_train.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a612cd6616cdb756e6c2ad587eebad584e76a1101e754d220bb01b22224c0221
+size 27055103

data/Whisper-Sidecar-data-metadata/data/aishell1mix3_dev.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bbe7c57288ad1565ffd0d630004508a26b48204b189cf30dd8a0f8f295e01a1
+size 2870838

data/Whisper-Sidecar-data-metadata/data/aishell1mix3_test.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2d2fb69a6fbe3cd6c89f89a002e28e01944d96c3d85a6cd240fcb52bfbd8ec2
+size 1442387

data/Whisper-Sidecar-data-metadata/data/aishell1mix3_train.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f583f9f8ff84df2ed1eb4da4dfd3a3d615d87c91bd4a3fda34dcf565028e76c
+size 23867370

data/Whisper-Sidecar-data-metadata/data/data_prepare_aishellmix.py

@@ -0,0 +1,71 @@
+# prepare from msra-dev-node
+
+import jsonlines
+import soundfile as sf
+import glob
+import numpy as np
+import os
+import pandas as pd
+
+def generate_jsonl_from_fairseq_datafile(root_data_dir, output_dir, with_timestamps=False):
+    splits = ['test', 'dev', 'train']
+    num_spks = ["2", "3"]
+    # convert transcript file to two list, ID and text
+    transcript_path = '/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/aishell1/data_aishell/transcript/aishell_transcript_v0.8.txt'
+    transcripts = open(transcript_path, 'r').readlines()        # ID\ttext\n for each line
+    ID_list = []
+    text_list = []
+    for line in transcripts:
+        line = line.strip()
+        _id, *text = line.split(' ')
+        ID_list.append(_id)
+        text_list.append(''.join(text))
+    
+    id_text_dict = dict(zip(ID_list, text_list))
+
+    for num_spk in num_spks:
+        for split in splits:
+            # data_dir = os.path,join(root_data_dir, 'Aishell1Mix', "data", f'Aishell1Mix{num_spk}', 'wav16k', 'max', split, 'mix_clean')
+            
+            metadata = os.path.join(root_data_dir, 'Aishell1Mix', "data", f'Aishell1Mix{num_spk}', 'wav16k', 'max', 'metadata', f'mixture_{split}_mix_clean.csv')
+            df = pd.read_csv(metadata)
+            mix_id_list = df['mixture_ID'].tolist()
+            mix_path_list = df['mixture_path'].tolist()
+            source_wav_root = os.path.join("/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/aishell1/data_aishell/wav", split)
+            new_jsonl = os.path.join(output_dir, f'aishell1mix{num_spk}_{split}.jsonl')
+            if os.path.exists(new_jsonl):
+                os.remove(new_jsonl)
+            
+            
+            for mix_id, mix_path in zip(mix_id_list, mix_path_list):
+                source_ids = mix_id.split('_')
+                source_texts = [id_text_dict[source_id] for source_id in source_ids]
+                source_text = '</s>'.join(source_texts)
+                
+                speakers = [source_id.split("S")[1].split("W")[0] for source_id in source_ids]
+                
+                source_paths = [os.path.join(source_wav_root, "S"+str(speakers[i]), source_id + '.wav') for i, source_id in enumerate(source_ids)]
+                source_durations = [sf.info(source_path).duration for source_path in source_paths]
+                
+                duration = max(source_durations)
+                dic = {"audio": {"path": mix_path}, 
+                        "language": "zh", 
+                        "duration": duration,
+                        "speakers": speakers,
+                        "sentence": source_text}
+
+                starts = [0] * len(source_durations)
+                ends = source_durations
+                sentences_dict = []
+                for i in range(len(starts)):
+                    sentences_dict.append({"start": starts[i], "end": ends[i], "text": source_texts[i]})
+                dic["sentences"] = sentences_dict
+                
+                with jsonlines.open(new_jsonl, mode='a') as writer:
+                    writer.write(dic)
+
+
+if __name__ == '__main__':    
+    root_data_dir = '/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset'
+    output_dir = '/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/'
+    generate_jsonl_from_fairseq_datafile(root_data_dir, output_dir, with_timestamps=True)

data/Whisper-Sidecar-data-metadata/data/data_prepare_librimix.py

@@ -0,0 +1,49 @@
+# prepare from SEPC
+
+import jsonlines
+# import soundfile as sf
+import glob
+import numpy as np
+import os
+
+def generate_jsonl_from_fairseq_datafile(root_data_dir, output_dir):
+    splits = ['test', 'dev', 'train']
+    num_spks = ["2", "3"]
+    
+    for num_spk in num_spks:
+        for split in splits:
+            data_dir = root_data_dir + f'Libri{num_spk}Mix_wav16k_max/'
+            wrd =  data_dir + split + '.wrd'
+            fairseq_jsonl = data_dir + split + '_clean.jsonl'
+            new_jsonl = output_dir + f"libri{num_spk}mix_" + split + '.jsonl'
+            if os.path.exists(new_jsonl):
+                os.remove(new_jsonl)
+            
+            with jsonlines.open(fairseq_jsonl) as reader:
+                with open(wrd, 'r') as f:
+                    for meta, text in zip(reader, f.readlines()):
+                        # print(meta, text)
+                        dic = {"audio": {"path": f"./dataset/LibriMix/data/Libri{num_spk}Mix/wav16k/max/" + meta['mixed_wav']}, 
+                            "language": "en", 
+                            "duration": max(meta['durations']),
+                            "speakers": meta['speakers'],
+                            "sentence": text.strip().lower()}
+
+                        starts = meta['delays']
+                        durations = meta['durations']
+                        ends = list(map(lambda x, y: x + y, starts, durations))
+                        texts = text.strip().lower().split(" </s> ")
+                        sentences_dict = []
+                        for i in range(len(starts)):
+                            sentences_dict.append({"start": starts[i], "end": ends[i], "text": texts[i].strip().lower()})
+                        dic["sentences"] = sentences_dict
+
+                        with jsonlines.open(new_jsonl, mode='a') as writer:
+                            writer.write(dic)
+                    
+             
+
+if __name__ == '__main__':    
+    root_data_dir = '/mnt/users/hccl.local/lmeng/workspaces/overlapASR/egs_wav2vec/data/'
+    output_dir = '/mnt/users/hccl.local/lmeng/workspaces/overlapASR/Whisper-Finetune-ovlp/dataset/'
+    generate_jsonl_from_fairseq_datafile(root_data_dir, output_dir)

data/Whisper-Sidecar-data-metadata/data/data_prepare_librispeech.py

@@ -0,0 +1,35 @@
+# prepare from SEPC
+
+import jsonlines
+# import soundfile as sf
+import glob
+import numpy as np
+
+def generate_jsonl_from_fairseq_datafile(data_dir, output_dir):
+    splits = ['test']
+    
+    for split in splits:
+        wrd =  data_dir + split + '.wrd'
+        # fairseq_jsonl = data_dir + split + '_clean.jsonl'
+        tsv = data_dir + split + '.tsv'
+        new_jsonl = output_dir + split + '.jsonl'
+        
+        with open(tsv, 'r') as flac_path_f:
+            # remove the first line
+            flac_path_f.readline()
+            with open(wrd, 'r') as trans_f:
+                for flac_path, trans in zip(flac_path_f.readlines(), trans_f.readlines()):
+                    # print(meta, text)
+                    flac_path, duration = flac_path.strip().split('\t')
+                    dic = {"audio": {"path": "./dataset/librispeech/" + flac_path}, 
+                           "language": "en", 
+                           "duration": int(duration)/16000.0,
+                           "sentence": trans.strip().lower()}
+                    with jsonlines.open(new_jsonl, mode='a') as writer:
+                        writer.write(dic)
+
+
+if __name__ == '__main__':    
+    data_dir = '/mnt/users/hccl.local/lmeng/workspaces/overlapASR/egs_wav2vec/data/LibriSpeech/'
+    output_dir = '/mnt/users/hccl.local/lmeng/workspaces/overlapASR/Whisper-Finetune/dataset/'
+    generate_jsonl_from_fairseq_datafile(data_dir, output_dir)

data/Whisper-Sidecar-data-metadata/data/data_prepare_librispeechmix.py

@@ -0,0 +1,53 @@
+# prepare from SEPC
+
+import jsonlines
+# import soundfile as sf
+import glob
+import numpy as np
+import os
+
+def generate_jsonl_from_fairseq_datafile(root_data_dir, output_dir, with_timestamps=False):
+    splits = ['test', 'dev', 'train']
+    num_spks = ["2", "3"]
+    for num_spk in num_spks:
+        for split in splits:
+            data_dir = root_data_dir + f'LibriSpeechMix-{num_spk}spkr/'
+            wrd =  data_dir + split + '.wrd'
+            fairseq_jsonl = data_dir + split + '_clean.jsonl'
+            if with_timestamps:
+                new_jsonl = output_dir + f"librispeech{num_spk}mix_timestamps_" + split + '.jsonl'
+            else:
+                new_jsonl = output_dir + f"librispeech{num_spk}mix_" + split + '.jsonl'
+
+            if os.path.exists(new_jsonl):
+                os.remove(new_jsonl)
+            
+            with jsonlines.open(fairseq_jsonl) as reader:
+                with open(wrd, 'r') as f:
+                    for meta, text in zip(reader, f.readlines()):
+                        # print(meta, text)
+                        dic = {"audio": {"path": "./dataset/LibriSpeechMix/" + meta['mixed_wav']}, 
+                            "language": "en", 
+                            "duration": max(list(map(lambda x, y: x + y, meta['durations'], meta['delays']))),
+                            "speakers": meta['speakers'],
+                            "sentence": text.strip().lower(),
+                            }
+                        if with_timestamps:
+                            starts = meta['delays']
+                            durations = meta['durations']
+                            ends = list(map(lambda x, y: x + y, starts, durations))
+                            texts = meta['texts']
+                            sentences_dict = []
+                            for i in range(len(starts)):
+                                sentences_dict.append({"start": starts[i], "end": ends[i], "text": texts[i].strip().lower()})
+                            dic["sentences"] = sentences_dict
+                        
+                        with jsonlines.open(new_jsonl, mode='a') as writer:
+                            writer.write(dic)
+                    
+             
+
+if __name__ == '__main__':    
+    root_data_dir = '/mnt/users/hccl.local/lmeng/workspaces/overlapASR/egs_wav2vec/data/'
+    output_dir = '/mnt/users/hccl.local/lmeng/workspaces/overlapASR/Whisper-Finetune-ovlp/dataset/'
+    generate_jsonl_from_fairseq_datafile(root_data_dir, output_dir, with_timestamps=True)

data/Whisper-Sidecar-data-metadata/data/generate_librimix_wav_from_jsonl.py

@@ -0,0 +1,60 @@
+import json
+import os
+import glob
+import soundfile
+import librosa
+from tqdm import tqdm
+import numpy as np
+
+def get_delayed_audio(wav_file, delay, sampling_rate=16000):
+    audio, _ = soundfile.read(wav_file)
+    delay_frame = int(delay * sampling_rate)
+    if delay_frame != 0:
+        audio = np.append(np.zeros(delay_frame), audio)
+    return audio
+
+def mix_audio(wav_files, delays):
+    for i, wav_file in enumerate(wav_files):
+        if i == 0:
+            audio = get_delayed_audio(wav_file, delays[i])
+        else:
+            additional_audio = get_delayed_audio(wav_file, delays[i])
+            # tune length & sum up to audio
+            target_length = max(len(audio), len(additional_audio))
+            # print(additional_audio.shape)
+            audio = librosa.util.fix_length(audio, size=target_length)
+            additional_audio = librosa.util.fix_length(additional_audio, size=target_length)
+            audio = audio + additional_audio
+    return audio
+
+
+jsonl_path = "/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/libri3mix_train.jsonl"
+if "test" in jsonl_path:
+    subset = "test"
+elif "train" in jsonl_path:
+    subset = "train"
+else:
+    subset = "dev"
+librispeech_dir = f"/home/v-lingmeng/datasets/LibriSpeech/{subset}*/"
+output_dir = "/home/v-lingmeng/datasets"
+
+
+with open(jsonl_path, 'r', encoding='utf-8') as file:
+    json_list = [json.loads(line.strip()) for line in file]
+
+for line in tqdm(json_list):
+    audio_name = line['audio']['path'].replace("./dataset", output_dir)
+    if not os.path.exists(os.path.split(audio_name)[0]):
+        os.makedirs(os.path.split(audio_name)[0])
+    
+    source_ids = os.path.split(audio_name)[1].split(".")[0].split("_")
+    source_files = [glob.glob(librispeech_dir + "/".join(i.split("-")[:-1]) + f"/{i}*")[0] for i in source_ids]
+    delays = [l["start"] for l in line['sentences']]
+    # ends = [l["end"] for l in line['sentences']]
+    
+    # for source_audio, start, end in zip(source_auidos, starts, ends):
+    mixed_audio = mix_audio(source_files, delays)
+    soundfile.write(audio_name, mixed_audio, samplerate=16000)
+    print(audio_name)
+    
+# print(delays)

data/Whisper-Sidecar-data-metadata/data/libri2mix_dev-both.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a00333b0e46f84e4389e63df5045f8a762487eb29bb89eb7de9f70941e1434c
+size 2224222

data/Whisper-Sidecar-data-metadata/data/libri2mix_dev.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c08740dd203b33312ea9793a8859302c2ff41f44de77ffb404c6d5859df2287
+size 2227222

data/Whisper-Sidecar-data-metadata/data/libri2mix_test-both.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88e089c638beff65237045f526348e50341026b0324e0230e1ee17568b3173c1
+size 2094572

data/Whisper-Sidecar-data-metadata/data/libri2mix_test.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbf70cb2bca21af6465f534a1910756cce13eb410815eccc90fc74b24567e4b6
+size 2097572

data/Whisper-Sidecar-data-metadata/data/libri2mix_test20.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9831f61da77ba9a17367845c6288c9a7b24858cf69dcbb53e4377cb23640820b
+size 14417

data/Whisper-Sidecar-data-metadata/data/libri2mix_train-100-both.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f0eb2462fa951da717d18e172a5b781b6149d3483e56022bcea7459d163b76e
+size 14778639

data/Whisper-Sidecar-data-metadata/data/libri2mix_train-100.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10ac8db4f8c3851c7e34f4643268355c2d34a7f8952aa316b0b70d2019733f70
+size 14792539

data/Whisper-Sidecar-data-metadata/data/libri2mix_train-200.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94e70011854f573bf05586a9793f19510d6be7188be89c032459910ad1d3e11d
+size 29571178

data/Whisper-Sidecar-data-metadata/data/libri2mix_train-both.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f72957a086f85d8fee3aea616c471dd7c7baf465d355db361f796f13c7e6478e
+size 68694922

data/Whisper-Sidecar-data-metadata/data/libri2mix_train.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9bd7eceb100a8f842e1b708066e5da7ac77e5008ef013266c567528ab1f8c4f
+size 68759621

data/Whisper-Sidecar-data-metadata/data/libri2mix_train20.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41d2502e124c4cdf60adb5a83f8942f2fb5d710634485b5a0382e45873570048
+size 22316

data/Whisper-Sidecar-data-metadata/data/libri2mix_train_remove_enroll.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3d3c34060e44c767cd70ebb53aa18d53db4a3828fa92037addeed9c04e9f8a5
+size 68268344

data/Whisper-Sidecar-data-metadata/data/libri3mix_test.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daf3f6411bb717e86d77b4f17a455323c5d9df5fdb37fe6513a11859d50f41e9
+size 2855611

data/Whisper-Sidecar-data-metadata/data/libri3mix_test20.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f66ad5735f3e3d1bd24bf960439142e181dbe4ab50a951a3807b4050040568c
+size 19612

data/Whisper-Sidecar-data-metadata/data/libri3mix_train.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ba19b8a28a4430dbab099d5450ae075f6096c869ca318846f1ed219b471c997
+size 65232138

data/Whisper-Sidecar-data-metadata/data/librispeech2mix_test.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dfd13ad50aeb92257fc7c8c4d721fad1d2e3901be95eed0aa4d58211027d471
+size 1945661

data/Whisper-Sidecar-data-metadata/data/librispeech2mix_test_30s.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1d07d099fe7d64cfec16efcc06b5dcac825984ca8668909152882adef2edcb9
+size 1945138

data/Whisper-Sidecar-data-metadata/data/librispeech2mix_train.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f629fb7f89d02d937e5f4d1ced117c70904fc5d97d520706758a3e41010c1c2
+size 287742080

data/Whisper-Sidecar-data-metadata/data/librispeech3mix_test.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4019227f5a2f827f2517e942c8c6f72b7a6d80fe8fe32db2909393254d7af771
+size 2730857

data/Whisper-Sidecar-data-metadata/data/librispeech3mix_test_temp.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be1fafb707b168a23d3c8225e3f04407da63360b27b99d435d11c8a7560dee9c
+size 2729249

data/Whisper-Sidecar-data-metadata/data/librispeech3mix_train.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47917569ed5a933e6ad66ccef5ccefded8f4c117a0296dafcbde9fc564e835e6
+size 410988677

data/Whisper-Sidecar-data-metadata/data/librispeech_dev.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca54eb22dd7f421bd9c8b8e60a5df32946a7823d5547e49eaa030769c48de0da
+size 2194617

data/Whisper-Sidecar-data-metadata/data/librispeech_test.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8421fbe45b046b2a5644782b3e982538bd8483d72964b6f08fc208e5e7059648
+size 2197082

data/Whisper-Sidecar-data-metadata/data/librispeech_train.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc923ead53a1248037e622bbeb75899a0c2637720600cc8c386530d5cdfa95b8
+size 91088243

data/Whisper-Sidecar-data-metadata/data/long_wav_resample.py

@@ -0,0 +1,52 @@
+# resamle long wav (>30s) to 16k 30s, and update the jsonl file
+
+
+import jsonlines
+import os
+
+from pydub import AudioSegment
+from pydub.playback import play
+import soundfile as sf
+
+
+
+jsonl_file = "/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/librispeech3mix_test.jsonl"
+temp_file = "/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/librispeech3mix_test_temp.jsonl"
+# time strench long wav (>30s) to 16k 30s, and update the jsonl line in-place
+
+with jsonlines.open(jsonl_file, 'r') as reader, jsonlines.open(temp_file, 'w') as writer:
+    for obj in reader:
+        wav_path = obj['audio']['path']
+        duration = obj['duration']
+        sentences = obj['sentences']
+        # resample wav
+        if duration > 30.1:
+            print(wav_path, duration)
+            wav = AudioSegment.from_file(wav_path)
+            target_len = 30.0 * 1000
+            speed_up_rate = len(wav) / target_len
+            wav = wav.speedup(playback_speed=speed_up_rate)
+            wav = wav[:target_len]
+            wav.export(wav_path, format="wav")
+            print(speed_up_rate)
+            obj['duration'] = 30
+            for sentence in sentences:
+                sentence['start'] = sentence['start'] / speed_up_rate
+                sentence['end'] = sentence['end'] / speed_up_rate
+                if sentence['start'] > 30:
+                    sentence['start'] = 30
+                if sentence['end'] > 30:
+                    sentence['end'] = 30
+            obj['sentences'] = sentences
+        elif duration > 30.0:
+            wav, sr = sf.read(wav_path)
+            wav = wav[:int(16000 * 30)]
+            obj['duration'] = 30 if duration > 30 else duration
+            for sentence in sentences:
+                sentence['start'] = sentence['start'] if sentence['start'] < 30 else 30
+                sentence['end'] = sentence['end'] if sentence['end'] < 30 else 30
+            obj['sentences'] = sentences
+            sf.write(wav_path, wav, sr)
+
+        
+        writer.write(obj)

data/Whisper-Sidecar-data-metadata/data/select_prompt_wav.py

@@ -0,0 +1,132 @@
+import os
+import glob 
+import random 
+
+import jsonlines
+
+# backup_dir = "/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/temp/removed_mix_wav"
+# librispeech_dir = "/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/LibriSpeech/train-clean-360"
+# enroll_dir = "/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/temp/new_enroll"
+
+# # 1. 获取所有的wav文件
+# wav_files = glob.glob("/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/LibriMix/data/Libri2Mix/wav16k/max/train-360/mix_clean/*.wav")
+
+# # 2. 每个wav文件有两个说话人，记录所有wav文件出现的说话人
+# all_speakers = set()
+# for wav_file in wav_files:
+#     speakers = [f.split('-')[0] for f in os.path.basename(wav_file).split("_")]
+#     all_speakers.update(speakers)
+# # random.shuffle(wav_files)
+
+# len_all_speakers = len(all_speakers)
+# # 3. 对每个说话人，复制且只复制一个具有它的语音文件
+# count = 0
+# for wav_file in wav_files:
+#     source_wavs = os.path.basename(wav_file).split("_")
+#     speakers = [f.split('-')[0] for f in source_wavs]
+#     # 如果有任意一个说话人不在all_speakers中，跳过这个文件
+#     if not all(s in all_speakers for s in speakers):
+#         continue
+#     else:
+#         # 从all_speakers中删除这两个说话人
+#         all_speakers.difference_update(speakers)
+#         # 复制这个文件
+#         os.system(f"cp {wav_file} {backup_dir}")
+#         # print(f"cp {wav_file} {backup_dir}")
+#         # 复制source_wavs中的每个说话人的语音文件
+#         for source_wav in source_wavs:
+#             count+=1
+#             source_wav_path = os.path.join(librispeech_dir, source_wav.split("-")[0], source_wav.split("-")[1], source_wav.split('.')[0] + ".flac")
+#             # 判断是否存在
+#             if not os.path.exists(source_wav_path):
+#                 print(f"source_wav_path: {source_wav_path} not exists")
+#                 continue
+#             os.system(f"cp {source_wav_path} {enroll_dir}")
+            
+# print(all_speakers)
+# print(count, len_all_speakers)
+
+
+
+# source = "/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/temp/new_enroll"
+# enroll_dir = "/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/LibriMix_enroll_audio/train-360"
+
+# flac_files = glob.glob(f"{source}/*.flac")
+
+# for flac_files in flac_files:
+#     # mkdir enroll_dir/spk_id
+#     spk_id = os.path.basename(flac_files).split("-")[0]
+#     spk_dir = os.path.join(enroll_dir, spk_id)
+#     if os.path.exists(spk_dir):
+#         # 删除
+#         os.system(f"rm -rf {spk_dir}")
+#     os.makedirs(spk_dir, exist_ok=True)
+#     # convert flac to wav, move to spk_dir
+#     wav_file = os.path.join(spk_dir, spk_id+ ".wav")
+#     os.system(f"ffmpeg -i {flac_files} {wav_file}")
+    
+
+# jsonl_file = "/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/libri2mix_train_remove_enroll.jsonl"
+# enrolled_dir = "/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/temp/removed_mix_wav"
+
+# # remove wav file in enrolled_dir from jsonl_file
+# with jsonlines.open(jsonl_file) as reader:
+#     lines = list(reader)
+#     print(len(lines))
+#     for line in lines:
+#         mix_wav = line['audio']['path']
+#         if os.path.exists(os.path.join(enrolled_dir, os.path.basename(mix_wav))):
+#             lines.remove(line)
+#     print(len(lines))
+# # write to new jsonl file
+# new_jsonl_file = "/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/libri2mix_train_remove_enroll.jsonl"
+# with jsonlines.open(new_jsonl_file, "w") as writer:
+#     for line in lines:
+#         writer.write(line)
+# print("done")
+
+
+librispeech_dir = "/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/LibriSpeech/train-other-500"
+enroll_path ="/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/LibriMix_enroll_audio/train-500"
+
+# 检查librispeech_dir中每个speaker是否存在在enroll_path中
+speaker_dirs = glob.glob(f"{librispeech_dir}/*")
+new_file = []
+for speaker_dir in speaker_dirs:
+    if ".TXT" in speaker_dir:
+        continue
+    speaker_id = os.path.basename(speaker_dir)
+    enroll_speaker_dir = os.path.join(enroll_path, speaker_id)
+    if not os.path.exists(enroll_speaker_dir):
+        print(f"{enroll_speaker_dir} not exists")
+        os.makedirs(enroll_speaker_dir, exist_ok=True)
+
+    if len(glob.glob(f"{enroll_speaker_dir}/*.wav")) == 0:
+        # 从librispeech_dir中复制一个语音文件到enroll_speaker_dir
+        flac_files = glob.glob(f"{speaker_dir}/*/*.flac")
+        # 从flac_files中随机选择一个, 并复制到enroll_speaker_dir
+        flac_file = random.choice(flac_files)
+        # 复制到enroll_speaker_dir
+        new_flac_file = os.path.join(enroll_speaker_dir, os.path.basename(flac_file))
+        print(new_flac_file)
+        os.system(f"cp {flac_file} {new_flac_file}")
+        # 记录flac_file的名字
+        new_file.append(flac_file)
+    else:
+        print(glob.glob(f"{enroll_speaker_dir}/*.wav"))
+    
+    # 检查enroll_path中speaker的语音命名是否为{speaker_id}.wav
+    enroll_wav_files = glob.glob(f"{enroll_speaker_dir}/*")
+    for enroll_wav_file in enroll_wav_files:
+        if os.path.basename(enroll_wav_file).split(".")[0] != speaker_id:
+            print(f"{enroll_wav_file} not match")
+            # 转为wav格式, 名字只保留speaker_id
+            wav_file = os.path.join(enroll_speaker_dir, speaker_id + ".wav")
+            os.system(f"ffmpeg -i {enroll_wav_file} {wav_file}")
+            # 删除原来的文件
+            os.system(f"rm -rf {enroll_wav_file}")
+
+# 记录新的flac文件
+with open("/home/v-lingmeng/codebase/Whisper-Finetune-ovlp/dataset/temp/new_flac_files.txt", "w") as f:
+    for flac_file in new_file:
+        f.write(flac_file + "\n")

data/Whisper-Sidecar-data-metadata/data/test_examples.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcf048b062b475972c1e6dad258244680a88a9acfc8841dd8ea284ad56efc3dc
+size 794

data/Whisper-Sidecar-data-metadata/data_for_wavllm/de-en-2mix_test.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53fb93ebd24db3e32a7dd7fd5908222f2f0a498a7d0ede6835140c2d4b1f3552
+size 8892480

data/Whisper-Sidecar-data-metadata/data_for_wavllm/de-en-2mix_test_1350.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec64b8b840655de399e374931c056dd4b22d4070c8b97cb197863182599af765
+size 889784

data/Whisper-Sidecar-data-metadata/data_for_wavllm/de-en-2mix_test_1350_targetLingual.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:971baf65ee1603b22483ef12a1040d884419077069b68dad7931a305ffe2f7aa
+size 632636

data/Whisper-Sidecar-data-metadata/data_for_wavllm/de-en-2mix_test_targetLingual.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12249076735100be44176352c65ab89a30751ae7ac5621583130674529774d57
+size 6361360

data/Whisper-Sidecar-data-metadata/data_for_wavllm/de-en-2mix_test_targetLingual_1350.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:968fd7a78bdca0783e5a2baf7db53f621ed274ba1295f5bd4d2553794a4718da
+size 636201

data/Whisper-Sidecar-data-metadata/data_for_wavllm/de-en-2mix_train.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21cb5e6b2243c0ce5a2cf6e431dd1c9c5dc5efb79fb3c2a9d548dbbc335c212f
+size 93936028

data/Whisper-Sidecar-data-metadata/data_for_wavllm/de-en-2mix_train_targetLingual.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aad7b8718ed09f7e98742caa6c85e496f68e332f73e65a557ecdac06cec8f2a
+size 65029852

data/Whisper-Sidecar-data-metadata/data_for_wavllm/de-en-3mix_test.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2e84678170247fd7ef2a06622a32d9d42abf3e1b83e8c4426e4a9d03e44ce72
+size 10851471