diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..a6fd5975f10b2f1727983876f5b621c3f9a72cdb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +An[[:space:]]Efficient[[:space:]]Speaker[[:space:]]Diarization[[:space:]]Pipeline[[:space:]]for[[:space:]]Conversational[[:space:]]Speech.pdf filter=lfs diff=lfs merge=lfs -text +Attention[[:space:]]is[[:space:]]All[[:space:]]You[[:space:]]Need[[:space:]]in[[:space:]]Speech[[:space:]]Separation.pdf filter=lfs diff=lfs merge=lfs -text +AV-SepFormer.[[:space:]]Cross-Attention[[:space:]]SepFormer[[:space:]]for[[:space:]]Audio-Visual[[:space:]]Target[[:space:]]Speaker[[:space:]]Extraction.pdf filter=lfs diff=lfs merge=lfs -text +Dual-path[[:space:]]RNN.[[:space:]]Efficient[[:space:]]long[[:space:]]sequence[[:space:]]modeling[[:space:]]for[[:space:]]time-domain[[:space:]]single-channel[[:space:]]speech[[:space:]]separation.pdf filter=lfs diff=lfs merge=lfs -text +models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/example_dns4-16k.wav filter=lfs diff=lfs merge=lfs -text +models/Sepformer/de/sepformer_rescuespeech/example_rescuespeech16k.wav filter=lfs diff=lfs merge=lfs -text +models/Sepformer/en/sepformer-whamr-enhancement/example_whamr.wav filter=lfs diff=lfs merge=lfs -text +SepFormer.[[:space:]]Coarse-to-fine[[:space:]]Separator[[:space:]]Regression[[:space:]]Network[[:space:]]for[[:space:]]Table[[:space:]]Structure[[:space:]]Recognition.pdf filter=lfs diff=lfs merge=lfs -text diff --git a/AV-SepFormer. Cross-Attention SepFormer for Audio-Visual Target Speaker Extraction.pdf b/AV-SepFormer. Cross-Attention SepFormer for Audio-Visual Target Speaker Extraction.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b2b408bb6d9156ab7cdcf0699c74cdf5be9708cb --- /dev/null +++ b/AV-SepFormer. Cross-Attention SepFormer for Audio-Visual Target Speaker Extraction.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc48943d188a9da9392dbe810fc1de2b3fe56c891eed621dd9572baf06944a64 +size 474358 diff --git a/An Efficient Speaker Diarization Pipeline for Conversational Speech.pdf b/An Efficient Speaker Diarization Pipeline for Conversational Speech.pdf new file mode 100644 index 0000000000000000000000000000000000000000..bf0d47f9253939022d0df94fee177fa8dda71d26 --- /dev/null +++ b/An Efficient Speaker Diarization Pipeline for Conversational Speech.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5de3dd5af601cadc673dd42bba5178f23846dfbc7aca0b1e30d170fa91ede8 +size 823356 diff --git a/Attention is All You Need in Speech Separation.pdf b/Attention is All You Need in Speech Separation.pdf new file mode 100644 index 0000000000000000000000000000000000000000..80ee66ced4936ad2a5e350ce6a2ac8a034129d17 --- /dev/null +++ b/Attention is All You Need in Speech Separation.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96ec5acd6458a965b79f511cb87a9b5bd317a7057b45bc0c51eb96e2bd3ca33f +size 271339 diff --git a/Dual-path RNN. Efficient long sequence modeling for time-domain single-channel speech separation.pdf b/Dual-path RNN. Efficient long sequence modeling for time-domain single-channel speech separation.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f5f50bcf5d283597ff1c9dd3303d4a37c396e9a5 --- /dev/null +++ b/Dual-path RNN. Efficient long sequence modeling for time-domain single-channel speech separation.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db0922e9a32d0d781e11103b477c879a69b7690b408ef2032ae711276dd5eeb1 +size 322545 diff --git a/SepFormer. Coarse-to-fine Separator Regression Network for Table Structure Recognition.pdf b/SepFormer. Coarse-to-fine Separator Regression Network for Table Structure Recognition.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fffb4a0de357d336d8d6e1c0dbee4793f58fe08a --- /dev/null +++ b/SepFormer. Coarse-to-fine Separator Regression Network for Table Structure Recognition.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79fe6233c1909a9061565fe32e9e9996cfacadc5e268b2e7381a8127977e2a89 +size 16477595 diff --git a/code/AV-Sepformer.zip b/code/AV-Sepformer.zip new file mode 100644 index 0000000000000000000000000000000000000000..c5ff7de8fc0f289eae9dc21a0c1cfaa75b410b2e --- /dev/null +++ b/code/AV-Sepformer.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9bb563e66c495d5000ff7390abc83ff05290492510c6195b18023c496e93ade +size 2607359 diff --git a/code/Separation-of-vocals-in-songs-based-on-sepformer.zip b/code/Separation-of-vocals-in-songs-based-on-sepformer.zip new file mode 100644 index 0000000000000000000000000000000000000000..7ff1c84f916487235c4adfb23486674d857ece48 --- /dev/null +++ b/code/Separation-of-vocals-in-songs-based-on-sepformer.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cc95fa4ec8b87e03a4f87431bafd2bdf154b4a247af8583757ec405063cdb8c +size 93790 diff --git a/code/SeparationForKazakhSpeech.zip b/code/SeparationForKazakhSpeech.zip new file mode 100644 index 0000000000000000000000000000000000000000..ca895264f534a1606380d69c7538156df77041f8 --- /dev/null +++ b/code/SeparationForKazakhSpeech.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e3702bef31536be2b76f2e3d00c2ae870d935573de4aab24c906037d6feec19 +size 10157402 diff --git a/code/Sepformer.zip b/code/Sepformer.zip new file mode 100644 index 0000000000000000000000000000000000000000..1e57dff8bcbec031dcca35fe121c88dbba8ddde6 --- /dev/null +++ b/code/Sepformer.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecb2a2dccbfe81108666dfbce9ff2bc22df3c24ab2de42abe305bfefe6709b2b +size 7977839 diff --git a/code/SpeechSeparation.zip b/code/SpeechSeparation.zip new file mode 100644 index 0000000000000000000000000000000000000000..41143e2e0de08a20470ebe6d91ff03a494161e17 --- /dev/null +++ b/code/SpeechSeparation.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:082c00f26761812b0da192e69b6eabb4bb18dafebdd4bb483d3973c24ae1e857 +size 105214371 diff --git a/code/X-SepFormer.github.io.zip b/code/X-SepFormer.github.io.zip new file mode 100644 index 0000000000000000000000000000000000000000..c6d0f846151ae742fc86e13930ee4496477c0fc5 --- /dev/null +++ b/code/X-SepFormer.github.io.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72dc134d8d0c7d59a5f1ad9a74d23231644a90ed52d8b7fa92b97ea7791f0122 +size 44383489 diff --git a/code/mss-sepformer.zip b/code/mss-sepformer.zip new file mode 100644 index 0000000000000000000000000000000000000000..0de9bccf671f5a548e2bb85eab3f61802e87f355 --- /dev/null +++ b/code/mss-sepformer.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5777b26176f99f9d19fdc6f6e5045ec7a4ea5f2368f9234ad0088fb3f3f16816 +size 701807 diff --git a/code/sb_sepformer.zip b/code/sb_sepformer.zip new file mode 100644 index 0000000000000000000000000000000000000000..04322a0c31c821eca4fb13a34f2dc1c0642e94b1 --- /dev/null +++ b/code/sb_sepformer.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bd051cdf945d16eb99ac12b856619e81555ff674d4f44463a2713b53faa4b35 +size 55281 diff --git a/code/sepformer-48k.zip b/code/sepformer-48k.zip new file mode 100644 index 0000000000000000000000000000000000000000..864876039db307bbc338298893fb31b9e99c09fe --- /dev/null +++ b/code/sepformer-48k.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db0096ee407cbeaca6b200b4a727b9917c81ecb1112fc8057919886607a43fec +size 122754980 diff --git a/code/sepformer-tse.zip b/code/sepformer-tse.zip new file mode 100644 index 0000000000000000000000000000000000000000..dde78456e78c462c311c4f5f8b57dbdd6fe4b9aa --- /dev/null +++ b/code/sepformer-tse.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13905e009d88354fcb21579e11f5dcebe3114eb2d238eab0fa4cc11f9cc237ea +size 114198916 diff --git a/code/train-test-sepFormer.zip b/code/train-test-sepFormer.zip new file mode 100644 index 0000000000000000000000000000000000000000..3ee841cd8220a23ddf174ecfb5ce9b5f951b8eb7 --- /dev/null +++ b/code/train-test-sepFormer.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53c6d8346e1ceabb7a1d5469ca05f9878efb2d31332e2e373de6051d4632c370 +size 48661 diff --git a/models/AV-Sepformer/MuSE.pt b/models/AV-Sepformer/MuSE.pt new file mode 100644 index 0000000000000000000000000000000000000000..94799c3e0a71f063586721c8f5670dc96d89b091 --- /dev/null +++ b/models/AV-Sepformer/MuSE.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11e6e10a80523133070e214fa7475333f29c65bc8cb8a82cc446b72dea8a01b8 +size 60355363 diff --git a/models/AV-Sepformer/av-convtasnet.pt b/models/AV-Sepformer/av-convtasnet.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1c4b97375acdeab0850c3bd183f68b3e8e8ca99 --- /dev/null +++ b/models/AV-Sepformer/av-convtasnet.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8621a009271a39e8b8a20e26a202527f45eddb7cd8a3ad28868a87d2e2503d04 +size 44310819 diff --git a/models/AV-Sepformer/av-sepformer.pt b/models/AV-Sepformer/av-sepformer.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bd099e353ef23d35607b56cf0ae6261ae281780 --- /dev/null +++ b/models/AV-Sepformer/av-sepformer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a036fb38dcb02c6ce49034ca537e9206290348ee21d4407c9fc38489132ba9b +size 118763721 diff --git a/models/AV-Sepformer/source.txt b/models/AV-Sepformer/source.txt new file mode 100644 index 0000000000000000000000000000000000000000..e3e6908c996ad938371876628a1397421eb08da5 --- /dev/null +++ b/models/AV-Sepformer/source.txt @@ -0,0 +1,2 @@ +https://github.com/lin9x/AV-Sepformer +https://drive.google.com/drive/folders/1M26x5qCE0LuaEtZ76E_YSnW0hf_chGeq \ No newline at end of file diff --git a/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/.gitattributes b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/README.md b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fa3e055d13b16c1549a8c2bdce690842791b4d75 --- /dev/null +++ b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/README.md @@ -0,0 +1,136 @@ +--- +language: + - "en" + - "de" + - "ru" + - "fr" + - "it" + - "es" +thumbnail: +tags: +- audio-to-audio +- Speech Enhancement +- DNS-4 +- SepFormer +- Transformer +- pytorch +- speechbrain +- Microsoft DNS Challenge +- Deep Noise Suppression Challenge – ICASSP 2022 +license: "apache-2.0" +datasets: +- DNS-4 +metrics: +- SI-SNR +- PESQ +- SIG +- BAK +- OVRL +model-index: +- name: sepformer-dns4-16k-enhancement + results: + - task: + name: Speech Enhancement + type: speech-enhancement + dataset: + name: DNS-4 + type: deep-noise-suppression-challenge-icassp-2022 + split: baseline-dev-set + args: + language: de + metrics: + - name: DNSMOS SIG + type: sig + value: '2.999' + - name: DNSMOS BAK + type: bak + value: '3.076' + - name: DNSMOS OVRL + type: ovrl + value: '2.437' +--- + + +

+ +# SepFormer trained on Microsoft DNS-4 (Deep Noise Suppression Challenge 4 – ICASSP 2022) for speech enhancement (16k sampling frequency) +This repository provides all the necessary tools to perform speech enhancement (denoising) with a [SepFormer](https://arxiv.org/abs/2010.13154v2) model, implemented with SpeechBrain. The model is trained on 1300HRS of Microsoft-DNS 4 dataset with 16k sampling frequency. For a better experience we encourage you to learn more about [SpeechBrain](https://speechbrain.github.io). Evaluation on DNS4 2022 baseline dev set using DNSMOS are- + + +| Release | SIG | BAK | OVRL | +|:-------------:|:--------------:|:--------------:|:--------------:| +| 08-01-23 | 2.999 | 3.076 | 2.437 | + +DNSMOS - deep noise suppression (DNS)- mean opinion score (MOS) is a non-intrusive evaluation metric. It computes 3 scores– SIG (speech quality), BAK (background noise quality), and OVRL (overall quality) each on a scale of 1 to 5, with 5 being the best quality. + +## Install SpeechBrain + +First of all, please install SpeechBrain with the following command: + +``` +pip install speechbrain +``` + +Please notice that we encourage you to read our tutorials and learn more about [SpeechBrain](https://speechbrain.github.io). + +### Perform speech enhancement on your own audio file + +```python +from speechbrain.inference.separation import SepformerSeparation as separator +import torchaudio + +model = separator.from_hparams(source="speechbrain/sepformer-dns4-16k-enhancement", savedir='pretrained_models/sepformer-dns4-16k-enhancement') + +# for custom file, change path +est_sources = model.separate_file(path='speechbrain/sepformer-dns4-16k-enhancement/example_dns4-16k.wav') + +torchaudio.save("enhanced_dns4-16k.wav", est_sources[:, :, 0].detach().cpu(), 16000) + +``` + +### Inference on GPU +To perform inference on the GPU, add `run_opts={"device":"cuda"}` when calling the `from_hparams` method. + +You can find our training results (models, logs, etc) [here](https://www.dropbox.com/sh/02c3wesc65402f6/AAApoxBApft-JwqHK-bddedBa?dl=0). + +### Limitations +The SpeechBrain team does not provide any warranty on the performance achieved by this model when used on other datasets. + +### Referencing SpeechBrain +```bibtex +@misc{speechbrain, + title={{SpeechBrain}: A General-Purpose Speech Toolkit}, + author={Mirco Ravanelli and Titouan Parcollet and Peter Plantinga and Aku Rouhe and Samuele Cornell and Loren Lugosch and Cem Subakan and Nauman Dawalatabad and Abdelwahab Heba and Jianyuan Zhong and Ju-Chieh Chou and Sung-Lin Yeh and Szu-Wei Fu and Chien-Feng Liao and Elena Rastorgueva and François Grondin and William Aris and Hwidong Na and Yan Gao and Renato De Mori and Yoshua Bengio}, + year={2021}, + eprint={2106.04624}, + archivePrefix={arXiv}, + primaryClass={eess.AS}, + note={arXiv:2106.04624} +} +``` + +### Referencing SepFormer +```bibtex +@inproceedings{subakan2021attention, + title={Attention is All You Need in Speech Separation}, + author={Cem Subakan and Mirco Ravanelli and Samuele Cornell and Mirko Bronzi and Jianyuan Zhong}, + year={2021}, + booktitle={ICASSP 2021} +} +``` + +### Referencing ICASSP 2022 Deep Noise Suppression Challenge +```bibtex +@inproceedings{dubey2022icassp, + title={ICASSP 2022 Deep Noise Suppression Challenge}, + author={Dubey, Harishchandra and Gopal, Vishak and Cutler, Ross and Matusevych, Sergiy and Braun, Sebastian and Eskimez, Emre Sefik and Thakker, Manthan and Yoshioka, Takuya and Gamper, Hannes and Aichner, Robert}, + booktitle={ICASSP}, + year={2022} +} +``` + + +# **About SpeechBrain** +- Website: https://speechbrain.github.io/ +- Code: https://github.com/speechbrain/speechbrain/ +- HuggingFace: https://huggingface.co/speechbrain/ \ No newline at end of file diff --git a/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/config.json b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c97b7fa67edd2ea949e48af55f4af3221e95795e --- /dev/null +++ b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/config.json @@ -0,0 +1,3 @@ +{ + "speechbrain_interface": "SepformerSeparation" +} diff --git a/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/decoder.ckpt b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/decoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..12c796dac129f1fcc5f341ed9834dc8c9606a77b --- /dev/null +++ b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/decoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab10ab1aac337bbe77ad0cbdf75acb1fba952e2d34474d515de2f41ddfd9a24c +size 17195 diff --git a/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/encoder.ckpt b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/encoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7e88b1a96a6af25073743f97b0b92d04502d1580 --- /dev/null +++ b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/encoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33de46bbf0b55031f92346827723137478cf2f77204fd1360ba8833b5915e0ee +size 17259 diff --git a/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/example_dns4-16k.wav b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/example_dns4-16k.wav new file mode 100644 index 0000000000000000000000000000000000000000..b9f8717495616f77b8ffbc3d0065641cda26e23c --- /dev/null +++ b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/example_dns4-16k.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09ca62a0abb72b896219a49fadd9ede1aa5a86610e35e352681a0b3aa0f20341 +size 256058 diff --git a/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/hyperparams.yaml b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f482ea8f282f7c42f46029218dc5475d10c483a --- /dev/null +++ b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/hyperparams.yaml @@ -0,0 +1,67 @@ +# Generated 2023-02-21 from: +# /netscratch/sagar/thesis/speechbrain/recipes/DNS/enhancement/hparams/resume-sepformer-dns_16k.yaml +# yamllint disable +# ################################ +# Model: SepFormer for source separation +# https://arxiv.org/abs/2010.13154 +# +# Author: Sangeet Sagar 2022 +# Dataset : Microsoft-DNS +# ################################ + +# Basic parameters +sample_rate: 16000 +num_spks: 1 + +# Specifying the network +Encoder: &id003 !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + +SBtfintra: &id001 !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: &id002 !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: &id005 !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + num_spks: 1 + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: *id001 + inter_model: *id002 + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: &id004 !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +modules: + encoder: *id003 + decoder: *id004 + masknet: *id005 + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + encoder: !ref + masknet: !ref + decoder: !ref \ No newline at end of file diff --git a/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/masknet.ckpt b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/masknet.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7f07f9724bc1f3ac47c3209d5fa83011fc7572d2 --- /dev/null +++ b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/masknet.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfba4276bf051fc48b1d49350ef0612845328c1999cefb4c637fc02cdbabbbdd +size 112839555 diff --git a/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/source.txt b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/source.txt new file mode 100644 index 0000000000000000000000000000000000000000..965a8cdf03e97aa310199dac3b1ada096f4e78c5 --- /dev/null +++ b/models/Sepformer/de,en,es,fr,it,ru/sepformer-dns4-16k-enhancement/source.txt @@ -0,0 +1 @@ +https://huggingface.co/speechbrain/sepformer-dns4-16k-enhancement \ No newline at end of file diff --git a/models/Sepformer/de/sepformer_rescuespeech/.gitattributes b/models/Sepformer/de/sepformer_rescuespeech/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/models/Sepformer/de/sepformer_rescuespeech/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/models/Sepformer/de/sepformer_rescuespeech/README.md b/models/Sepformer/de/sepformer_rescuespeech/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3c5acc1b0e869405f2617a2ce165655968c42560 --- /dev/null +++ b/models/Sepformer/de/sepformer_rescuespeech/README.md @@ -0,0 +1,125 @@ +--- +language: "de" +thumbnail: +tags: +- audio-to-audio +- Speech Enhancement +- RescueSpeech +- SepFormer +- Transformer +- pytorch +- speechbrain +- Search and Rescue +license: "apache-2.0" +datasets: +- RescueSpeech +metrics: +- SI-SNR +- PESQ +- SDR +model-index: +- name: rescuespeech_sepformer + results: + - task: + name: Speech Enhancement + type: speech-enhancement + metrics: + - name: Test PESQ + type: pesq + value: '2.24' + - name: Test SI-SNRi + type: si-snri + value: '7.849' + - name: Test SI-SDRi + type: si-sdri + value: '8.414' +--- + + +

+ +# SepFormer trained on RescueSpeech for speech enhancement (16k sampling frequency) +This repository provides all the necessary tools to perform speech enhancement (denoising) with a [SepFormer](https://arxiv.org/abs/2010.13154v2) model, implemented with SpeechBrain. The model was first trained on Microsoft-DNS 4 dataset and further fine-tuned on RescueSpeech dataset 16k sampling frequency. For a better experience we encourage you to learn more about [SpeechBrain](https://speechbrain.github.io). Given below is model performance on RescueSpeech test set. + + +| Release | Test-Set SI-SNRi | Test-Set SI-SDRi |Test-Set PESQ | +|:-------------:|:--------------:|:--------------:|:--------------:| +| 07-01-23 | 7.849 | 8.414 | 2.24 | + +where SI-SNRi and SI-SDRi indicates the improvement in SI-SNR and SI-SDR metric. + +## Install SpeechBrain + +First of all, please install SpeechBrain with the following command: + +``` +pip install speechbrain +``` + +Please notice that we encourage you to read our tutorials and learn more about [SpeechBrain](https://speechbrain.github.io). + +### Perform speech enhancement on your own audio file + +```python +from speechbrain.inference.separation import SepformerSeparation as separator +import torchaudio + +model = separator.from_hparams(source="speechbrain/rescuespeech_sepformer", savedir='pretrained_models/rescuespeech_sepformer') + +# for custom file, change path +est_sources = model.separate_file(path='speechbrain/rescuespeech_sepformer/example_rescuespeech16k.wav') + +torchaudio.save("enhanced_rescuespeech16k.wav", est_sources[:, :, 0].detach().cpu(), 16000) + +``` + +### Inference on GPU +To perform inference on the GPU, add `run_opts={"device":"cuda"}` when calling the `from_hparams` method. + +You can find our training results (models, logs, etc) [here](https://www.dropbox.com/sh/02c3wesc65402f6/AAApoxBApft-JwqHK-bddedBa?dl=0). + +### Limitations +The SpeechBrain team does not provide any warranty on the performance achieved by this model when used on other datasets. + +#### Referencing SpeechBrain + +```bibtex +@misc{speechbrain, + title={{SpeechBrain}: A General-Purpose Speech Toolkit}, + author={Mirco Ravanelli and Titouan Parcollet and Peter Plantinga and Aku Rouhe and Samuele Cornell and Loren Lugosch and Cem Subakan and Nauman Dawalatabad and Abdelwahab Heba and Jianyuan Zhong and Ju-Chieh Chou and Sung-Lin Yeh and Szu-Wei Fu and Chien-Feng Liao and Elena Rastorgueva and François Grondin and William Aris and Hwidong Na and Yan Gao and Renato De Mori and Yoshua Bengio}, + year={2021}, + eprint={2106.04624}, + archivePrefix={arXiv}, + primaryClass={eess.AS}, + note={arXiv:2106.04624} +} +``` + + +#### Referencing SepFormer +```bibtex +@inproceedings{subakan2021attention, + title={Attention is All You Need in Speech Separation}, + author={Cem Subakan and Mirco Ravanelli and Samuele Cornell and Mirko Bronzi and Jianyuan Zhong}, + year={2021}, + booktitle={ICASSP 2021} +} +``` + +### Referencing RescueSpeech +```bibtex +@misc{sagar2023rescuespeech, + title={RescueSpeech: A German Corpus for Speech Recognition in Search and Rescue Domain}, + author={Sangeet Sagar and Mirco Ravanelli and Bernd Kiefer and Ivana Kruijff Korbayova and Josef van Genabith}, + year={2023}, + eprint={2306.04054}, + archivePrefix={arXiv}, + primaryClass={eess.AS} +} +``` + + +# **About SpeechBrain** +- Website: https://speechbrain.github.io/ +- Code: https://github.com/speechbrain/speechbrain/ +- HuggingFace: https://huggingface.co/speechbrain/ \ No newline at end of file diff --git a/models/Sepformer/de/sepformer_rescuespeech/config.json b/models/Sepformer/de/sepformer_rescuespeech/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c97b7fa67edd2ea949e48af55f4af3221e95795e --- /dev/null +++ b/models/Sepformer/de/sepformer_rescuespeech/config.json @@ -0,0 +1,3 @@ +{ + "speechbrain_interface": "SepformerSeparation" +} diff --git a/models/Sepformer/de/sepformer_rescuespeech/decoder.ckpt b/models/Sepformer/de/sepformer_rescuespeech/decoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f2edea69835b918f61df1a10c13d527a6d7c0bf0 --- /dev/null +++ b/models/Sepformer/de/sepformer_rescuespeech/decoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00d272f965100f627a4a43d45dd919a7caf867372035139a91b8ece174c8b5f1 +size 17195 diff --git a/models/Sepformer/de/sepformer_rescuespeech/encoder.ckpt b/models/Sepformer/de/sepformer_rescuespeech/encoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7f53086f8fa36b32bc6f047d25bce48ba6b9dd8f --- /dev/null +++ b/models/Sepformer/de/sepformer_rescuespeech/encoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a04882b82c623a27af12376cc05dc0eb6cc1d59fb483bfb463d89d7d6072ac4d +size 17259 diff --git a/models/Sepformer/de/sepformer_rescuespeech/example_rescuespeech16k.wav b/models/Sepformer/de/sepformer_rescuespeech/example_rescuespeech16k.wav new file mode 100644 index 0000000000000000000000000000000000000000..1e218d9f7334f85dd31b3e58b36d0fda6932216c --- /dev/null +++ b/models/Sepformer/de/sepformer_rescuespeech/example_rescuespeech16k.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d804a2b9e506804ca9ee108e1cbe5634c8de9394ff2f41233856506ef44e39c +size 445498 diff --git a/models/Sepformer/de/sepformer_rescuespeech/hyperparams.yaml b/models/Sepformer/de/sepformer_rescuespeech/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5ecd622e727d303bdb85ec138a26b7f4cfc284d --- /dev/null +++ b/models/Sepformer/de/sepformer_rescuespeech/hyperparams.yaml @@ -0,0 +1,68 @@ + +# ################################ +# Model: SepFormer for source separation +# https://arxiv.org/abs/2010.13154 +# Dataset : RescueSpeech +# ################################ + +sample_rate: 16000 +num_spks: 1 + +# Encoder parameters +N_encoder_out: 256 +out_channels: 256 +kernel_size: 16 +kernel_stride: 8 + +# Specifying the network +Encoder: !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + +SBtfintra: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + num_spks: 1 + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: !ref + inter_model: !ref + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +modules: + encoder: !ref + decoder: !ref + masknet: !ref + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + encoder: !ref + masknet: !ref + decoder: !ref \ No newline at end of file diff --git a/models/Sepformer/de/sepformer_rescuespeech/masknet.ckpt b/models/Sepformer/de/sepformer_rescuespeech/masknet.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ad6a802f69a740358452fb60d95d1df1b4575fdd --- /dev/null +++ b/models/Sepformer/de/sepformer_rescuespeech/masknet.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e8fa1f883ca0d47cc825a69ca6bf8920f8325c6ac2efcf4e06bfa3c96d648c5 +size 112839555 diff --git a/models/Sepformer/de/sepformer_rescuespeech/source.txt b/models/Sepformer/de/sepformer_rescuespeech/source.txt new file mode 100644 index 0000000000000000000000000000000000000000..386ce73e1544165f7caedfa800f2ae24bcf8917f --- /dev/null +++ b/models/Sepformer/de/sepformer_rescuespeech/source.txt @@ -0,0 +1 @@ +https://huggingface.co/speechbrain/sepformer_rescuespeech \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-libri3mix/.gitattributes b/models/Sepformer/en/sepformer-libri3mix/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..83ad0597a7e3a7a99fdab94a74d16a673f6d8627 --- /dev/null +++ b/models/Sepformer/en/sepformer-libri3mix/.gitattributes @@ -0,0 +1,38 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +optimizer.ckpt filter=lfs diff=lfs merge=lfs -text +brain.ckpt filter=lfs diff=lfs merge=lfs -text +counter.ckpt filter=lfs diff=lfs merge=lfs -text +decoder.ckpt filter=lfs diff=lfs merge=lfs -text +encoder.ckpt filter=lfs diff=lfs merge=lfs -text +masknet.ckpt filter=lfs diff=lfs merge=lfs -text diff --git a/models/Sepformer/en/sepformer-libri3mix/README.md b/models/Sepformer/en/sepformer-libri3mix/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5a116fa90e9935825943d9e3a866710168ada196 --- /dev/null +++ b/models/Sepformer/en/sepformer-libri3mix/README.md @@ -0,0 +1,134 @@ +--- +language: "en" +thumbnail: +tags: +- Source Separation +- Speech Separation +- Audio Source Separation +- Libri3Mix +- SepFormer +- Transformer +- audio-to-audio +- audio-source-separation +- speechbrain +license: "apache-2.0" +datasets: +- Libri3Mix +metrics: +- SI-SNRi +- SDRi + +--- + + +

+ +# SepFormer trained on Libri3Mix + +This repository provides all the necessary tools to perform audio source separation with a [SepFormer](https://arxiv.org/abs/2010.13154v2) +model, implemented with SpeechBrain, and pretrained on Libri3Mix dataset. For a better experience we encourage you to learn more about +[SpeechBrain](https://speechbrain.github.io). The model performance is 19.8 dB SI-SNRi on the test set of Libri3Mix dataset. + +| Release | Test-Set SI-SNRi | Test-Set SDRi | +|:-------------:|:--------------:|:--------------:| +| 16-09-22 | 19.0dB | 19.4dB | + + +## Install SpeechBrain + +First of all, please install SpeechBrain with the following command: + +``` +pip install speechbrain +``` + +Please notice that we encourage you to read our tutorials and learn more about +[SpeechBrain](https://speechbrain.github.io). + +### Perform source separation on your own audio file + +```python +from speechbrain.pretrained import SepformerSeparation as separator +import torchaudio + +model = separator.from_hparams(source="speechbrain/sepformer-libri3mix", savedir='pretrained_models/sepformer-libri3mix') + +est_sources = model.separate_file(path='speechbrain/sepformer-wsj03mix/test_mixture_3spks.wav') + +torchaudio.save("source1hat.wav", est_sources[:, :, 0].detach().cpu(), 8000) +torchaudio.save("source2hat.wav", est_sources[:, :, 1].detach().cpu(), 8000) +torchaudio.save("source3hat.wav", est_sources[:, :, 2].detach().cpu(), 8000) + +``` + +The system expects input recordings sampled at 8kHz (single channel). +If your signal has a different sample rate, resample it (e.g, using torchaudio or sox) before using the interface. + +### Inference on GPU +To perform inference on the GPU, add `run_opts={"device":"cuda"}` when calling the `from_hparams` method. + +### Training +The model was trained with SpeechBrain (fc2eabb7). +To train it from scratch follows these steps: +1. Clone SpeechBrain: +```bash +git clone https://github.com/speechbrain/speechbrain/ +``` +2. Install it: +``` +cd speechbrain +pip install -r requirements.txt +pip install -e . +``` + +3. Run Training: +``` +cd recipes/LibriMix/separation +python train.py hparams/sepformer.yaml --data_folder=your_data_folder +``` +Note: change num_spks to 3 in the yaml file. + + +You can find our training results (models, logs, etc) [here](https://drive.google.com/drive/folders/1DN49LtAs6cq1X0jZ8tRMlh2Pj6AecClz). + +### Limitations +The SpeechBrain team does not provide any warranty on the performance achieved by this model when used on other datasets. + +#### Referencing SpeechBrain + +```bibtex +@misc{speechbrain, + title={{SpeechBrain}: A General-Purpose Speech Toolkit}, + author={Mirco Ravanelli and Titouan Parcollet and Peter Plantinga and Aku Rouhe and Samuele Cornell and Loren Lugosch and Cem Subakan and Nauman Dawalatabad and Abdelwahab Heba and Jianyuan Zhong and Ju-Chieh Chou and Sung-Lin Yeh and Szu-Wei Fu and Chien-Feng Liao and Elena Rastorgueva and François Grondin and William Aris and Hwidong Na and Yan Gao and Renato De Mori and Yoshua Bengio}, + year={2021}, + eprint={2106.04624}, + archivePrefix={arXiv}, + primaryClass={eess.AS}, + note={arXiv:2106.04624} +} +``` + + +#### Referencing SepFormer +```bibtex +@inproceedings{subakan2021attention, + title={Attention is All You Need in Speech Separation}, + author={Cem Subakan and Mirco Ravanelli and Samuele Cornell and Mirko Bronzi and Jianyuan Zhong}, + year={2021}, + booktitle={ICASSP 2021} +} + +@article{subakan2023exploring, + author={Subakan, Cem and Ravanelli, Mirco and Cornell, Samuele and Grondin, François and Bronzi, Mirko}, + journal={IEEE/ACM Transactions on Audio, Speech, and Language Processing}, + title={Exploring Self-Attention Mechanisms for Speech Separation}, + year={2023}, + volume={31}, + pages={2169-2180}, + +``` + +# **About SpeechBrain** +- Website: https://speechbrain.github.io/ +- Code: https://github.com/speechbrain/speechbrain/ +- HuggingFace: https://huggingface.co/speechbrain/ \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-libri3mix/config.json b/models/Sepformer/en/sepformer-libri3mix/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a30d2e75f3931394ecc92ad260d82e7d3419cac3 --- /dev/null +++ b/models/Sepformer/en/sepformer-libri3mix/config.json @@ -0,0 +1,3 @@ +{ + "speechbrain_interface": "SepformerSeparation" +} diff --git a/models/Sepformer/en/sepformer-libri3mix/counter.ckpt b/models/Sepformer/en/sepformer-libri3mix/counter.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fa51190e76c6b3bab41e4dc399a8e65121260e77 --- /dev/null +++ b/models/Sepformer/en/sepformer-libri3mix/counter.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ada92f28b4ceda38562ebf047c6ff05400d4c572352a1142eedfef67d21e662 +size 2 diff --git a/models/Sepformer/en/sepformer-libri3mix/decoder.ckpt b/models/Sepformer/en/sepformer-libri3mix/decoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..bcd2ed100abc034837f47dcd37f2eab77c7d702a --- /dev/null +++ b/models/Sepformer/en/sepformer-libri3mix/decoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9afdda4e4ecb9eefb5d49329193c7a9215ae7443aab206f2fe4eb0f50988ca53 +size 17272 diff --git a/models/Sepformer/en/sepformer-libri3mix/encoder.ckpt b/models/Sepformer/en/sepformer-libri3mix/encoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5ed738750785cd9e12001fac44344fddfcca4336 --- /dev/null +++ b/models/Sepformer/en/sepformer-libri3mix/encoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e949e80c1a96965b723460209bb155eac1fdb2f6d2cf4e7b3e1d3152716f38d0 +size 17272 diff --git a/models/Sepformer/en/sepformer-libri3mix/hyperparams.yaml b/models/Sepformer/en/sepformer-libri3mix/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a534fe774685247d7000508836c2e8425ecd0c0d --- /dev/null +++ b/models/Sepformer/en/sepformer-libri3mix/hyperparams.yaml @@ -0,0 +1,68 @@ +# ################################ +# Model: Pretrained SepFormer for source separation +# Dataset : Libri3Mix +# ################################ + +# Experiment params + +sample_rate: 8000 +num_spks: 3 + +# Encoder parameters +N_encoder_out: 256 +out_channels: 256 +kernel_size: 16 +kernel_stride: 8 + +# Specifying the network +Encoder: !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + +SBtfintra: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + num_spks: 3 + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: !ref + inter_model: !ref + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +modules: + encoder: !ref + decoder: !ref + masknet: !ref + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + encoder: !ref + masknet: !ref + decoder: !ref diff --git a/models/Sepformer/en/sepformer-libri3mix/masknet.ckpt b/models/Sepformer/en/sepformer-libri3mix/masknet.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..766a802e86c07de8e47df9d1bd569835adee4733 --- /dev/null +++ b/models/Sepformer/en/sepformer-libri3mix/masknet.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:416af7b09d912c486261dd824788dd92d3b21373c8ec4d029209792522f4dbcb +size 113375814 diff --git a/models/Sepformer/en/sepformer-libri3mix/optimizer.ckpt b/models/Sepformer/en/sepformer-libri3mix/optimizer.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c55514191270f3d47205e67c442ce570f22723dd --- /dev/null +++ b/models/Sepformer/en/sepformer-libri3mix/optimizer.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2523adf0429ccfcca4ce95e87a5e62cbdac5a2240c11da40326272d8c4482bac +size 206221049 diff --git a/models/Sepformer/en/sepformer-libri3mix/source.txt b/models/Sepformer/en/sepformer-libri3mix/source.txt new file mode 100644 index 0000000000000000000000000000000000000000..55e96b82fd3dba9ea6038cbcfc9cc289fa1df60d --- /dev/null +++ b/models/Sepformer/en/sepformer-libri3mix/source.txt @@ -0,0 +1 @@ +https://huggingface.co/speechbrain/sepformer-libri3mix \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-wham/.gitattributes b/models/Sepformer/en/sepformer-wham/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..7d3ed876292ce8d883c5257aefd3917e15024946 --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/.gitattributes @@ -0,0 +1 @@ +*.ckpt filter=lfs diff=lfs merge=lfs -text diff --git a/models/Sepformer/en/sepformer-wham/CKPT.yaml b/models/Sepformer/en/sepformer-wham/CKPT.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6de3d73ecfaf931aa280bf5286ba35690beb0f2 --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/CKPT.yaml @@ -0,0 +1,4 @@ +# yamllint disable +end-of-epoch: true +si-snr: -12.183298104639045 +unixtime: 1615254590.2154887 diff --git a/models/Sepformer/en/sepformer-wham/README.md b/models/Sepformer/en/sepformer-wham/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ad8de94441d351a4267c4654a1bdeae0dbcb2f42 --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/README.md @@ -0,0 +1,116 @@ +--- +language: "en" +thumbnail: +tags: +- audio-to-audio +- audio-source-separation +- Source Separation +- Speech Separation +- Audio Source Separation +- WHAM! +- SepFormer +- Transformer +- speechbrain +license: "apache-2.0" +metrics: +- SI-SNRi +- SDRi + +--- + + +

+ +# SepFormer trained on WHAM! +This repository provides all the necessary tools to perform audio source separation with a [SepFormer](https://arxiv.org/abs/2010.13154v2) model, implemented with SpeechBrain, and pretrained on [WHAM!](http://wham.whisper.ai/) dataset, which is basically a version of WSJ0-Mix dataset with environmental noise. For a better experience we encourage you to learn more about [SpeechBrain](https://speechbrain.github.io). The model performance is 16.3 dB SI-SNRi on the test set of WHAM! dataset. + +| Release | Test-Set SI-SNRi | Test-Set SDRi | +|:-------------:|:--------------:|:--------------:| +| 09-03-21 | 16.3 dB | 16.7 dB | + + +## Install SpeechBrain + +First of all, please install SpeechBrain with the following command: + +``` +pip install speechbrain +``` + +Please notice that we encourage you to read our tutorials and learn more about [SpeechBrain](https://speechbrain.github.io). + +### Perform source separation on your own audio file + +```python +from speechbrain.inference.separation import SepformerSeparation as separator +import torchaudio + +model = separator.from_hparams(source="speechbrain/sepformer-wham", savedir='pretrained_models/sepformer-wham') + +# for custom file, change path +est_sources = model.separate_file(path='speechbrain/sepformer-wsj02mix/test_mixture.wav') + +torchaudio.save("source1hat.wav", est_sources[:, :, 0].detach().cpu(), 8000) +torchaudio.save("source2hat.wav", est_sources[:, :, 1].detach().cpu(), 8000) +``` +The system expects input recordings sampled at 8kHz (single channel). +If your signal has a different sample rate, resample it (e.g, using torchaudio or sox) before using the interface. + +### Inference on GPU +To perform inference on the GPU, add `run_opts={"device":"cuda"}` when calling the `from_hparams` method. + +### Training +The model was trained with SpeechBrain (e375cd13). +To train it from scratch follows these steps: +1. Clone SpeechBrain: +```bash +git clone https://github.com/speechbrain/speechbrain/ +``` +2. Install it: +``` +cd speechbrain +pip install -r requirements.txt +pip install -e . +``` + +3. Run Training: +``` +cd recipes/WHAMandWHAMR/separation +python train.py hparams/sepformer-wham.yaml --data_folder=your_data_folder +``` + +You can find our training results (models, logs, etc) [here](https://drive.google.com/drive/folders/1dIAT8hZxvdJPZNUb8Zkk3BuN7GZ9-mZb?usp=sharing). + + +### Limitations +The SpeechBrain team does not provide any warranty on the performance achieved by this model when used on other datasets. + +#### Referencing SpeechBrain + + +```bibtex +@misc{speechbrain, + title={{SpeechBrain}: A General-Purpose Speech Toolkit}, + author={Mirco Ravanelli and Titouan Parcollet and Peter Plantinga and Aku Rouhe and Samuele Cornell and Loren Lugosch and Cem Subakan and Nauman Dawalatabad and Abdelwahab Heba and Jianyuan Zhong and Ju-Chieh Chou and Sung-Lin Yeh and Szu-Wei Fu and Chien-Feng Liao and Elena Rastorgueva and François Grondin and William Aris and Hwidong Na and Yan Gao and Renato De Mori and Yoshua Bengio}, + year={2021}, + eprint={2106.04624}, + archivePrefix={arXiv}, + primaryClass={eess.AS}, + note={arXiv:2106.04624} +} +``` + +#### Referencing SepFormer +```bibtex +@inproceedings{subakan2021attention, + title={Attention is All You Need in Speech Separation}, + author={Cem Subakan and Mirco Ravanelli and Samuele Cornell and Mirko Bronzi and Jianyuan Zhong}, + year={2021}, + booktitle={ICASSP 2021} +} +``` + +# **About SpeechBrain** +- Website: https://speechbrain.github.io/ +- Code: https://github.com/speechbrain/speechbrain/ +- HuggingFace: https://huggingface.co/speechbrain/ \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-wham/config.json b/models/Sepformer/en/sepformer-wham/config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b1f71cf150b6f3ddc7fe135ac86a6e8eb946a1 --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/config.json @@ -0,0 +1,3 @@ +{ + "speechbrain_interface": "SepformerSeparation" +} \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-wham/counter.ckpt b/models/Sepformer/en/sepformer-wham/counter.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cfb993e56564c3b04e908f5e6b81c82a6971f040 --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/counter.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85daaf6f7055cd5736287faed9603d712920092c4f8fd0097ec3b650bf27530e +size 3 diff --git a/models/Sepformer/en/sepformer-wham/dataloader-TRAIN.ckpt b/models/Sepformer/en/sepformer-wham/dataloader-TRAIN.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b5a49049e5ab6ce5335145f010e83e445d0d95c8 --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/dataloader-TRAIN.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:876c9b16254e157d1eb645390dcfae6f29b9d3cd394e73a91de8ee5d0e67ee43 +size 5 diff --git a/models/Sepformer/en/sepformer-wham/decoder.ckpt b/models/Sepformer/en/sepformer-wham/decoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1ca83cc8eea4eefeed1d72d3c9f787d5c29bf66b --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/decoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9a06998d188f9fa0cb79915dc855aac4737c41392488b11d500b453000f4dfa +size 17272 diff --git a/models/Sepformer/en/sepformer-wham/encoder.ckpt b/models/Sepformer/en/sepformer-wham/encoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c62ba11161366153573ccf158fd71167cb2025b0 --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/encoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a33283041ba9a4cc292bc31ea14b1bfccad8304be822480571eaf95f9ccb0301 +size 17272 diff --git a/models/Sepformer/en/sepformer-wham/hyperparams.yaml b/models/Sepformer/en/sepformer-wham/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bae651275815fe835aded63ae4d9f98c0682e3c5 --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/hyperparams.yaml @@ -0,0 +1,64 @@ +# ################################ +# Model: Inference for source separation with SepFormer +# https://arxiv.org/abs/2010.13154 +# Generated from speechbrain/recipes/WSJ0Mix/separation/train/hparams/sepformer-wham.yaml +# Dataset : Wham +# ################################ + + +# Parameters +sample_rate: 8000 +num_spks: 2 + +# Specifying the network +Encoder: !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + +SBtfintra: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + num_spks: !ref + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: !ref + inter_model: !ref + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +modules: + encoder: !ref + decoder: !ref + masknet: !ref + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + masknet: !ref + encoder: !ref + decoder: !ref diff --git a/models/Sepformer/en/sepformer-wham/hyperparams_train.yaml b/models/Sepformer/en/sepformer-wham/hyperparams_train.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ee780d437b66b5a91431c3961c5b8889fb89eb6 --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/hyperparams_train.yaml @@ -0,0 +1,177 @@ +# Generated 2021-03-09 from: +# /home/mila/s/subakany/speechbrain_new/recipes/WSJ0Mix/separation/train/hparams/sepformer-wham.yaml +# yamllint disable +# ################################ +# Model: SepFormer for source separation +# https://arxiv.org/abs/2010.13154 +# +# Dataset : WSJ0-2mix and WSJ0-3mix +# ################################ +# Basic parameters +# Seed needs to be set at top of yaml, before objects with parameters are made +# +seed: 1234 +__set_seed: !apply:torch.manual_seed [1234] + +# Data params + +# the data folder for the wham dataset +# needs to end with wham_original for the wham dataset +# needs to end with wham_reverb for the whamr dataset +data_folder: /network/tmp1/subakany/wham_original + +# the path for wsj0/si_tr_s/ folder -- only needed if dynamic mixing is used +# e.g. /yourpath/wsj0-processed/si_tr_s/ +wsj0_tr: /yourpath/wsj0-processed/si_tr_s/ + +experiment_name: sepformer-wham +output_folder: results/sepformer-wham/1234 +train_log: results/sepformer-wham/1234/train_log.txt +save_folder: results/sepformer-wham/1234/save + +# the file names should start with whamr instead of whamorg +train_data: results/sepformer-wham/1234/save/whamorg_tr.csv +valid_data: results/sepformer-wham/1234/save/whamorg_cv.csv +test_data: results/sepformer-wham/1234/save/whamorg_tt.csv +skip_prep: false + + +# Experiment params +auto_mix_prec: false # Set it to True for mixed precision +test_only: false +num_spks: 2 # set to 3 for wsj0-3mix +progressbar: true +save_audio: false # Save estimated sources on disk +sample_rate: 8000 + +# Training parameters +N_epochs: 200 +batch_size: 1 +lr: 0.00015 +clip_grad_norm: 5 +loss_upper_lim: 999999 # this is the upper limit for an acceptable loss +# if True, the training sequences are cut to a specified length +limit_training_signal_len: false +# this is the length of sequences if we choose to limit +# the signal length of training sequences +training_signal_len: 32000000 + +# Set it to True to dynamically create mixtures at training time +dynamic_mixing: false + +# Parameters for data augmentation +use_wavedrop: false +use_speedperturb: true +use_speedperturb_sameforeachsource: false +use_rand_shift: false +min_shift: -8000 +max_shift: 8000 + +speedperturb: !new:speechbrain.lobes.augment.TimeDomainSpecAugment + perturb_prob: 1.0 + drop_freq_prob: 0.0 + drop_chunk_prob: 0.0 + sample_rate: 8000 + speeds: [95, 100, 105] + +wavedrop: !new:speechbrain.lobes.augment.TimeDomainSpecAugment + perturb_prob: 0.0 + drop_freq_prob: 1.0 + drop_chunk_prob: 1.0 + sample_rate: 8000 + +# loss thresholding -- this thresholds the training loss +threshold_byloss: true +threshold: -30 + +# Encoder parameters +N_encoder_out: 256 +out_channels: 256 +kernel_size: 16 +kernel_stride: 8 + +# Dataloader options +dataloader_opts: + batch_size: 1 + num_workers: 3 + + +# Specifying the network +Encoder: &id003 !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + + +SBtfintra: &id001 !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: &id002 !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: &id005 !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + + num_spks: 2 + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: *id001 + inter_model: *id002 + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: &id004 !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +optimizer: !name:torch.optim.Adam + lr: 0.00015 + weight_decay: 0 + +loss: !name:speechbrain.nnet.losses.get_si_snr_with_pitwrapper + +lr_scheduler: &id007 !new:speechbrain.nnet.schedulers.ReduceLROnPlateau + + factor: 0.5 + patience: 2 + dont_halve_until_epoch: 65 + +epoch_counter: &id006 !new:speechbrain.utils.epoch_loop.EpochCounter + limit: 200 + +modules: + encoder: *id003 + decoder: *id004 + masknet: *id005 +checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer + checkpoints_dir: results/sepformer-wham/1234/save + recoverables: + encoder: *id003 + decoder: *id004 + masknet: *id005 + counter: *id006 + lr_scheduler: *id007 +train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger + save_file: results/sepformer-wham/1234/train_log.txt + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + masknet: !ref + encoder: !ref + decoder: !ref diff --git a/models/Sepformer/en/sepformer-wham/lr_scheduler.ckpt b/models/Sepformer/en/sepformer-wham/lr_scheduler.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d129c5ad48e82c08c06e06737d5fbf0625fa8aae --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/lr_scheduler.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b16ecb0ad4a0d1cdfa31c45f79d74cbc35e283bf4a1bfdccac64bff1ab04cb6c +size 1519 diff --git a/models/Sepformer/en/sepformer-wham/masknet.ckpt b/models/Sepformer/en/sepformer-wham/masknet.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3e18aa61e99dd03101c5c45ff54f750adaabe64e --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/masknet.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f058bffbc94d3ac3fb2481239b9eda589cb53484d312714180c98ae5fee8bbf3 +size 113112646 diff --git a/models/Sepformer/en/sepformer-wham/optimizer.ckpt b/models/Sepformer/en/sepformer-wham/optimizer.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a2dfab00c4ba3b58574b40c1afc7e744a1978ba4 --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/optimizer.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4b60e4ed1f7cd399fe23556d4dfa5fe3e1f09904f04968bb84cad2936ce063d +size 205694713 diff --git a/models/Sepformer/en/sepformer-wham/source.txt b/models/Sepformer/en/sepformer-wham/source.txt new file mode 100644 index 0000000000000000000000000000000000000000..481c0e608f4ab9244a0d6b42c18033279299185b --- /dev/null +++ b/models/Sepformer/en/sepformer-wham/source.txt @@ -0,0 +1 @@ +https://huggingface.co/speechbrain/sepformer-wham \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/.gitattributes b/models/Sepformer/en/sepformer-whamr-enhancement/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..26664b32660bc53593dc81f98744c6b9d0494435 --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/.gitattributes @@ -0,0 +1,34 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +brain.ckpt filter=lfs diff=lfs merge=lfs -text +counter.ckpt filter=lfs diff=lfs merge=lfs -text +decoder.ckpt filter=lfs diff=lfs merge=lfs -text +encoder.ckpt filter=lfs diff=lfs merge=lfs -text +lr_scheduler.ckpt filter=lfs diff=lfs merge=lfs -text +masknet.ckpt filter=lfs diff=lfs merge=lfs -text +optimizer.ckpt filter=lfs diff=lfs merge=lfs -text diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/CKPT.yaml b/models/Sepformer/en/sepformer-whamr-enhancement/CKPT.yaml new file mode 100644 index 0000000000000000000000000000000000000000..181846d047a58089a383a987f21045d29cc7186c --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/CKPT.yaml @@ -0,0 +1,4 @@ +# yamllint disable +end-of-epoch: true +si-snr: -11.609822317790979 +unixtime: 1636546512.8062298 diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/README.md b/models/Sepformer/en/sepformer-whamr-enhancement/README.md new file mode 100644 index 0000000000000000000000000000000000000000..387cc01e962453bb89c9edece1ce4322ac3af01e --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/README.md @@ -0,0 +1,97 @@ +--- +language: "en" +thumbnail: +tags: +- audio-to-audio +- Speech Enhancement +- WHAMR! +- SepFormer +- Transformer +- pytorch +- speechbrain +license: "apache-2.0" +metrics: +- SI-SNR +- PESQ + +--- + + +

+ +# SepFormer trained on WHAMR! for speech enhancement (8k sampling frequency) +This repository provides all the necessary tools to perform speech enhancement (denoising + dereverberation) with a [SepFormer](https://arxiv.org/abs/2010.13154v2) model, implemented with SpeechBrain, and pretrained on [WHAMR!](http://wham.whisper.ai/) dataset with 8k sampling frequency, which is basically a version of WSJ0-Mix dataset with environmental noise and reverberation in 8k. For a better experience we encourage you to learn more about [SpeechBrain](https://speechbrain.github.io). The given model performance is 10.59 dB SI-SNR on the test set of WHAMR! dataset. + + +| Release | Test-Set SI-SNR | Test-Set PESQ | +|:-------------:|:--------------:|:--------------:| +| 01-12-21 | 10.59 | 2.84 | + + +## Install SpeechBrain + +First of all, please install SpeechBrain with the following command: + +``` +pip install speechbrain +``` + +Please notice that we encourage you to read our tutorials and learn more about [SpeechBrain](https://speechbrain.github.io). + +### Perform speech enhancement on your own audio file + +```python +from speechbrain.inference.separation import SepformerSeparation as separator +import torchaudio + +model = separator.from_hparams(source="speechbrain/sepformer-whamr-enhancement", savedir='pretrained_models/sepformer-whamr-enhancement') + +# for custom file, change path +est_sources = model.separate_file(path='speechbrain/sepformer-whamr-enhancement/example_whamr.wav') + +torchaudio.save("enhanced_whamr.wav", est_sources[:, :, 0].detach().cpu(), 8000) + +``` + +### Inference on GPU +To perform inference on the GPU, add `run_opts={"device":"cuda"}` when calling the `from_hparams` method. + +### Training +The training script is currently being worked on an ongoing pull-request. + +We will update the model card as soon as the PR is merged. + +You can find our training results (models, logs, etc) [here](https://drive.google.com/drive/folders/1V0KwkEfWwomZ0Vjox0BTnQ694_uxgu8G). + +### Limitations +The SpeechBrain team does not provide any warranty on the performance achieved by this model when used on other datasets. + +#### Referencing SpeechBrain + +```bibtex +@misc{speechbrain, + title={{SpeechBrain}: A General-Purpose Speech Toolkit}, + author={Mirco Ravanelli and Titouan Parcollet and Peter Plantinga and Aku Rouhe and Samuele Cornell and Loren Lugosch and Cem Subakan and Nauman Dawalatabad and Abdelwahab Heba and Jianyuan Zhong and Ju-Chieh Chou and Sung-Lin Yeh and Szu-Wei Fu and Chien-Feng Liao and Elena Rastorgueva and François Grondin and William Aris and Hwidong Na and Yan Gao and Renato De Mori and Yoshua Bengio}, + year={2021}, + eprint={2106.04624}, + archivePrefix={arXiv}, + primaryClass={eess.AS}, + note={arXiv:2106.04624} +} +``` + + +#### Referencing SepFormer +```bibtex +@inproceedings{subakan2021attention, + title={Attention is All You Need in Speech Separation}, + author={Cem Subakan and Mirco Ravanelli and Samuele Cornell and Mirko Bronzi and Jianyuan Zhong}, + year={2021}, + booktitle={ICASSP 2021} +} +``` + +# **About SpeechBrain** +- Website: https://speechbrain.github.io/ +- Code: https://github.com/speechbrain/speechbrain/ +- HuggingFace: https://huggingface.co/speechbrain/ \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/config.json b/models/Sepformer/en/sepformer-whamr-enhancement/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4fc9c7ea07ea0ecbff148564d211a7b17411f322 --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/config.json @@ -0,0 +1,5 @@ +{ + + "speechbrain_interface": "SepformerSeparation" + +} \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/counter.ckpt b/models/Sepformer/en/sepformer-whamr-enhancement/counter.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f86c899b526883e1f6be104216eab77284e45398 --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/counter.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbae772db29058a88f9bd830e957c695347c41b6162a7eb9a9ea13def34be56b +size 3 diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/decoder.ckpt b/models/Sepformer/en/sepformer-whamr-enhancement/decoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7854912515805ad18ac9740b1c82db86d43f007d --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/decoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91794bd8c7aba5b7ca503574bd9f4997869e253021142958658c8e1e977b5485 +size 17272 diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/encoder.ckpt b/models/Sepformer/en/sepformer-whamr-enhancement/encoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..58afb473cb69f4831f2251d891e849d7e88dc0cc --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/encoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f19c8a4d767bef3091c7fd5c55064c4cc6e570b794b501b7c6ff95ee3ea3cef +size 17272 diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/example_whamr.wav b/models/Sepformer/en/sepformer-whamr-enhancement/example_whamr.wav new file mode 100644 index 0000000000000000000000000000000000000000..cc3e8f56aa32eb7d1679790bce9c92f4c7616002 --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/example_whamr.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec0f4e7ceabcd9e54ed22cbf83a8fefaff689ad3f88f466f2ab600edba44feb8 +size 163766 diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/hyperparams.yaml b/models/Sepformer/en/sepformer-whamr-enhancement/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1bc01bafc5caed540d9076dcd7bd4ffd5d484cb --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/hyperparams.yaml @@ -0,0 +1,67 @@ + +# ################################ +# Model: SepFormer for source separation +# Dataset : WHAMR! +# ################################ + +sample_rate: 8000 +num_spks: 1 + +# Encoder parameters +N_encoder_out: 256 +out_channels: 256 +kernel_size: 16 +kernel_stride: 8 + +# Specifying the network +Encoder: !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + +SBtfintra: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + num_spks: 1 + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: !ref + inter_model: !ref + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +modules: + encoder: !ref + decoder: !ref + masknet: !ref + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + encoder: !ref + masknet: !ref + decoder: !ref \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/lr_scheduler.ckpt b/models/Sepformer/en/sepformer-whamr-enhancement/lr_scheduler.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f36882e511ec16105e7cac2ffc945bc26c4e61e6 --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/lr_scheduler.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf1b43deb7479588149147d22a956e1d20fbe00ac1aaaa0774e2a2dba0b7fdd0 +size 1711 diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/masknet.ckpt b/models/Sepformer/en/sepformer-whamr-enhancement/masknet.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c1aa097595681e3b69891a05e276351702c8024f --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/masknet.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26e6b5e91dd8fd7eb6c7583e87b51c000f142598a749402058d9ec5bc3c3683f +size 112849478 diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/optimizer.ckpt b/models/Sepformer/en/sepformer-whamr-enhancement/optimizer.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3fe6b588c8b059273a023409d3222753e9798704 --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/optimizer.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4f4ce52a298a9567f0951c0020c8d95a7f7cd3f0f4ef9a2929f02f22bec0d4d +size 205168377 diff --git a/models/Sepformer/en/sepformer-whamr-enhancement/source.txt b/models/Sepformer/en/sepformer-whamr-enhancement/source.txt new file mode 100644 index 0000000000000000000000000000000000000000..a68ca45458282d25792598f5799eb4f1a1cbb500 --- /dev/null +++ b/models/Sepformer/en/sepformer-whamr-enhancement/source.txt @@ -0,0 +1 @@ +https://huggingface.co/speechbrain/sepformer-whamr-enhancement \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-wsj02mix/.gitattributes b/models/Sepformer/en/sepformer-wsj02mix/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..366cfd059ad60558d91ab62d686c9a5500f78589 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj02mix/.gitattributes @@ -0,0 +1,23 @@ +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tar.gz filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +masknet.ckpt filter=lfs diff=lfs merge=lfs -text +optimizer.ckpt filter=lfs diff=lfs merge=lfs -text +brain.ckpt filter=lfs diff=lfs merge=lfs -text +counter.ckpt filter=lfs diff=lfs merge=lfs -text +decoder.ckpt filter=lfs diff=lfs merge=lfs -text +encoder.ckpt filter=lfs diff=lfs merge=lfs -text +lr_scheduler.ckpt filter=lfs diff=lfs merge=lfs -text diff --git a/models/Sepformer/en/sepformer-wsj02mix/README.md b/models/Sepformer/en/sepformer-wsj02mix/README.md new file mode 100644 index 0000000000000000000000000000000000000000..437fca933723bb3e180011190f3438613db38d02 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj02mix/README.md @@ -0,0 +1,123 @@ +--- +language: "en" +thumbnail: +tags: +- Source Separation +- Speech Separation +- Audio Source Separation +- WSJ02Mix +- SepFormer +- Transformer +- audio-to-audio +- audio-source-separation +- speechbrain +license: "apache-2.0" +datasets: +- WSJ0-2Mix +metrics: +- SI-SNRi +- SDRi + +--- + + +

+ +# SepFormer trained on WSJ0-2Mix + +This repository provides all the necessary tools to perform audio source separation with a [SepFormer](https://arxiv.org/abs/2010.13154v2) +model, implemented with SpeechBrain, and pretrained on WSJ0-2Mix dataset. For a better experience we encourage you to learn more about +[SpeechBrain](https://speechbrain.github.io). The model performance is 22.4 dB on the test set of WSJ0-2Mix dataset. + +| Release | Test-Set SI-SNRi | Test-Set SDRi | +|:-------------:|:--------------:|:--------------:| +| 09-03-21 | 22.4dB | 22.6dB | + +You can listen to example results obtained on the test set of WSJ0-2/3Mix through [here](https://sourceseparationresearch.com/static/sepformer_example_results/sepformer_results.html). + + +## Install SpeechBrain + +First of all, please install SpeechBrain with the following command: + +``` +pip install speechbrain +``` + +Please notice that we encourage you to read our tutorials and learn more about +[SpeechBrain](https://speechbrain.github.io). + +### Perform source separation on your own audio file +```python +from speechbrain.inference.separation import SepformerSeparation as separator +import torchaudio + +model = separator.from_hparams(source="speechbrain/sepformer-wsj02mix", savedir='pretrained_models/sepformer-wsj02mix') + +# for custom file, change path +est_sources = model.separate_file(path='speechbrain/sepformer-wsj02mix/test_mixture.wav') + +torchaudio.save("source1hat.wav", est_sources[:, :, 0].detach().cpu(), 8000) +torchaudio.save("source2hat.wav", est_sources[:, :, 1].detach().cpu(), 8000) +``` + +The system expects input recordings sampled at 8kHz (single channel). +If your signal has a different sample rate, resample it (e.g, using torchaudio or sox) before using the interface. + +### Inference on GPU +To perform inference on the GPU, add `run_opts={"device":"cuda"}` when calling the `from_hparams` method. + +### Training +The model was trained with SpeechBrain (fc2eabb7). +To train it from scratch follows these steps: +1. Clone SpeechBrain: +```bash +git clone https://github.com/speechbrain/speechbrain/ +``` +2. Install it: +``` +cd speechbrain +pip install -r requirements.txt +pip install -e . +``` + +3. Run Training: +``` +cd recipes/WSJ0Mix/separation +python train.py hparams/sepformer.yaml --data_folder=your_data_folder +``` + +You can find our training results (models, logs, etc) [here](https://drive.google.com/drive/folders/1cON-eqtKv_NYnJhaE9VjLT_e2ybn-O7u?usp=sharing). + +### Limitations +The SpeechBrain team does not provide any warranty on the performance achieved by this model when used on other datasets. + +#### Referencing SpeechBrain + +```bibtex +@misc{speechbrain, + title={{SpeechBrain}: A General-Purpose Speech Toolkit}, + author={Mirco Ravanelli and Titouan Parcollet and Peter Plantinga and Aku Rouhe and Samuele Cornell and Loren Lugosch and Cem Subakan and Nauman Dawalatabad and Abdelwahab Heba and Jianyuan Zhong and Ju-Chieh Chou and Sung-Lin Yeh and Szu-Wei Fu and Chien-Feng Liao and Elena Rastorgueva and François Grondin and William Aris and Hwidong Na and Yan Gao and Renato De Mori and Yoshua Bengio}, + year={2021}, + eprint={2106.04624}, + archivePrefix={arXiv}, + primaryClass={eess.AS}, + note={arXiv:2106.04624} +} +``` + + +#### Referencing SepFormer +```bibtex +@inproceedings{subakan2021attention, + title={Attention is All You Need in Speech Separation}, + author={Cem Subakan and Mirco Ravanelli and Samuele Cornell and Mirko Bronzi and Jianyuan Zhong}, + year={2021}, + booktitle={ICASSP 2021} +} +``` + +# **About SpeechBrain** +- Website: https://speechbrain.github.io/ +- Code: https://github.com/speechbrain/speechbrain/ +- HuggingFace: https://huggingface.co/speechbrain/ \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-wsj02mix/config.json b/models/Sepformer/en/sepformer-wsj02mix/config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b1f71cf150b6f3ddc7fe135ac86a6e8eb946a1 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj02mix/config.json @@ -0,0 +1,3 @@ +{ + "speechbrain_interface": "SepformerSeparation" +} \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-wsj02mix/decoder.ckpt b/models/Sepformer/en/sepformer-wsj02mix/decoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..04b4bb3a75f13495745ba70adc4ce05c51e15b07 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj02mix/decoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abea1a2d41151331b4c36071d1b3205aed940a189721f008b12a703e9c63e7e4 +size 17202 diff --git a/models/Sepformer/en/sepformer-wsj02mix/encoder.ckpt b/models/Sepformer/en/sepformer-wsj02mix/encoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cd9ad5c3beee416083210e8999ab21fa07c334aa --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj02mix/encoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3139bb880b29ea77ae8a168b8f2ad6e8eb5c2c0904289676c223d0e93cd2a35d +size 17267 diff --git a/models/Sepformer/en/sepformer-wsj02mix/hyperparams.yaml b/models/Sepformer/en/sepformer-wsj02mix/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..167b5c2253db3e725bdecd071bc777da74fb606d --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj02mix/hyperparams.yaml @@ -0,0 +1,65 @@ +# ################################ +# Model: Inference for source separation with SepFormer +# https://arxiv.org/abs/2010.13154 +# Generated from speechbrain/recipes/WSJ0Mix/separation/train/hparams/sepformer-wsj02mix.yaml +# Dataset : wsj02mix +# ############################### + +# Parameters +sample_rate: 8000 +num_spks: 2 + +# Specifying the network +Encoder: !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + +SBtfintra: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + num_spks: !ref + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: !ref + inter_model: !ref + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +modules: + encoder: !ref + decoder: !ref + masknet: !ref + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + masknet: !ref + encoder: !ref + decoder: !ref + + diff --git a/models/Sepformer/en/sepformer-wsj02mix/hyperparams_train.yaml b/models/Sepformer/en/sepformer-wsj02mix/hyperparams_train.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48d71f3a1fb17a037fd704163376a1dd6fd93a31 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj02mix/hyperparams_train.yaml @@ -0,0 +1,163 @@ +# Generated 2021-02-26 from: +# /scratch/csubakan/speechbrain_new/recipes/WSJ2Mix/separation/yamls/dptransformer78.yaml +# yamllint disable +# ################################ +# Model: SepFormer for source separation +# https://arxiv.org/abs/2010.13154 +# +# Dataset : WSJ0-mix +# ################################ +# Basic parameters +# Seed needs to be set at top of yaml, before objects with parameters are made +# +seed: 1234 +__set_seed: !apply:torch.manual_seed [1234] + +# Data params +data_folder: /localscratch/csubakan.62709298.0/wsj0-mix/2speakers # wsj2mix or wsj3mix +experiment_name: 78-speedchange-dynamicmix-hardcodegaussian +output_folder: results/78-speedchange-dynamicmix-hardcodegaussian/1234 +train_log: results/78-speedchange-dynamicmix-hardcodegaussian/1234/train_log.txt +save_folder: results/78-speedchange-dynamicmix-hardcodegaussian/1234/save +train_data: results/78-speedchange-dynamicmix-hardcodegaussian/1234/save/wsj_tr.csv +valid_data: results/78-speedchange-dynamicmix-hardcodegaussian/1234/save/wsj_cv.csv +test_data: results/78-speedchange-dynamicmix-hardcodegaussian/1234/save/wsj_tt.csv +wsj0_tr: /localscratch/csubakan.62709298.0/wsj0-processed/si_tr_s/ + +# Experiment params +auto_mix_prec: true +test_only: false +num_spks: 2 # set to 3 for wsj0-3mix +progressbar: true +save_audio: false # Save estimated sources on disk +sample_rate: 8000 + +# Training parameters +N_epochs: 200 +batch_size: 1 +lr: 0.00015 +clip_grad_norm: 5 +loss_upper_lim: 999999 # this is the upper limit for an acceptable loss +# if True, the training sequences are cut to a specified length +limit_training_signal_len: false +# this is the length of sequences if we choose to limit +# the signal length of training sequences +training_signal_len: 128000 +dynamic_mixing: regular + +# Augment parameters +use_wavedrop: false +use_speedperturb: true +use_speedperturb_sameforeachsource: false +use_rand_shift: false +min_shift: -8000 +max_shift: 8000 + +# Neural parameters +N_encoder_out: 256 +out_channels: 256 +kernel_size: 16 +kernel_stride: 8 + +threshold_byloss: true +threshold: -30 + +# Dataloader options +dataloader_opts: + batch_size: 1 + num_workers: 3 + +speedperturb: !new:speechbrain.lobes.augment.TimeDomainSpecAugment + perturb_prob: 1.0 + drop_freq_prob: 0.0 + drop_chunk_prob: 0.0 + sample_rate: 8000 + speeds: [95, 100, 105] + +wavedrop: !new:speechbrain.lobes.augment.TimeDomainSpecAugment + perturb_prob: 0.0 + drop_freq_prob: 1.0 + drop_chunk_prob: 1.0 + sample_rate: 8000 + + +Encoder: &id003 !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + + +SBtfintra: &id001 !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: &id002 !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: &id005 !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + + num_spks: 2 + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: *id001 + inter_model: *id002 + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: &id004 !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +optimizer: !name:torch.optim.Adam + lr: 0.00015 + weight_decay: 0 + +loss: !name:speechbrain.nnet.losses.get_si_snr_with_pitwrapper + +lr_scheduler: &id007 !new:speechbrain.nnet.schedulers.ReduceLROnPlateau + + factor: 0.5 + patience: 4 + dont_halve_until_epoch: 100 + +epoch_counter: &id006 !new:speechbrain.utils.epoch_loop.EpochCounter + limit: 200 + +modules: + encoder: *id003 + decoder: *id004 + masknet: *id005 +checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer + checkpoints_dir: results/78-speedchange-dynamicmix-hardcodegaussian/1234/save + recoverables: + encoder: *id003 + decoder: *id004 + masknet: *id005 + counter: *id006 + lr_scheduler: *id007 +train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger + save_file: results/78-speedchange-dynamicmix-hardcodegaussian/1234/train_log.txt + + + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + masknet: !ref + encoder: !ref + decoder: !ref diff --git a/models/Sepformer/en/sepformer-wsj02mix/masknet.ckpt b/models/Sepformer/en/sepformer-wsj02mix/masknet.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..de65e949f771861769d82b0f193ea54321cf881c --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj02mix/masknet.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57dd5f49bf21c5a2101bb4e46648d05d34d517a59e26f0b06646d0bebe8214c7 +size 113108458 diff --git a/models/Sepformer/en/sepformer-wsj02mix/source.txt b/models/Sepformer/en/sepformer-wsj02mix/source.txt new file mode 100644 index 0000000000000000000000000000000000000000..47ab6783a7b7d730633119c28544f446884f4f52 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj02mix/source.txt @@ -0,0 +1 @@ +https://huggingface.co/speechbrain/sepformer-wsj02mix \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-wsj02mix/test_mixture.wav b/models/Sepformer/en/sepformer-wsj02mix/test_mixture.wav new file mode 100644 index 0000000000000000000000000000000000000000..d54c44b89dafc417be98797ea26a595fae4bce6d Binary files /dev/null and b/models/Sepformer/en/sepformer-wsj02mix/test_mixture.wav differ diff --git a/models/Sepformer/en/sepformer-wsj03mix/.gitattributes b/models/Sepformer/en/sepformer-wsj03mix/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..640e96128b54965e24df088e1d772c5f8b8adf77 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/.gitattributes @@ -0,0 +1,23 @@ +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tar.gz filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +encoder.ckpt filter=lfs diff=lfs merge=lfs -text +lr_scheduler.ckpt filter=lfs diff=lfs merge=lfs -text +masknet.ckpt filter=lfs diff=lfs merge=lfs -text +optimizer.ckpt filter=lfs diff=lfs merge=lfs -text +brain.ckpt filter=lfs diff=lfs merge=lfs -text +counter.ckpt filter=lfs diff=lfs merge=lfs -text +decoder.ckpt filter=lfs diff=lfs merge=lfs -text diff --git a/models/Sepformer/en/sepformer-wsj03mix/CKPT.yaml b/models/Sepformer/en/sepformer-wsj03mix/CKPT.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65c33c7f2f54b0300944ebac1e356c06337cece8 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/CKPT.yaml @@ -0,0 +1,4 @@ +# yamllint disable +end-of-epoch: true +si-snr: -17.452686241187454 +unixtime: 1614493079.2818 diff --git a/models/Sepformer/en/sepformer-wsj03mix/README.md b/models/Sepformer/en/sepformer-wsj03mix/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1335e7e8f69a875d3dcd58002ec06804e27cf5ca --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/README.md @@ -0,0 +1,125 @@ +--- +language: "en" +thumbnail: +tags: +- Source Separation +- Speech Separation +- Audio Source Separation +- WSJ0-3Mix +- SepFormer +- Transformer +- audio-to-audio +- audio-source-separation +- speechbrain +license: "apache-2.0" +datasets: +- WSJ0-3Mix +metrics: +- SI-SNRi +- SDRi + +--- + + +

+ +# SepFormer trained on WSJ0-3Mix + +This repository provides all the necessary tools to perform audio source separation with a [SepFormer](https://arxiv.org/abs/2010.13154v2) +model, implemented with SpeechBrain, and pretrained on WSJ0-3Mix dataset. For a better experience we encourage you to learn more about +[SpeechBrain](https://speechbrain.github.io). The model performance is 19.8 dB SI-SNRi on the test set of WSJ0-3Mix dataset. + +| Release | Test-Set SI-SNRi | Test-Set SDRi | +|:-------------:|:--------------:|:--------------:| +| 09-03-21 | 19.8dB | 20.0dB | + + +## Install SpeechBrain + +First of all, please install SpeechBrain with the following command: + +``` +pip install speechbrain +``` + +Please notice that we encourage you to read our tutorials and learn more about +[SpeechBrain](https://speechbrain.github.io). + +### Perform source separation on your own audio file + +```python +from speechbrain.inference.separation import SepformerSeparation as separator +import torchaudio + +model = separator.from_hparams(source="speechbrain/sepformer-wsj03mix", savedir='pretrained_models/sepformer-wsj03mix') + +est_sources = model.separate_file(path='speechbrain/sepformer-wsj03mix/test_mixture_3spks.wav') + +torchaudio.save("source1hat.wav", est_sources[:, :, 0].detach().cpu(), 8000) +torchaudio.save("source2hat.wav", est_sources[:, :, 1].detach().cpu(), 8000) +torchaudio.save("source3hat.wav", est_sources[:, :, 2].detach().cpu(), 8000) + +``` + +The system expects input recordings sampled at 8kHz (single channel). +If your signal has a different sample rate, resample it (e.g, using torchaudio or sox) before using the interface. + +### Inference on GPU +To perform inference on the GPU, add `run_opts={"device":"cuda"}` when calling the `from_hparams` method. + +### Training +The model was trained with SpeechBrain (fc2eabb7). +To train it from scratch follows these steps: +1. Clone SpeechBrain: +```bash +git clone https://github.com/speechbrain/speechbrain/ +``` +2. Install it: +``` +cd speechbrain +pip install -r requirements.txt +pip install -e . +``` + +3. Run Training: +``` +cd recipes/WSJ0Mix/separation +python train.py hparams/sepformer.yaml --data_folder=your_data_folder +``` +Note: change num_spks to 3 in the yaml file. + + +You can find our training results (models, logs, etc) [here](https://drive.google.com/drive/folders/1ruScDoqiSDNeoDa__u5472UUPKPu54b2?usp=sharing). + +### Limitations +The SpeechBrain team does not provide any warranty on the performance achieved by this model when used on other datasets. + +#### Referencing SpeechBrain + +```bibtex +@misc{speechbrain, + title={{SpeechBrain}: A General-Purpose Speech Toolkit}, + author={Mirco Ravanelli and Titouan Parcollet and Peter Plantinga and Aku Rouhe and Samuele Cornell and Loren Lugosch and Cem Subakan and Nauman Dawalatabad and Abdelwahab Heba and Jianyuan Zhong and Ju-Chieh Chou and Sung-Lin Yeh and Szu-Wei Fu and Chien-Feng Liao and Elena Rastorgueva and François Grondin and William Aris and Hwidong Na and Yan Gao and Renato De Mori and Yoshua Bengio}, + year={2021}, + eprint={2106.04624}, + archivePrefix={arXiv}, + primaryClass={eess.AS}, + note={arXiv:2106.04624} +} +``` + + +#### Referencing SepFormer +```bibtex +@inproceedings{subakan2021attention, + title={Attention is All You Need in Speech Separation}, + author={Cem Subakan and Mirco Ravanelli and Samuele Cornell and Mirko Bronzi and Jianyuan Zhong}, + year={2021}, + booktitle={ICASSP 2021} +} +``` + +# **About SpeechBrain** +- Website: https://speechbrain.github.io/ +- Code: https://github.com/speechbrain/speechbrain/ +- HuggingFace: https://huggingface.co/speechbrain/ \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-wsj03mix/brain.ckpt b/models/Sepformer/en/sepformer-wsj03mix/brain.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b7c35d9d9c2f9520d4975291c60e1e5e2d16f815 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/brain.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9e24193f36931b7f57932532efbdcf64971f42732383ba6808825f77db258f6 +size 28 diff --git a/models/Sepformer/en/sepformer-wsj03mix/config.json b/models/Sepformer/en/sepformer-wsj03mix/config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b1f71cf150b6f3ddc7fe135ac86a6e8eb946a1 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/config.json @@ -0,0 +1,3 @@ +{ + "speechbrain_interface": "SepformerSeparation" +} \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-wsj03mix/counter.ckpt b/models/Sepformer/en/sepformer-wsj03mix/counter.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3f3befe14e05e5e28ce6a4020b801c894b9e7be1 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/counter.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bcbb4c131df56f7c79066016241cc4bdf4e58db55c4f674e88b22365bd2e2ad +size 3 diff --git a/models/Sepformer/en/sepformer-wsj03mix/decoder.ckpt b/models/Sepformer/en/sepformer-wsj03mix/decoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0735c1a033f22954d778765055d370ccb5955203 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/decoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4744290ae39cee1c011bbb33e437ccf7b5b94c6c3b70fef595b6b7d53c32b0b7 +size 17202 diff --git a/models/Sepformer/en/sepformer-wsj03mix/encoder.ckpt b/models/Sepformer/en/sepformer-wsj03mix/encoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..44918fe00180bbcc0f9f522139c51e6df88aaabe --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/encoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afdabc2f213c6391002088c0faade7253f62c1ad910c58b3c42a3147f3421451 +size 17267 diff --git a/models/Sepformer/en/sepformer-wsj03mix/hyperparams.yaml b/models/Sepformer/en/sepformer-wsj03mix/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47c9ef35d348e69817d95ef00645abf1611a4719 --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/hyperparams.yaml @@ -0,0 +1,66 @@ +# ################################ +# Model: Inference for source separation with SepFormer +# https://arxiv.org/abs/2010.13154 +# Generated from speechbrain/recipes/WSJ0Mix/separation/train/hparams/sepformer-wsj03mix.yaml +# Dataset : wsj03mix +# ############################### + +# Parameters +sample_rate: 8000 +num_spks: 3 + +# Specifying the network +Encoder: !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + +SBtfintra: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + num_spks: !ref + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: !ref + inter_model: !ref + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +modules: + encoder: !ref + decoder: !ref + masknet: !ref + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + masknet: !ref + encoder: !ref + decoder: !ref + + + diff --git a/models/Sepformer/en/sepformer-wsj03mix/hyperparams_train.yaml b/models/Sepformer/en/sepformer-wsj03mix/hyperparams_train.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f59a81f582d1cb505e743985d68fc31eede2558e --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/hyperparams_train.yaml @@ -0,0 +1,163 @@ +# Generated 2021-03-09 from: +# /home/mila/s/subakany/speechbrain_new/recipes/WSJ0Mix/separation/yamls/dpt-3mix-17.yaml +# yamllint disable +# ################################ +# Model: SepFormer for source separation +# https://arxiv.org/abs/2010.13154 +# +# Dataset : WSJ0-mix +# ################################ +# Basic parameters +# Seed needs to be set at top of yaml, before objects with parameters are made +# +seed: 1234 +__set_seed: !apply:torch.manual_seed [1234] + +# Data params +data_folder: /network/tmp1/subakany/wsj0-mix/3speakers # wsj2mix or wsj3mix +experiment_name: 17-augment-3mix-dm +output_folder: results/17-augment-3mix-dm/1234 +train_log: results/17-augment-3mix-dm/1234/train_log.txt +save_folder: results/17-augment-3mix-dm/1234/save +train_data: results/17-augment-3mix-dm/1234/save/wsj_tr.csv +valid_data: results/17-augment-3mix-dm/1234/save/wsj_cv.csv +test_data: results/17-augment-3mix-dm/1234/save/wsj_tt.csv +wsj0_tr: /network/tmp1/subakany/wsj0-processed/si_tr_s/ +skip_prep: false + +# Experiment params +auto_mix_prec: true +test_only: true +num_spks: 3 # set to 3 for wsj0-3mix +progressbar: true +save_audio: false # Save estimated sources on disk +sample_rate: 8000 + +# Training parameters +N_epochs: 200 +batch_size: 1 +lr: 0.00015 +clip_grad_norm: 5 +loss_upper_lim: 999999 # this is the upper limit for an acceptable loss +# if True, the training sequences are cut to a specified length +limit_training_signal_len: false +# this is the length of sequences if we choose to limit +# the signal length of training sequences +training_signal_len: 32000 +dynamic_mixing: regular + +# Augment parameters +use_wavedrop: false +use_speedperturb: true +use_speedperturb_sameforeachsource: false +use_rand_shift: false +min_shift: -8000 +max_shift: 8000 + +# Neural parameters +N_encoder_out: 256 +out_channels: 256 +kernel_size: 16 +kernel_stride: 8 + +threshold_byloss: true +threshold: -30 + +# Dataloader options +dataloader_opts: + batch_size: 1 + num_workers: 3 + +speedperturb: !new:speechbrain.lobes.augment.TimeDomainSpecAugment + perturb_prob: 1.0 + drop_freq_prob: 0.0 + drop_chunk_prob: 0.0 + sample_rate: 8000 + speeds: [95, 100, 105] + +wavedrop: !new:speechbrain.lobes.augment.TimeDomainSpecAugment + perturb_prob: 0.0 + drop_freq_prob: 1.0 + drop_chunk_prob: 1.0 + sample_rate: 8000 + + +Encoder: &id003 !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + + +SBtfintra: &id001 !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: &id002 !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: &id005 !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + + num_spks: 3 + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: *id001 + inter_model: *id002 + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: &id004 !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +optimizer: !name:torch.optim.Adam + lr: 0.00015 + weight_decay: 0 + +loss: !name:speechbrain.nnet.losses.get_si_snr_with_pitwrapper + +lr_scheduler: &id007 !new:speechbrain.nnet.schedulers.ReduceLROnPlateau + + factor: 0.5 + patience: 2 + dont_halve_until_epoch: 100 + +epoch_counter: &id006 !new:speechbrain.utils.epoch_loop.EpochCounter + limit: 200 + +modules: + encoder: *id003 + decoder: *id004 + masknet: *id005 +checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer + checkpoints_dir: results/17-augment-3mix-dm/1234/save + recoverables: + encoder: *id003 + decoder: *id004 + masknet: *id005 + counter: *id006 + lr_scheduler: *id007 +train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger + save_file: results/17-augment-3mix-dm/1234/train_log.txt + + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + masknet: !ref + encoder: !ref + decoder: !ref diff --git a/models/Sepformer/en/sepformer-wsj03mix/lr_scheduler.ckpt b/models/Sepformer/en/sepformer-wsj03mix/lr_scheduler.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..af03386222d3c1552778525b0751dd408aed015f --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/lr_scheduler.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b399abc1f483ba137f394dc2ce4757f642de66584e0f3bb53602ab9af07ed17 +size 2223 diff --git a/models/Sepformer/en/sepformer-wsj03mix/masknet.ckpt b/models/Sepformer/en/sepformer-wsj03mix/masknet.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..42f339887ec5940848b20dd6b81255ad6bf52eee --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/masknet.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c11786f74723c8e7021f6c9769057724c5e6833d1ed0f09fc935adfcb8a0078 +size 113371626 diff --git a/models/Sepformer/en/sepformer-wsj03mix/optimizer.ckpt b/models/Sepformer/en/sepformer-wsj03mix/optimizer.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..097c8a43a9486deb8e3838aa56c69102a74ebccc --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/optimizer.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f94ab3d414ef043530ddb2d55baf995ac55297b82f70538cf11e3e40e5fae9e +size 206212823 diff --git a/models/Sepformer/en/sepformer-wsj03mix/source.txt b/models/Sepformer/en/sepformer-wsj03mix/source.txt new file mode 100644 index 0000000000000000000000000000000000000000..3e208d2fdb8b277687da26597fbd3ee501a0142f --- /dev/null +++ b/models/Sepformer/en/sepformer-wsj03mix/source.txt @@ -0,0 +1 @@ +https://huggingface.co/speechbrain/sepformer-wsj03mix \ No newline at end of file diff --git a/models/Sepformer/en/sepformer-wsj03mix/test_mixture_3spks.wav b/models/Sepformer/en/sepformer-wsj03mix/test_mixture_3spks.wav new file mode 100644 index 0000000000000000000000000000000000000000..af9e676006d6f6351adfea8d9f7d8968690efda4 Binary files /dev/null and b/models/Sepformer/en/sepformer-wsj03mix/test_mixture_3spks.wav differ diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/.gitattributes b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/CKPT.yaml b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/CKPT.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e25b4d7a3f9876a19d6c12ceffa3cb3169bda50f --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/CKPT.yaml @@ -0,0 +1,4 @@ +# yamllint disable +end-of-epoch: true +si-snr: -14.852269322423679 +unixtime: 1754496357.8188198 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/README.md b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0dcb142461f4aab73a7d66fbc96e9a65249bcc20 --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/README.md @@ -0,0 +1,5 @@ +--- +license: apache-2.0 +language: +- zh +--- \ No newline at end of file diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/brain.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/brain.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5066b059f205965fa2211d45674a0e7d996ba88b --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/brain.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33809a026a2c1febce7b03c8aafaee4ddfc851b2c70f180f8c06bf1017f4df5c +size 46 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/config.json b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b1f71cf150b6f3ddc7fe135ac86a6e8eb946a1 --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/config.json @@ -0,0 +1,3 @@ +{ + "speechbrain_interface": "SepformerSeparation" +} \ No newline at end of file diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/counter.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/counter.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..adccc0fc7ab99f7775e9c95a11722f6558898235 --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/counter.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a44dc15364204a80fe80e9039455cc1608281820fe2b24f1e5233ade6af1dd5 +size 2 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/dataloader-TRAIN.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/dataloader-TRAIN.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..65352b5ab3382773f77ea35d7a33dabd9fdc0abe --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/dataloader-TRAIN.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50280a378e2fb02fb74fe7bc5e306d16e22ce982a7e70d5a89f195464b40225f +size 5 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/decoder.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/decoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..73687e1d83e595c8eaf729de7e9c53aa44277fde --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/decoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25a65ebf66040572b12ae02e04a50f1c3cabcf2ab7801f8bc5bac07f75dbdbd1 +size 18025 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/encoder.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/encoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e4e1bb6d49dae231bc66b681e3e44dc322168d78 --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/encoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:341993bdc0cfc4e0411e05789c5e6209aeed05f05025befe6dcaa37b11958ced +size 18089 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/hyperparams.yaml b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a36698785e46c5312599387d0a9a0fcfe61f8f5b --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/hyperparams.yaml @@ -0,0 +1,64 @@ +# ################################ +# Model: Inference for source separation with SepFormer +# https://arxiv.org/abs/2010.13154 +# Generated from speechbrain/recipes/LibriMix/separation/hparams/sepformer-libri3mix.yaml +# Dataset : TCC300-edu +# ############################### + + +# Parameters +sample_rate: 16000 +num_spks: 3 + +# Specifying the network +Encoder: !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + +SBtfintra: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + num_spks: !ref + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: !ref + inter_model: !ref + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +modules: + encoder: !ref + decoder: !ref + masknet: !ref + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + masknet: !ref + encoder: !ref + decoder: !ref diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/hyperparams_train.yaml b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/hyperparams_train.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e1d1b272497e7e5047e7dcf7bbe31970f7c4d99 --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/hyperparams_train.yaml @@ -0,0 +1,209 @@ +# Generated 2025-08-06 from: +# /home/cgm/桌面/train_model/speechbrain/recipes/Aishell1Mix/separation/hparams/sepformer-tcc-300-3_dm.yaml +# yamllint disable +# ################################ +# Model: SepFormer for source separation +# https://arxiv.org/abs/2010.13154 +# Dataset : TCC-300-edu +# ################################ +# +# Basic parameters +# Seed needs to be set at top of yaml, before objects with parameters are made +# +seed: 1234 +__set_seed: !apply:speechbrain.utils.seed_everything [1234] + +# Data params + +# this is the storage folder for the whole needed dataset e.g. /yourpath +# it should contain at least 250G space for aishell1, wham_noise and aishell1mix. +num_spks: 3 +data_folder: /home/cgm/桌面/voice/TCC-300-MIX/revised_data +data_folder_nspks: /home/cgm/桌面/voice/TCC-300-MIX/revised_data +data_freqs: [16k] # or if you want the whole dataset ['8k','16k'] +data_modes: [min] # or if you want the whole dataset ['min', 'max'] + +# this is the base folder for dynamic mixing, usually not changed. +base_folder_dm: /home/cgm/桌面/voice/TCC-300-clean_processed16k + +experiment_name: sepformer-TCC300-3spks-16k-dm34-noise_ft +output_folder: results/sepformer-TCC300-3spks-16k-dm34-noise_ft/1234 +train_log: results/sepformer-TCC300-3spks-16k-dm34-noise_ft/1234/train_log.txt +save_folder: results/sepformer-TCC300-3spks-16k-dm34-noise_ft/1234/save +train_data: + /home/cgm/桌面/voice/TCC-300-MIX/revised_data/noise_train_data/train_mixture_final_noise.csv +valid_data: + /home/cgm/桌面/voice/TCC-300-MIX/revised_data/noise_train_data/valid_mixture_final_noise.csv +test_data: + /home/cgm/桌面/voice/TCC-300-MIX/revised_data/noise_train_data/test_mixture_final_noise.csv +skip_prep: true + +ckpt_interval_minutes: 60 + +# Experiment params +precision: fp16 # bf16, fp16 or fp32 # Set it to True for mixed precision +noprogressbar: false +save_audio: true # Save estimated sources on disk +sample_rate: 16000 + +####################### Training Parameters #################################### +N_epochs: 10 +batch_size: 1 +lr: 0.000015 +clip_grad_norm: 5 +loss_upper_lim: 999999 # this is the upper limit for an acceptable loss +# if True, the training sequences are cut to a specified length +limit_training_signal_len: true +# this is the length of sequences if we choose to limit +# the signal length of training sequences +training_signal_len: 96000 # ~ 6 secs + +# Set it to True to dynamically create mixtures at training time +dynamic_mixing: false +use_wham_noise: true +noise_snr: [0, 10] + +# Parameters for data augmentation +use_wavedrop: false +use_speedperturb: true +use_rand_shift: true +min_shift: -8000 +max_shift: 8000 + +# Speed perturbation +speed_changes: &id001 [95, 100, 105] + +# Frequency drop: randomly drops a number of frequency bands to zero. +speed_perturb: !new:speechbrain.augment.time_domain.SpeedPerturb + orig_freq: 16000 + speeds: *id001 +drop_freq_low: 0 # Min frequency band dropout probability +drop_freq_high: 1 # Max frequency band dropout probability +drop_freq_count_low: 1 # Min number of frequency bands to drop +drop_freq_count_high: 3 # Max number of frequency bands to drop +drop_freq_width: 0.05 # Width of frequency bands to drop + +drop_freq: !new:speechbrain.augment.time_domain.DropFreq + drop_freq_low: 0 + drop_freq_high: 1 + drop_freq_count_low: 1 + drop_freq_count_high: 3 + drop_freq_width: 0.05 + +# Time drop: randomly drops a number of temporal chunks. +drop_chunk_count_low: 1 # Min number of audio chunks to drop +drop_chunk_count_high: 5 # Max number of audio chunks to drop +drop_chunk_length_low: 1000 # Min length of audio chunks to drop +drop_chunk_length_high: 2000 # Max length of audio chunks to drop + +drop_chunk: !new:speechbrain.augment.time_domain.DropChunk + drop_length_low: 1000 + drop_length_high: 2000 + drop_count_low: 1 + drop_count_high: 5 + +# loss thresholding -- this thresholds the training loss +threshold_byloss: true +threshold: -30 + +# Encoder parameters +N_encoder_out: 256 +out_channels: 256 +kernel_size: 16 +kernel_stride: 8 +d_ffn: 1024 + +# Dataloader options +dataloader_opts: + batch_size: 1 + num_workers: 4 + + +# Specifying the network +Encoder: &id004 !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + + +SBtfintra: &id002 !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: &id003 !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: &id006 !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + + num_spks: 3 + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: *id002 + inter_model: *id003 + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: &id005 !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +optimizer: !name:torch.optim.Adam + lr: 0.000015 + weight_decay: 0 + +loss: !name:speechbrain.nnet.losses.get_si_snr_with_pitwrapper + +lr_scheduler: !new:speechbrain.nnet.schedulers.ReduceLROnPlateau + factor: 0.5 + patience: 2 + dont_halve_until_epoch: 5 + +epoch_counter: &id007 !new:speechbrain.utils.epoch_loop.EpochCounter + # lr_scheduler: !ref + + limit: 10 + +modules: + encoder: *id004 + decoder: *id005 + masknet: *id006 +checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer + checkpoints_dir: results/sepformer-TCC300-3spks-16k-dm34-noise_ft/1234/save + recoverables: + encoder: *id004 + decoder: *id005 + masknet: *id006 + counter: *id007 +train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger + save_file: results/sepformer-TCC300-3spks-16k-dm34-noise_ft/1234/train_log.txt + +# If you do not want to use the pretrained separator you can simply delete pretrained_separator field. +pretrained_separator: !new:speechbrain.utils.parameter_transfer.Pretrainer + collect_in: results/sepformer-TCC300-3spks-16k-dm34-noise_ft/1234/save + loadables: + encoder: *id004 + decoder: *id005 + masknet: *id006 + paths: + encoder: + /home/cgm/桌面/train_model/speechbrain/recipes/Aishell1Mix/separation/results/sepformer-TCC300-3spks-16k-dm/1234/save/CKPT+2025-07-12+19-19-42+00/encoder.ckpt + decoder: + /home/cgm/桌面/train_model/speechbrain/recipes/Aishell1Mix/separation/results/sepformer-TCC300-3spks-16k-dm/1234/save/CKPT+2025-07-12+19-19-42+00/decoder.ckpt + masknet: + /home/cgm/桌面/train_model/speechbrain/recipes/Aishell1Mix/separation/results/sepformer-TCC300-3spks-16k-dm/1234/save/CKPT+2025-07-12+19-19-42+00/masknet.ckpt diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/masknet.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/masknet.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..043b9b929866ecc7254985e2003e41e7b5775fff --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/masknet.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f80104b11065a24401e355c59bdb1ee1902baec155eef974e0f7924637723244 +size 113366721 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/optimizer.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/optimizer.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c9fdcf428737a4b3d2553f5664c2829355a05f4d --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/optimizer.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e40253ce461d9bd7e95e688900b2ca2c82d57843c2471c85f33b7603dda07e0b +size 206306938 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/scaler.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/scaler.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6c41e0bb2722b9282b24f3e79355d479fae7b44e --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/scaler.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b21c5349d5e7d02de630ebc1cb53ade1d9c6079eeb8594d223bb786011a0428b +size 1383 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/source.txt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/source.txt new file mode 100644 index 0000000000000000000000000000000000000000..a9fe7bdf885323c3b45aaac5f8c469e8d4879c6b --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k-noisy/source.txt @@ -0,0 +1 @@ +https://huggingface.co/AlvinLo62/sepformer-tcc300-3spks-16k-noisy \ No newline at end of file diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k/.gitattributes b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..0fefaacb01d3b3ac3457401f178c9902a1928a2f --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +mix_0002.wav filter=lfs diff=lfs merge=lfs -text +test_mixture_16k.wav filter=lfs diff=lfs merge=lfs -text diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k/README.md b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0dcb142461f4aab73a7d66fbc96e9a65249bcc20 --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/README.md @@ -0,0 +1,5 @@ +--- +license: apache-2.0 +language: +- zh +--- \ No newline at end of file diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k/brain.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/brain.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f1c8597c76a6b9a2500881c062f2430fc01e199e --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/brain.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f34b50994fb096d62715fb82e3bbc29280dcd62e1f6dbfe5de2dd7fb9a780fe0 +size 66 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k/config.json b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b1f71cf150b6f3ddc7fe135ac86a6e8eb946a1 --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/config.json @@ -0,0 +1,3 @@ +{ + "speechbrain_interface": "SepformerSeparation" +} \ No newline at end of file diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k/counter.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/counter.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5bcc51784518acfe53ddec7ed84f66c2977593bc --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/counter.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f14025af0065b30e47e23ebb3b491d39ae8ed17d33739e5ff3827ffb3634953 +size 2 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k/decoder.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/decoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..14c0b8b0f55325497a77ef25ee21a8e892cb8732 --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/decoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb8b076933bd9d5612269c5fb49eb959b2e03253572c38cbe9a334e1e95d55c2 +size 18025 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k/encoder.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/encoder.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..423711f008b27ee4aa3ec86b15a777dd40407ec8 --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/encoder.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfc34c3fbd062f2df545597d42a28e5a609776de601794fbab4a87526efa5a89 +size 18089 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k/hyperparams.yaml b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21e4c4fb13058b31733f40c6e126f7fd4eb6e9bd --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/hyperparams.yaml @@ -0,0 +1,64 @@ +# ################################ +# Model: Inference for source separation with SepFormer +# https://arxiv.org/abs/2010.13154 +# Generated from speechbrain/recipes/LibriMix/separation/hparams/sepformer-libri3mix.yaml +# Dataset : TCC300-edu +# ############################### + + +# Parameters +sample_rate: 16000 +num_spks: 3 + +# Specifying the network +Encoder: !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + +SBtfintra: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + num_spks: !ref + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: !ref + inter_model: !ref + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +modules: + encoder: !ref + decoder: !ref + masknet: !ref + +pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer + loadables: + masknet: !ref + encoder: !ref + decoder: !ref diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k/hyperparams_train.yaml b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/hyperparams_train.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a240df622f7e4fe73b7ebeebea1f907bdfda6c3 --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/hyperparams_train.yaml @@ -0,0 +1,202 @@ +# Generated 2025-07-08 from: +# /home/cgm/桌面/train_model/speechbrain/recipes/Aishell1Mix/separation/hparams/sepformer-tcc-300-3_dm.yaml +# yamllint disable +# ################################ +# Model: SepFormer for source separation +# https://arxiv.org/abs/2010.13154 +# Dataset : TCC-300-edu +# ################################ +# +# Basic parameters +# Seed needs to be set at top of yaml, before objects with parameters are made +# +seed: 1234 +__set_seed: !apply:speechbrain.utils.seed_everything [1234] + +# Data params + +# this is the storage folder for the whole needed dataset e.g. /yourpath +# it should contain at least 250G space for aishell1, wham_noise and aishell1mix. +num_spks: 3 +data_folder: /home/cgm/桌面/voice/TCC-300-MIX/revised_data +data_folder_nspks: /home/cgm/桌面/voice/TCC-300-MIX/revised_data +data_freqs: [16k] # or if you want the whole dataset ['8k','16k'] +data_modes: [min] # or if you want the whole dataset ['min', 'max'] + +# this is the base folder for dynamic mixing, usually not changed. +base_folder_dm: /home/cgm/桌面/voice/TCC-300-clean_processed16k + +experiment_name: sepformer-TCC300-3spks-16k-dm +output_folder: results/sepformer-TCC300-3spks-16k-dm/1234 +train_log: results/sepformer-TCC300-3spks-16k-dm/1234/train_log.txt +save_folder: results/sepformer-TCC300-3spks-16k-dm/1234/save +train_data: /home/cgm/桌面/voice/TCC-300-MIX/revised_data/train_mixture_final.csv +valid_data: /home/cgm/桌面/voice/TCC-300-MIX/revised_data/valid_mixture_final.csv +test_data: /home/cgm/桌面/voice/TCC-300-MIX/revised_data/test_mixture_final.csv +skip_prep: true + +ckpt_interval_minutes: 60 + +# Experiment params +precision: fp16 # bf16, fp16 or fp32 # Set it to True for mixed precision +noprogressbar: false +save_audio: true # Save estimated sources on disk +sample_rate: 16000 + +####################### Training Parameters #################################### +N_epochs: 200 +batch_size: 1 +lr: 0.00015 +clip_grad_norm: 5 +loss_upper_lim: 999999 # this is the upper limit for an acceptable loss +# if True, the training sequences are cut to a specified length +limit_training_signal_len: true +# this is the length of sequences if we choose to limit +# the signal length of training sequences +training_signal_len: 96000 # ~ 6 secs + +# Set it to True to dynamically create mixtures at training time +dynamic_mixing: true +use_wham_noise: false + +# Parameters for data augmentation +use_wavedrop: false +use_speedperturb: true +use_rand_shift: true +min_shift: -8000 +max_shift: 8000 + +# Speed perturbation +speed_changes: &id001 [95, 100, 105] + +# Frequency drop: randomly drops a number of frequency bands to zero. +speed_perturb: !new:speechbrain.augment.time_domain.SpeedPerturb + orig_freq: 16000 + speeds: *id001 +drop_freq_low: 0 # Min frequency band dropout probability +drop_freq_high: 1 # Max frequency band dropout probability +drop_freq_count_low: 1 # Min number of frequency bands to drop +drop_freq_count_high: 3 # Max number of frequency bands to drop +drop_freq_width: 0.05 # Width of frequency bands to drop + +drop_freq: !new:speechbrain.augment.time_domain.DropFreq + drop_freq_low: 0 + drop_freq_high: 1 + drop_freq_count_low: 1 + drop_freq_count_high: 3 + drop_freq_width: 0.05 + +# Time drop: randomly drops a number of temporal chunks. +drop_chunk_count_low: 1 # Min number of audio chunks to drop +drop_chunk_count_high: 5 # Max number of audio chunks to drop +drop_chunk_length_low: 1000 # Min length of audio chunks to drop +drop_chunk_length_high: 2000 # Max length of audio chunks to drop + +drop_chunk: !new:speechbrain.augment.time_domain.DropChunk + drop_length_low: 1000 + drop_length_high: 2000 + drop_count_low: 1 + drop_count_high: 5 + +# loss thresholding -- this thresholds the training loss +threshold_byloss: true +threshold: -30 + +# Encoder parameters +N_encoder_out: 256 +out_channels: 256 +kernel_size: 16 +kernel_stride: 8 +d_ffn: 1024 + +# Dataloader options +dataloader_opts: + batch_size: 1 + num_workers: 4 + + +# Specifying the network +Encoder: &id004 !new:speechbrain.lobes.models.dual_path.Encoder + kernel_size: 16 + out_channels: 256 + + +SBtfintra: &id002 !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +SBtfinter: &id003 !new:speechbrain.lobes.models.dual_path.SBTransformerBlock + num_layers: 8 + d_model: 256 + nhead: 8 + d_ffn: 1024 + dropout: 0 + use_positional_encoding: true + norm_before: true + +MaskNet: &id006 !new:speechbrain.lobes.models.dual_path.Dual_Path_Model + + num_spks: 3 + in_channels: 256 + out_channels: 256 + num_layers: 2 + K: 250 + intra_model: *id002 + inter_model: *id003 + norm: ln + linear_layer_after_inter_intra: false + skip_around_intra: true + +Decoder: &id005 !new:speechbrain.lobes.models.dual_path.Decoder + in_channels: 256 + out_channels: 1 + kernel_size: 16 + stride: 8 + bias: false + +optimizer: !name:torch.optim.Adam + lr: 0.00015 + weight_decay: 0 + +loss: !name:speechbrain.nnet.losses.get_si_snr_with_pitwrapper + +lr_scheduler: !new:speechbrain.nnet.schedulers.ReduceLROnPlateau + factor: 0.5 + patience: 2 + dont_halve_until_epoch: 5 + +epoch_counter: &id007 !new:speechbrain.utils.epoch_loop.EpochCounter + # lr_scheduler: !ref + + limit: 200 + +modules: + encoder: *id004 + decoder: *id005 + masknet: *id006 +checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer + checkpoints_dir: results/sepformer-TCC300-3spks-16k-dm/1234/save + recoverables: + encoder: *id004 + decoder: *id005 + masknet: *id006 + counter: *id007 +train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger + save_file: results/sepformer-TCC300-3spks-16k-dm/1234/train_log.txt + +# If you do not want to use the pretrained separator you can simply delete pretrained_separator field. +#pretrained_separator: !new:speechbrain.utils.parameter_transfer.Pretrainer +# collect_in: !ref +# loadables: +# encoder: !ref +# decoder: !ref +# masknet: !ref +# paths: +# encoder: speechbrain/sepformer-wsj03mix/encoder.ckpt +# decoder: speechbrain/sepformer-wsj03mix/decoder.ckpt +# masknet: speechbrain/sepformer-wsj03mix/masknet.ckpt diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k/masknet.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/masknet.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6d63c3f98734a7788bdb5e7e0e3195d76f67124b --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/masknet.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f8c1783721da5a28b2eeb98fc604a7e18b3ceef64dcf4552eb4c1ecb3cf3d32 +size 113366721 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k/optimizer.ckpt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/optimizer.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..09e16ac2e8ba6461bfc72e42f027562cbc6e00bb --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/optimizer.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e855f9b454b006a536b32502ff273ca7fc847408082ede7bc4f5a1af853efc04 +size 206306938 diff --git a/models/Sepformer/zh/sepformer-tcc300-3spks-16k/source.txt b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/source.txt new file mode 100644 index 0000000000000000000000000000000000000000..842fcb65a83e7e20309c9b2480a8a9b3e58b3711 --- /dev/null +++ b/models/Sepformer/zh/sepformer-tcc300-3spks-16k/source.txt @@ -0,0 +1 @@ +https://huggingface.co/AlvinLo62/sepformer-tcc300-3spks-16k \ No newline at end of file diff --git a/tools/ONNX converter/sepsep.py b/tools/ONNX converter/sepsep.py new file mode 100644 index 0000000000000000000000000000000000000000..6f92e411f4da8a5796f50a845f1ad63278755d46 --- /dev/null +++ b/tools/ONNX converter/sepsep.py @@ -0,0 +1,12 @@ +from speechbrain.pretrained import SepformerSeparation as separator +import torchaudio +import torch + +model = separator.from_hparams(source="speechbrain/sepformer-wsj02mix", savedir='tests/integration/neural_networks/separation/results/sepformer/1234/save/CKPT+2021-05-14+21-09-15+00') +dummy_input = torch.randn((1, 50000), requires_grad=True) + +# for custom file, change path +est_sources = model.separate_file(path='/home/weiyaodu/Desktop/mixture_0.wav') +torch.onnx.export(model.modules, dummy_input, '/home/weiyaodu/Desktop', verbose=True) +torchaudio.save("test_source1.wav", est_sources[:, :, 0].detach().cpu(), 16000) +torchaudio.save("test_source2.wav", est_sources[:, :, 1].detach().cpu(), 16000) \ No newline at end of file diff --git a/tools/ONNX converter/source.txt b/tools/ONNX converter/source.txt new file mode 100644 index 0000000000000000000000000000000000000000..3ef46346c8ae753128dbb62e6638a4a7c96cbeea --- /dev/null +++ b/tools/ONNX converter/source.txt @@ -0,0 +1,2 @@ +https://github.com/speechbrain/speechbrain/issues/756 +https://drive.google.com/file/d/1NKnUhe1vzWDjXohiT1HpQ8GZfPeFfb7p/view?usp=sharing \ No newline at end of file