speecht5_hifigan

speecht5_hifigan/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

speecht5_hifigan/README.md

@@ -0,0 +1,67 @@
+---
+base_model: microsoft/speecht5_hifigan
+library_name: transformers.js
+---
+
+https://huggingface.co/microsoft/speecht5_hifigan with ONNX weights to be compatible with Transformers.js.
+
+## Usage (Transformers.js)
+
+If you haven't already, you can install the [Transformers.js](https://huggingface.co/docs/transformers.js) JavaScript library from [NPM](https://www.npmjs.com/package/@huggingface/transformers) using:
+```bash
+npm i @huggingface/transformers
+```
+
+**Example:** Generate speech from text.
+```js
+import { AutoTokenizer, AutoProcessor, SpeechT5ForTextToSpeech, SpeechT5HifiGan, Tensor } from '@huggingface/transformers';
+
+// Load the tokenizer and processor
+const tokenizer = await AutoTokenizer.from_pretrained('Xenova/speecht5_tts');
+const processor = await AutoProcessor.from_pretrained('Xenova/speecht5_tts');
+
+// Load the models
+// NOTE: We use the unquantized versions as they are more accurate
+const model = await SpeechT5ForTextToSpeech.from_pretrained('Xenova/speecht5_tts', { dtype: "fp32" }); // Options: "fp32", "fp16", "q8", "q4"
+const vocoder = await SpeechT5HifiGan.from_pretrained('Xenova/speecht5_hifigan', { dtype: "fp32" }); // Options: "fp32", "fp16", "q8", "q4"
+
+// Load speaker embeddings from URL
+const speaker_embeddings_data = new Float32Array(
+    await (await fetch('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin')).arrayBuffer()
+);
+const speaker_embeddings = new Tensor(
+    'float32',
+    speaker_embeddings_data,
+    [1, speaker_embeddings_data.length]
+)
+
+// Run tokenization
+const { input_ids } = tokenizer('Hello, my dog is cute');
+
+// Generate waveform
+const { waveform } = await model.generate_speech(input_ids, speaker_embeddings, { vocoder });
+console.log(waveform);
+// Tensor {
+//   dims: [ 26112 ],
+//   type: 'float32',
+//   size: 26112,
+//   data: Float32Array(26112) [ -0.00043630177970044315, -0.00018082228780258447, ... ],
+// }
+```
+
+Optionally, save the audio to a wav file (Node.js):
+```js
+// Write to file (Node.js)
+import wavefile from 'wavefile';
+import fs from 'fs';
+
+const wav = new wavefile.WaveFile();
+wav.fromScratch(1, processor.feature_extractor.config.sampling_rate, '32f', waveform.data);
+fs.writeFileSync('out.wav', wav.toBuffer());
+```
+
+<audio controls src="https://cdn-uploads.huggingface.co/production/uploads/61b253b7ac5ecaae3d1efe0c/on1ij9Y269ne9zlYN9mdb.wav"></audio>
+
+---
+
+Note: Having a separate repo for ONNX weights is intended to be a temporary solution until WebML gains more traction. If you would like to make your models web-ready, we recommend converting to ONNX using [🤗 Optimum](https://huggingface.co/docs/optimum/index) and structuring your repo like this one (with ONNX weights located in a subfolder named `onnx`).

speecht5_hifigan/config.json

@@ -0,0 +1,48 @@
+{
+  "architectures": [
+    "SpeechT5HifiGan"
+  ],
+  "initializer_range": 0.01,
+  "leaky_relu_slope": 0.1,
+  "model_in_dim": 80,
+  "model_type": "hifigan",
+  "normalize_before": true,
+  "resblock_dilation_sizes": [
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ]
+  ],
+  "resblock_kernel_sizes": [
+    3,
+    7,
+    11
+  ],
+  "sampling_rate": 16000,
+  "torch_dtype": "float32",
+  "transformers_version": "4.27.0.dev0",
+  "upsample_initial_channel": 512,
+  "upsample_kernel_sizes": [
+    8,
+    8,
+    8,
+    8
+  ],
+  "upsample_rates": [
+    4,
+    4,
+    4,
+    4
+  ]
+}

speecht5_hifigan/onnx/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26408d9b8b3e83e66a3ef614a5621b84cacdcabd81b3a6a2fa984eb788566e6c
+size 55432026

speecht5_hifigan/onnx/model_quantized.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d092ea63bbc7839f7317657b5f531ebb68377ee05ec866ec096dba25911f9feb
+size 18251672

speecht5_hifigan/pth/.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

speecht5_hifigan/pth/README.md

@@ -0,0 +1,28 @@
+---
+license: mit
+tags:
+- audio
+---
+
+# SpeechT5 HiFi-GAN Vocoder
+
+This is the HiFi-GAN vocoder for use with the SpeechT5 text-to-speech and voice conversion models.
+
+SpeechT5 was first released in [this repository](https://github.com/microsoft/SpeechT5/), [original weights](https://huggingface.co/mechanicalsea/speecht5-tts). The license used is [MIT](https://github.com/microsoft/SpeechT5/blob/main/LICENSE).
+
+Disclaimer: The team releasing SpeechT5 did not write a model card for this model so this model card has been written by the Hugging Face team.
+
+## Citation
+
+**BibTeX:**
+
+```bibtex
+@inproceedings{ao-etal-2022-speecht5,
+    title = {{S}peech{T}5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing},
+    author = {Ao, Junyi and Wang, Rui and Zhou, Long and Wang, Chengyi and Ren, Shuo and Wu, Yu and Liu, Shujie and Ko, Tom and Li, Qing and Zhang, Yu and Wei, Zhihua and Qian, Yao and Li, Jinyu and Wei, Furu},
+    booktitle = {Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
+    month = {May},
+    year = {2022},
+    pages={5723--5738},
+}
+```

speecht5_hifigan/pth/config.json

@@ -0,0 +1,48 @@
+{
+  "architectures": [
+    "SpeechT5HifiGan"
+  ],
+  "initializer_range": 0.01,
+  "leaky_relu_slope": 0.1,
+  "model_in_dim": 80,
+  "model_type": "hifigan",
+  "normalize_before": true,
+  "resblock_dilation_sizes": [
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ]
+  ],
+  "resblock_kernel_sizes": [
+    3,
+    7,
+    11
+  ],
+  "sampling_rate": 16000,
+  "torch_dtype": "float32",
+  "transformers_version": "4.27.0.dev0",
+  "upsample_initial_channel": 512,
+  "upsample_kernel_sizes": [
+    8,
+    8,
+    8,
+    8
+  ],
+  "upsample_rates": [
+    4,
+    4,
+    4,
+    4
+  ]
+}

speecht5_hifigan/pth/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b171e9bcd8a2b50dc9780040478dfa26783a9ee4be012cf5776914f091d6887b
+size 50672453

speecht5_hifigan/source.txt

@@ -0,0 +1,2 @@
+https://huggingface.co/Xenova/speecht5_hifigan
+https://huggingface.co/microsoft/speecht5_hifigan