Duplicate from onnx-community/Supertonic-TTS-ONNX

Co-authored-by: Joshua <Xenova@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,38 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+onnx/latent_denoiser.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/text_encoder.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/voice_decoder.onnx_data filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,151 @@
+---
+license: openrail
+base_model:
+- Supertone/supertonic
+library_name: transformers.js
+language:
+- en
+pipeline_tag: text-to-speech
+---
+
+## Usage
+
+### Transformers.js
+
+If you haven't already, you can install the [Transformers.js](https://huggingface.co/docs/transformers.js) JavaScript library from [NPM](https://www.npmjs.com/package/@huggingface/transformers) using:
+```bash
+npm i @huggingface/transformers
+```
+
+You can then generate audio as follows:
+```js
+import { pipeline } from '@huggingface/transformers';
+
+const tts = await pipeline('text-to-speech', 'onnx-community/Supertonic-TTS-ONNX');
+
+const input_text = 'This is really cool!';
+const audio = await tts(input_text, {
+    speaker_embeddings: 'https://huggingface.co/onnx-community/Supertonic-TTS-ONNX/resolve/main/voices/F1.bin',
+    num_inference_steps: 5, // Higher = better quality (typically 1-50)
+    speed: 1.05, // Higher = faster speech (typically 0.8-1.2)
+});
+await audio.save('output.wav'); // or `audio.toBlob()`;
+```
+
+### ONNXRuntime
+
+First, let's create a helper class, `SupertonicTTS`:
+
+```py
+import os
+import numpy as np
+import onnxruntime as ort
+from transformers import AutoTokenizer
+
+class SupertonicTTS:
+    SAMPLE_RATE = 44100
+    CHUNK_COMPRESS_FACTOR = 6
+    BASE_CHUNK_SIZE = 512
+    LATENT_DIM = 24
+    STYLE_DIM = 128
+    LATENT_SIZE = BASE_CHUNK_SIZE * CHUNK_COMPRESS_FACTOR
+
+    def __init__(self, model_path):
+        self.model_path = model_path
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_path)
+
+        # Load ONNX sessions
+        self.text_encoder = ort.InferenceSession(os.path.join(self.model_path, "onnx", "text_encoder.onnx"))
+        self.latent_denoiser = ort.InferenceSession(os.path.join(self.model_path, "onnx", "latent_denoiser.onnx"))
+        self.voice_decoder = ort.InferenceSession(os.path.join(self.model_path, "onnx", "voice_decoder.onnx"))
+
+    def _load_style(self, voice: str) -> np.ndarray:
+        voice_path = os.path.join(self.model_path, "voices", f"{voice}.bin")
+        if not os.path.exists(voice_path):
+            raise ValueError(f"Voice '{voice}' not found.")
+
+        style_vec = np.fromfile(voice_path, dtype=np.float32)
+        return style_vec.reshape(1, -1, self.STYLE_DIM)
+
+    def generate(self, text: list[str], *, voice: str = "M1", speed: float = 1.0, steps: int = 5) -> list[np.ndarray]:
+        # 1. Prepare Text Inputs
+        inputs = self.tokenizer(text, return_tensors="np", padding=True, truncation=True)
+        input_ids = inputs["input_ids"]
+        attn_mask = inputs["attention_mask"]
+        batch_size = input_ids.shape[0]
+
+        # 2. Prepare Style
+        style = self._load_style(voice).repeat(batch_size, axis=0)
+
+        # 3. Text Encoding
+        last_hidden_state, raw_durations = self.text_encoder.run(
+            None, 
+            {"input_ids": input_ids, "attention_mask": attn_mask, "style": style}
+        )
+        durations = (raw_durations / speed * self.SAMPLE_RATE).astype(np.int64)
+
+        # 4. Latent Preparation
+        latent_lengths = (durations + self.LATENT_SIZE - 1) // self.LATENT_SIZE
+        max_len = latent_lengths.max()
+        latent_mask = (np.arange(max_len) < latent_lengths[:, None]).astype(np.int64)
+        latents = np.random.randn(batch_size, self.LATENT_DIM * self.CHUNK_COMPRESS_FACTOR, max_len).astype(np.float32)
+        latents *= latent_mask[:, None, :]
+
+        # 5. Denoising Loop
+        num_inference_steps = np.full(batch_size, steps, dtype=np.float32)
+        for step in range(steps):
+            timestep = np.full(batch_size, step, dtype=np.float32)
+            latents = self.latent_denoiser.run(
+                None,
+                {
+                    "noisy_latents": latents,
+                    "latent_mask": latent_mask,
+                    "style": style,
+                    "encoder_outputs": last_hidden_state,
+                    "attention_mask": attn_mask,
+                    "timestep": timestep,
+                    "num_inference_steps": num_inference_steps,
+                },
+            )[0]
+
+        # 6. Decode Latents to Audio
+        waveforms = self.voice_decoder.run(None, {"latents": latents})[0]
+
+        # 7. Post-process: Trim padding and return list of arrays
+        results = []
+        for i, length in enumerate(latent_mask.sum(axis=1) * self.LATENT_SIZE):
+            results.append(waveforms[i, :length])
+
+        return results
+```
+
+Next, we clone this repository (using whichever way you want, `git clone`, `huggingface_hub`, etc.)
+```py
+# (Optional) Download model files (or use existing local directory)
+from huggingface_hub import snapshot_download
+model_id = "onnx-community/Supertonic-TTS-ONNX"
+local_dir = "supertonic"
+snapshot_download(model_id, local_dir=local_dir)
+```
+
+We can then use the model as follows:
+
+```py
+# Initialize TTS
+tts = SupertonicTTS(local_dir)
+
+# Generate audio
+prompts = [
+    "Once upon a time, there was a brave knight.",
+    "Refactoring code makes it much easier to read!",
+    "I love this!"
+]
+audio_data = tts.generate(prompts, voice="M1", speed=1.0, steps=10)
+
+# (Optional) Save to files
+import soundfile as sf
+for i, audio in enumerate(audio_data):
+    filename = f"output_{i}.wav"
+    sf.write(filename, audio, tts.SAMPLE_RATE)
+    print(f"Saved {filename}")
+```

config.json

@@ -0,0 +1,12 @@
+{
+  "base_chunk_size": 512,
+  "chunk_compress_factor": 6,
+  "latent_dim": 24,
+  "model_type": "supertonic",
+  "sampling_rate": 44100,
+  "style_dim": 128,
+  "transformers.js_config": {
+    "dtype": "fp32",
+    "use_external_data_format": true
+  }
+}

onnx/latent_denoiser.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a639a8c05c9be111848562c5cf10ea2697a589c6341830aac479d0ce7b75aa9
+size 398102

onnx/latent_denoiser.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cde4abf1136defce235bc446eaab4954a57721ae8d5a4754cdd337bf191b612f
+size 132098880

onnx/text_encoder.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50a03d29d5dc95918eeff578f542b814f3cf5a741f927116f5a8462a76ff6898
+size 433169

onnx/text_encoder.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6415854f135a318909dc716e90f83a391d9a91bd9da09bdb6d6763d6b0a6c102
+size 28426752

onnx/voice_decoder.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83c104006dabcd6b568c0d5acb6fec18f65609d2391dd2c459e4440e85027669
+size 59921

onnx/voice_decoder.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea52402c9ba5131ee2b3901a86db2f0b435b322169cd75157e053493d967d17f
+size 101353472

tokenizer.json

@@ -0,0 +1,130 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "NFKD"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "\\s+"
+        },
+        "content": " "
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "[\u2013\u2014]"
+        },
+        "content": "-"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": "[^ -\"$-.0-;?A-Za-z£́]"
+        },
+        "content": ""
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "FixedLength",
+    "length": 1
+  },
+  "post_processor": null,
+  "decoder": {
+    "type": "Fuse"
+  },
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      " ": 0,
+      "!": 1,
+      "\"": 2,
+      "$": 3,
+      "%": 4,
+      "&": 5,
+      "'": 6,
+      "(": 7,
+      ")": 8,
+      "*": 9,
+      "+": 10,
+      ",": 11,
+      "-": 12,
+      ".": 13,
+      "0": 14,
+      "1": 15,
+      "2": 16,
+      "3": 17,
+      "4": 18,
+      "5": 19,
+      "6": 20,
+      "7": 21,
+      "8": 22,
+      "9": 23,
+      ":": 24,
+      ";": 25,
+      "?": 26,
+      "A": 27,
+      "B": 28,
+      "C": 29,
+      "D": 30,
+      "E": 31,
+      "F": 32,
+      "G": 33,
+      "H": 34,
+      "I": 35,
+      "J": 36,
+      "K": 37,
+      "L": 38,
+      "M": 39,
+      "N": 40,
+      "O": 41,
+      "P": 42,
+      "Q": 43,
+      "R": 44,
+      "S": 45,
+      "T": 46,
+      "U": 47,
+      "V": 48,
+      "W": 49,
+      "X": 50,
+      "Y": 51,
+      "Z": 52,
+      "a": 53,
+      "b": 54,
+      "c": 55,
+      "d": 56,
+      "e": 57,
+      "f": 58,
+      "g": 59,
+      "h": 60,
+      "i": 61,
+      "j": 62,
+      "k": 63,
+      "l": 64,
+      "m": 65,
+      "n": 66,
+      "o": 67,
+      "p": 68,
+      "q": 69,
+      "r": 70,
+      "s": 71,
+      "t": 72,
+      "u": 73,
+      "v": 74,
+      "w": 75,
+      "x": 76,
+      "y": 77,
+      "z": 78,
+      "£": 79,
+      "\u0301": 80
+    },
+    "unk_token": "\u0301"
+  }
+}

tokenizer_config.json

@@ -0,0 +1,6 @@
+{
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "model_max_length": 1000,
+  "pad_token": " ",
+  "pad_token_id": 0
+}

voices/F1.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ef84e3421e4f80994a5a40a18ba39ba9fc48175c41ae6cf3e56418820872dbf
+size 51712

voices/F2.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1949cf0e066c4278980d2b835cf334dab0f8f781704c9116bf48a072278f7c72
+size 51712

voices/F3.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38ee1d62ad8a02877ab0d08b501742b76cf3586ed888514df1a7f27cc0f8d171
+size 51712

voices/F4.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63890c361868a296c51f9aee114f51e0a9a92c3f46a91582539545f7ab408a72
+size 51712

voices/F5.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:793223d8d11e0ee49721842ebdc7bd46b4487579588f646953e75ad3fc8ffb9c
+size 51712

voices/M1.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d53fbaaccf39a358010dcc5f289fc1d5cb350fe5f518be35f62cc518d794892
+size 51712

voices/M2.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e02979a394f89002d920f0bcc006206d4cd8da90e8cc82d0532831a5bb20e79
+size 51712

voices/M3.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:470d2b6b77239628ce90ba879ca5366fb5e6103fdd7e7053954a7b6d5dc2142a
+size 51712

voices/M4.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4700e92c614fd34971a8ed9c8140c2f2162ab8ef3067f8e1e7ef67c3e6488fb7
+size 51712

voices/M5.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c40fbc4093d113ef261cbc7bfe3f080dd813d3168347d682c78b1ca71a07da1f
+size 51712