Fish Speech, OpenAudio

.gitattributes

@@ -33,3 +33,14 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+fish-speech-gui/releases/v1.4.5/fish.exe filter=lfs diff=lfs merge=lfs -text
+openaudio-gguf/codec-bf16.gguf filter=lfs diff=lfs merge=lfs -text
+openaudio-gguf/codec-f16.gguf filter=lfs diff=lfs merge=lfs -text
+openaudio-gguf/codec-f32.gguf filter=lfs diff=lfs merge=lfs -text
+openaudio-gguf/codec-q2_k.gguf filter=lfs diff=lfs merge=lfs -text
+openaudio-gguf/codec-q3_k_m.gguf filter=lfs diff=lfs merge=lfs -text
+openaudio-gguf/codec-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text
+openaudio-gguf/codec-q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text
+openaudio-gguf/codec-q6_k.gguf filter=lfs diff=lfs merge=lfs -text
+openaudio-gguf/samples/audio1.wav filter=lfs diff=lfs merge=lfs -text
+openaudio-gguf/samples/audio2.wav filter=lfs diff=lfs merge=lfs -text

fish-speech-gui/releases/v1.4.5/fish-amd64.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70735219758ba10f206907d0d56cac07ebba8dfdb34ed36a19adbfe5611e93d2
+size 73944982

fish-speech-gui/releases/v1.4.5/fish-speech-gui-v1.4.5.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ee695e5c1f8ac0d1cf90dcce24f8bdbf24ac4d2d767ef1783836cb62019cf44
+size 5790712

fish-speech-gui/releases/v1.4.5/fish.exe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3857a9c96bbf01909d18c466591ee4e897c6840cb8038f759eb8ac126656f9f0
+size 46259712

fish-speech.rs/releases/fish-speech.rs-v0.2.0.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2e42861abc754f5900bf72300926d2eceedbaf7a10e72664e38212b73ede2f9
+size 2056531

fish-speech.rs/releases/fish-speech.rs-v0.2.1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a108c7b6160291f61ec5fa15125e6cc74128bca25073ab64bb11fa5bea45e75b
+size 2057262

fish-speech.rs/releases/fish-speech.rs-v0.2.2.1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7daf3f8f62ade125de38edf24c980eee569ad9e6859d891403425a53098649b5
+size 2057828

fish-speech.rs/releases/fish-speech.rs-v0.2.2.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:579e57584b2b208cba19baa37aa05714e1135cb07d541ff003202f8f80a33cf3
+size 2057268

fish-speech.rs/releases/fish-speech.rs-v0.2.3.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a92987a8c4bf511be15344a4fe918c9fff48fc4f0915916ca49d02fa6e5414d3
+size 2057976

fish-speech.rs/releases/fish-speech.rs-v0.3.0.1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eaa3813bc3f53113d7ef2e648ed13c17cdaa53af1fdb42017912571c17e2424a
+size 2058697

fish-speech.rs/releases/fish-speech.rs-v0.3.0.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1aa614a8186c40b226bf51460eaed45feaf17143050628329c8f078dd1074886
+size 2057318

fish-speech/releases/fish-speech-v1.4.0.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be6c9466ad5ffc7e334fda81442c8818fd8090ffef47285b9dae15dcd4dcf480
+size 606436

fish-speech/releases/fish-speech-v1.4.1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:beeac30cdc5ceef826faa445c5a5fe3813ba48b6fcf5f35a632b6c4ed92ce6d1
+size 608016

fish-speech/releases/fish-speech-v1.4.2.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77c3e85969e21f1d1cf2789e6a16c6afd6fd0960724f1214ce7907f4c059e608
+size 627081

fish-speech/releases/fish-speech-v1.4.3.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bb033c62c63298d0d357f35bf34d184b8284ee35baf4ff2a6c7427440ef6a3f
+size 926563

fish-speech/releases/fish-speech-v1.5.0.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3022b240d5be41943bb300ad5d7bd8e9fa32e81aa5e37ad94f5a51aa532276ec
+size 935374

fish-speech/releases/fish-speech-v1.5.1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0161ab223786f0ac31dbdddeb4b423d19fe027be6dcfe7033c16843df0a320b7
+size 887692

openaudio-gguf/.gitattributes

@@ -0,0 +1,45 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+codec-bf16.gguf filter=lfs diff=lfs merge=lfs -text
+codec-f16.gguf filter=lfs diff=lfs merge=lfs -text
+codec-f32.gguf filter=lfs diff=lfs merge=lfs -text
+codec-q2_k.gguf filter=lfs diff=lfs merge=lfs -text
+codec-q3_k_m.gguf filter=lfs diff=lfs merge=lfs -text
+codec-q4_k_m.gguf filter=lfs diff=lfs merge=lfs -text
+codec-q5_k_m.gguf filter=lfs diff=lfs merge=lfs -text
+codec-q6_k.gguf filter=lfs diff=lfs merge=lfs -text
+samples/audio1.wav filter=lfs diff=lfs merge=lfs -text
+samples/audio2.wav filter=lfs diff=lfs merge=lfs -text

openaudio-gguf/README.md

@@ -0,0 +1,46 @@
+---
+license: cc-by-nc-sa-4.0
+base_model:
+- fishaudio/openaudio-s1-mini
+pipeline_tag: text-to-speech
+tags:
+- gguf-connector
+---
+## gguf quantized version of openaudio
+- base model from [fishaudio](https://huggingface.co/fishaudio)
+- text-to-speech synthesis
+
+### **run it with gguf-connector**
+```
+ggc o2
+```
+
+![screenshot](https://raw.githubusercontent.com/calcuis/text-to-speech-synthesis-2/master/demo.png)
+
+| Prompt | Audio Sample |
+|--------|---------------|
+|`Hey Connector, why your appearance looks so stupid?`<br/>`Oh, really? maybe I ate too much smart beans.`<br/>`Wow. Amazing (laughing).`<br/>`Let's go to get some more smart beans and you will become stupid as well.`<br/> | 🎧 **audio-sample-1**<br><audio controls src="https://huggingface.co/calcuis/openaudio-gguf/resolve/main/samples%5Caudio1.wav"></audio> |
+|`Suddenly the plane's engines began failing, and the pilot says there isn't much time, and he'll keep the plane in the air as long as he can, and told his two passengers to take the only two parachutes on board and bail out. The world's smartest man immediately took a parachute and said "I'm the world's smartest man! The world needs me, so I can't die here!", and then jumped out of the plane. The pilot tells the hippie to hurry up and take the other parachute, because there aren't any more. And the hippie says "Relax man. We'll be fine. The world's smartest man took my backpack."`<br/> | 🎧 **audio-sample-2**<br><audio controls src="https://huggingface.co/calcuis/openaudio-gguf/resolve/main/samples%5Caudio2.wav"></audio> |
+
+### **review/reference**
+- simply execute the command (`ggc o2`) above in console/terminal
+- opt a `codec` gguf file in the current directory to interact with (see example below)
+
+>
+>GGUF file(s) available. Select which one for codec:
+>
+>1. codec-bf16.gguf
+>2. codec-f16.gguf
+>3. codec-f32.gguf
+>4. codec-q2_k.gguf
+>5. codec-q3_k_m.gguf
+>6. codec-q4_k_m.gguf
+>7. codec-q5_k_m.gguf
+>8. codec-q6_k.gguf
+>
+>Enter your choice (1 to 8): _
+> 
+
+- note: tokenizer and model will be pulled to `models/fish` automatically during the first launch
+- then run it entirely offline; i.e., from local URL:  http://127.0.0.1:7860 with lazy webui
+- gguf-connector ([pypi](https://pypi.org/project/gguf-connector))

openaudio-gguf/codec-bf16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:261394a822e17e24dd24a222c4511fb560e72efa3125efc1760a7577d31a0bd2
+size 1390445280

openaudio-gguf/codec-f16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc386742876414b1810baa790fb2b545960b8932d9f0ba27396575fece7f4503
+size 1390445280

openaudio-gguf/codec-f32.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fae3c0fa7bd162dd45751502929da4fc7e6800aa38e43cd816f294ff9f15348
+size 2780022496

openaudio-gguf/codec-q2_k.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:046b2dc7ddc9292929210e886a7009ca7007b924a99ed9efebc1499fe0a64a1e
+size 878543584

openaudio-gguf/codec-q3_k_m.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da0575a707e08b74d2c4a2c1dd1647cf86de4adf2e66e461d27ac5dc76aa35f5
+size 909640416

openaudio-gguf/codec-q4_k_m.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16109c1c80cfeda62e78ec55d1f69c947a99119b5dba95eba0186c06a61d8984
+size 950305504

openaudio-gguf/codec-q5_k_m.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86fcb3dd9b5193b6ba06843ed2beddcd3534a84d751bb464fbb2a1f87131fad4
+size 988578528

openaudio-gguf/codec-q6_k.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12c1dee9265853be3fceed63b33d25dd9eb4f2496d2f6e8fea5a0788e499e5cd
+size 1029243616

openaudio-gguf/samples/audio1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:958951b601f18b74a4a896cdfac27ba00359600cf33756792d1297811e89df7d
+size 835628

openaudio-gguf/samples/audio2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6fa0bc4cf6a49d45b264532850239f41ef9bed407fd3a154b6d79c8c5f5a393
+size 2396204

openaudio-s1-mini/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

openaudio-s1-mini/README.md

@@ -0,0 +1,92 @@
+---
+tags:
+- text-to-speech
+license: cc-by-nc-sa-4.0
+language:
+- zh
+- en
+- de
+- ja
+- fr
+- es
+- ko
+- ar
+- nl
+- ru
+- it
+- pl
+- pt
+pipeline_tag: text-to-speech
+inference: false
+extra_gated_prompt: >-
+  You agree to not use the model to generate contents that violate DMCA or local
+  laws.
+extra_gated_fields:
+  Country: country
+  Specific date: date_picker
+  I agree to use this model for non-commercial use ONLY: checkbox
+---
+
+
+# OpenAudio S1
+
+**OpenAudio S1** is a leading text-to-speech (TTS) model trained on more than 2 million hours of audio data in multiple languages.
+
+Supported languages:
+- English (en)
+- Chinese (zh)
+- Japanese (ja)
+- German (de)
+- French (fr)
+- Spanish (es)
+- Korean (ko)
+- Arabic (ar)
+- Russian (ru)
+- Dutch (nl)
+- Italian (it)
+- Polish (pl)
+- Portuguese (pt)
+
+Please refer to [Fish Speech Github](https://github.com/fishaudio/fish-speech) for more info.
+Demo available at [Fish Audio Playground](https://fish.audio).
+Visit the [OpenAudio website](https://openaudio.com) for blog & tech report.
+
+## Emotion and Tone Support
+
+OpenAudio S1 supports a variety of emotional, tone, and special markers to enhance speech synthesis:
+
+**1. Emotional markers:**
+(angry) (sad) (disdainful) (excited) (surprised) (satisfied) (unhappy) (anxious) (hysterical) (delighted) (scared) (worried) (indifferent) (upset) (impatient) (nervous) (guilty) (scornful) (frustrated) (depressed) (panicked) (furious) (empathetic) (embarrassed) (reluctant) (disgusted) (keen) (moved) (proud) (relaxed) (grateful) (confident) (interested) (curious) (confused) (joyful) (disapproving) (negative) (denying) (astonished) (serious) (sarcastic) (conciliative) (comforting) (sincere) (sneering) (hesitating) (yielding) (painful) (awkward) (amused)
+
+**2. Tone markers:**
+(in a hurry tone) (shouting) (screaming) (whispering) (soft tone)
+
+**3. Special markers:**
+(laughing) (chuckling) (sobbing) (crying loudly) (sighing) (panting) (groaning) (crowd laughing) (background laughter) (audience laughing)
+
+**Special markers with corresponding onomatopoeia:**
+- Laughing: Ha,ha,ha
+- Chuckling: Hmm,hmm
+
+## Model Variants and Performance
+
+OpenAudio S1 includes the following models:
+-   **S1 (4B, proprietary):** The full-sized model.
+-   **S1-mini (0.5B):** A distilled version of S1.
+
+Both S1 and S1-mini incorporate online Reinforcement Learning from Human Feedback (RLHF).
+
+**Seed TTS Eval Metrics (English, auto eval, based on OpenAI gpt-4o-transcribe, speaker distance using Revai/pyannote-wespeaker-voxceleb-resnet34-LM):**
+
+-   **S1:**
+    -   WER (Word Error Rate): **0.008**
+    -   CER (Character Error Rate): **0.004**
+    -   Distance: **0.332**
+-   **S1-mini:**
+    -   WER (Word Error Rate): **0.011**
+    -   CER (Character Error Rate): **0.005**
+    -   Distance: **0.380**
+
+## License
+
+This model is permissively licensed under the CC-BY-NC-SA-4.0 license.

openaudio-s1-mini/codec.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74fc41c5a7151c6f350af8bd7e5d6e3accfcc7f3dfbfac23afd35af07052bb2f
+size 1871099728

openaudio-s1-mini/config.json

@@ -0,0 +1,32 @@
+{
+    "attention_o_bias": false,
+    "attention_qk_norm": true,
+    "attention_qkv_bias": false,
+    "codebook_size": 4096,
+    "dim": 1024,
+    "dropout": 0.0,
+    "fast_attention_o_bias": false,
+    "fast_attention_qk_norm": false,
+    "fast_attention_qkv_bias": false,
+    "fast_dim": 1024,
+    "fast_head_dim": 64,
+    "fast_intermediate_size": 3072,
+    "fast_n_head": 16,
+    "fast_n_local_heads": 8,
+    "head_dim": 128,
+    "initializer_range": 0.03125,
+    "intermediate_size": 3072,
+    "max_seq_len": 8192,
+    "model_type": "dual_ar",
+    "n_fast_layer": 4,
+    "n_head": 16,
+    "n_layer": 28,
+    "n_local_heads": 8,
+    "norm_eps": 1e-06,
+    "num_codebooks": 10,
+    "rope_base": 1000000,
+    "scale_codebook_embeddings": true,
+    "tie_word_embeddings": false,
+    "use_gradient_checkpointing": true,
+    "vocab_size": 155776
+}

openaudio-s1-mini/model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e59be7dc6714040dce3cde1f41e730c2f0daa5339785b1cd3b60041208c35e6
+size 1735122974