Duplicate from aufklarer/CosyVoice3-0.5B-MLX-4bit

Co-authored-by: Ivan <aufklarer@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,95 @@
+---
+language:
+  - zh
+  - en
+  - ja
+  - ko
+  - de
+  - es
+  - fr
+  - it
+  - ru
+license: apache-2.0
+tags:
+  - tts
+  - text-to-speech
+  - speech-synthesis
+  - mlx
+  - apple-silicon
+  - cosyvoice
+base_model: FunAudioLLM/Fun-CosyVoice3-0.5B-2512
+pipeline_tag: text-to-speech
+---
+
+# CosyVoice3-0.5B MLX 4-bit
+
+[CosyVoice 3](https://arxiv.org/abs/2505.17589) text-to-speech model converted to MLX safetensors format with 4-bit quantization for Apple Silicon inference.
+
+Converted from [FunAudioLLM/Fun-CosyVoice3-0.5B-2512](https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512).
+
+**Swift inference**: [ivan-digital/qwen3-asr-swift](https://github.com/ivan-digital/qwen3-asr-swift)
+
+## Model Details
+
+| Component | Architecture | Size |
+|-----------|-------------|------|
+| LLM | Qwen2.5-0.5B (24L, 896d, 14Q/2KV heads) | 467 MB (4-bit) |
+| DiT Flow Matching | 22-layer DiT (1024d, 16 heads, 10 ODE steps) | 634 MB (fp16) |
+| HiFi-GAN Vocoder | NSF + F0 predictor + ISTFT | 79 MB (fp16) |
+| **Total** | | **~1.2 GB** |
+
+## Pipeline
+
+```
+Text → LLM (Qwen2.5-0.5B) → Speech Tokens (FSQ 6561) → DiT Flow Matching → Mel (80-band) → HiFi-GAN → Audio (24kHz)
+```
+
+## Languages
+
+Chinese, English, Japanese, Korean, German, Spanish, French, Italian, Russian
+
+## Files
+
+- `llm.safetensors` — LLM weights (4-bit quantized)
+- `flow.safetensors` — DiT flow matching decoder (fp16)
+- `hifigan.safetensors` — HiFi-GAN vocoder (fp16, weight-norm folded)
+- `config.json` — Model configuration
+
+## Conversion Details
+
+- LLM: 4-bit quantization (group_size=64) of attention projections, MLP, and speech head
+- Flow: fp16 (flow matching is sensitive to quantization)
+- HiFi-GAN: fp16 with weight normalization folded (`w = g * v / ||v||`)
+- Conv1d weights transposed from PyTorch `[out, in, kernel]` to MLX `[out, kernel, in]`
+
+## Usage
+
+For use with [ivan-digital/qwen3-asr-swift](https://github.com/ivan-digital/qwen3-asr-swift):
+
+```swift
+import CosyVoiceTTS
+
+let model = try await CosyVoiceTTSModel.fromPretrained()
+let audio = model.synthesize(text: "Hello, how are you?", language: "english")
+```
+
+### CLI
+
+```bash
+swift run cosyvoice-tts-cli --text "Hello, how are you?" --lang english --output hello.wav
+```
+
+## License
+
+Apache 2.0 (same as upstream CosyVoice 3)
+
+## Citation
+
+```bibtex
+@article{du2025cosyvoice3,
+  title={CosyVoice 3: Towards In-the-wild Speech Generation via Scaling-up and Post-training},
+  author={Du, Zhihao and others},
+  journal={arXiv preprint arXiv:2505.17589},
+  year={2025}
+}
+```

config.json

@@ -0,0 +1,113 @@
+{
+  "model_type": "cosyvoice3",
+  "version": "Fun-CosyVoice3-0.5B-2512",
+  "llm": {
+    "hidden_size": 896,
+    "num_hidden_layers": 24,
+    "num_attention_heads": 14,
+    "num_key_value_heads": 2,
+    "intermediate_size": 4864,
+    "head_dim": 64,
+    "max_position_embeddings": 32768,
+    "vocab_size": 151936,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 1000000.0,
+    "tie_word_embeddings": true,
+    "speech_token_size": 6561,
+    "text_token_size": 151936
+  },
+  "flow": {
+    "input_size": 512,
+    "output_size": 80,
+    "vocab_size": 6561,
+    "spk_embed_dim": 192,
+    "token_frame_rate": 25,
+    "token_mel_ratio": 2,
+    "pre_lookahead_len": 3,
+    "dit": {
+      "dim": 1024,
+      "depth": 22,
+      "heads": 16,
+      "dim_head": 64,
+      "ff_mult": 2,
+      "mel_dim": 80,
+      "spk_dim": 80,
+      "static_chunk_size": 50
+    }
+  },
+  "hifigan": {
+    "sampling_rate": 24000,
+    "in_channels": 80,
+    "base_channels": 512,
+    "nb_harmonics": 8,
+    "upsample_rates": [
+      8,
+      5,
+      3
+    ],
+    "upsample_kernel_sizes": [
+      16,
+      11,
+      7
+    ],
+    "istft_n_fft": 16,
+    "istft_hop_len": 4,
+    "resblock_kernel_sizes": [
+      3,
+      7,
+      11
+    ],
+    "resblock_dilation_sizes": [
+      [
+        1,
+        3,
+        5
+      ],
+      [
+        1,
+        3,
+        5
+      ],
+      [
+        1,
+        3,
+        5
+      ]
+    ],
+    "source_resblock_kernel_sizes": [
+      7,
+      7,
+      11
+    ],
+    "nsf_alpha": 0.1,
+    "nsf_sigma": 0.003,
+    "nsf_voiced_threshold": 10,
+    "audio_limit": 0.99
+  },
+  "mel": {
+    "n_fft": 1920,
+    "num_mels": 80,
+    "hop_size": 480,
+    "win_size": 1920,
+    "sample_rate": 24000
+  },
+  "tokenizer": {
+    "type": "fsq",
+    "codebook_size": 6561,
+    "frame_rate": 25
+  },
+  "quantization": {
+    "bits": 4,
+    "group_size": 64,
+    "quantized_layers": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj",
+      "gate_proj",
+      "up_proj",
+      "down_proj",
+      "speech_head"
+    ]
+  }
+}

flow.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00357cda773dc8d17af72570aa655fc826f514c0314b40b57b9eeb690853db0a
+size 194964136

hifigan.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffae1b73e76efc7ea9778c3c07c24e64ffe7c124caf5f43a67f4fec9fcaca97c
+size 83086548

llm.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d7cfab99cf91c569509174a7383a46427405302b93dd60097e8187e46bc3757
+size 489278536

tokenizer_config.json

@@ -0,0 +1,40 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": ["<|im_start|>", "<|im_end|>"],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}