| { |
| "model_name": "styletts2-libritts-coreml", |
| "version": "1.0.0", |
| "base_model": "yl4579/StyleTTS2 LibriTTS checkpoint (epoch 20, 2nd-stage)", |
| "sample_rate": 24000, |
| "frame_hop": 300, |
| "phoneme_vocab": "espeak-ng en-us IPA + StyleTTS2 TextCleaner", |
| "limits": { |
| "max_phonemes": 57, |
| "note": "bert and diffusion_unet stages have a fixed token axis of 57 (CoreML CPU MLProgram backend rejects RangeDim on these graphs). Inputs producing more than 57 phonemes will fail until token-bucketed packages are added." |
| }, |
| "stages": [ |
| { |
| "name": "text_encoder", |
| "package": "packages/text_encoder_fp16.mlpackage", |
| "precision": "fp16", |
| "compute_units": "CPU_ONLY", |
| "inputs": [ |
| { "name": "tokens", "shape": [1, "T_token"], "dtype": "int32", "range": [1, 512] }, |
| { "name": "input_lengths", "shape": [1], "dtype": "int32" }, |
| { "name": "text_mask", "shape": [1, "T_token"], "dtype": "float32" } |
| ], |
| "outputs": [ |
| { "name": "t_en", "shape": [1, 512, "T_token"], "dtype": "float32" } |
| ] |
| }, |
| { |
| "name": "bert", |
| "package": "packages/bert_fp16.mlpackage", |
| "precision": "fp16", |
| "compute_units": "CPU_AND_NE", |
| "fixed_token_axis": 57, |
| "inputs": [ |
| { "name": "tokens", "shape": [1, 57], "dtype": "int32" }, |
| { "name": "attention_mask", "shape": [1, 57], "dtype": "int32" } |
| ], |
| "outputs": [ |
| { "name": "bert_dur", "shape": [1, 57, 768] }, |
| { "name": "d_en", "shape": [1, 512, 57] } |
| ] |
| }, |
| { |
| "name": "ref_encoder", |
| "package": "packages/ref_encoder_fp16.mlpackage", |
| "precision": "fp16", |
| "compute_units": "CPU_AND_NE", |
| "inputs": [ |
| { "name": "mel", "shape": [1, 1, 80, "T_mel"], "dtype": "float32", |
| "note": "24 kHz mel spectrogram of reference audio. n_fft=2048, hop=300, win=1200, n_mels=80." } |
| ], |
| "outputs": [ |
| { "name": "ref_s", "shape": [1, 256], "dtype": "float32", |
| "note": "Style embedding. ref_s[:, :128] is reference timbre, ref_s[:, 128:] is reference prosody." } |
| ] |
| }, |
| { |
| "name": "diffusion_unet", |
| "package": "packages/diffusion_unet_fp16.mlpackage", |
| "precision": "fp16", |
| "compute_units": "CPU_AND_NE", |
| "fixed_token_axis": 57, |
| "inputs": [ |
| { "name": "x_noisy", "shape": [1, 1, 256] }, |
| { "name": "sigma", "shape": [1] }, |
| { "name": "embedding", "shape": [1, 57, 768] }, |
| { "name": "features", "shape": [1, 256] } |
| ], |
| "outputs": [ |
| { "name": "x_denoised", "shape": [1, 1, 256] } |
| ], |
| "note": "Called num_steps × 2 dispatches per utterance under ADPM2 sampler. Use Karras sigmas (sigma_min=0.0001, sigma_max=3.0, rho_schedule=9.0). 5 steps default." |
| }, |
| { |
| "name": "duration_predictor", |
| "package": "packages/duration_predictor_fp16.mlpackage", |
| "precision": "fp16", |
| "compute_units": "CPU_ONLY", |
| "inputs": [ |
| { "name": "d_en", "shape": [1, 512, "T_token"] }, |
| { "name": "s", "shape": [1, 128] }, |
| { "name": "text_mask", "shape": [1, "T_token"] } |
| ], |
| "outputs": [ |
| { "name": "d", "shape": [1, "T_token", 640] }, |
| { "name": "duration_logits", "shape": [1, "T_token", 50] } |
| ] |
| }, |
| { |
| "name": "f0n_predictor", |
| "package": "packages/f0n_predictor_fp16.mlpackage", |
| "precision": "fp16", |
| "compute_units": "CPU_AND_NE", |
| "inputs": [ |
| { "name": "en", "shape": [1, 640, "T_frame"] }, |
| { "name": "s", "shape": [1, 128] } |
| ], |
| "outputs": [ |
| { "name": "f0_pred", "shape": [1, "F0_LEN"] }, |
| { "name": "n_pred", "shape": [1, "F0_LEN"] } |
| ], |
| "note": "F0_LEN = 2 * T_frame." |
| }, |
| { |
| "name": "har_source", |
| "package": "packages/har_source.mlpackage", |
| "precision": "fp32", |
| "compute_units": "CPU_AND_GPU", |
| "inputs": [ |
| { "name": "f0", "shape": [1, "F0_LEN"] } |
| ], |
| "outputs": [ |
| { "name": "har", "shape": [1, 1, "HAR_LEN"] } |
| ], |
| "note": "HAR_LEN = 300 * F0_LEN. fp32 required: computes sin(2π · cumsum(f0)) at audio rate; fp16 cumsum drifts ~10 bits over 74400 samples and produces audible phase distortion." |
| }, |
| { |
| "name": "decoder_pre", |
| "package": "packages/decoder_pre_fp16.mlpackage", |
| "precision": "fp16", |
| "compute_units": "CPU_AND_NE", |
| "inputs": [ |
| { "name": "asr", "shape": [1, 512, "T_frame"] }, |
| { "name": "f0_pred", "shape": [1, "F0_LEN"] }, |
| { "name": "n_pred", "shape": [1, "F0_LEN"] }, |
| { "name": "ref", "shape": [1, 128] } |
| ], |
| "outputs": [ |
| { "name": "x_pre", "shape": [1, 512, "T_frame2"] } |
| ], |
| "note": "T_frame2 = 2 * T_frame. Splits the HiFi-GAN decoder: pre-stage (AdaIN encode/decode + F0/N convs) is ANE-clean." |
| }, |
| { |
| "name": "decoder_upsample", |
| "package": "packages/decoder_upsample_fp16.mlpackage", |
| "precision": "fp16", |
| "compute_units": "CPU_ONLY", |
| "inputs": [ |
| { "name": "x_pre", "shape": [1, 512, "T_frame2"] }, |
| { "name": "ref", "shape": [1, 128] }, |
| { "name": "har_source", "shape": [1, 1, "HAR_LEN"] } |
| ], |
| "outputs": [ |
| { "name": "audio", "shape": [1, 1, "AUDIO_LEN"] } |
| ], |
| "note": "HiFi-GAN Generator (ConvTranspose1d ups stack). ANE compile fails (ANECCompile() FAILED), CPU_ONLY is the most predictable. Tail-trim 50 samples." |
| } |
| ], |
| "pipeline_order": [ |
| "text_encoder", |
| "bert", |
| "ref_encoder", |
| "diffusion_unet (×N steps × 2 dispatches under ADPM2)", |
| "duration_predictor", |
| "f0n_predictor", |
| "har_source", |
| "decoder_pre", |
| "decoder_upsample" |
| ], |
| "non_coreml_pipeline_steps": [ |
| "espeak-ng phonemize + StyleTTS2 TextCleaner tokenize", |
| "Karras sigma schedule (CPU)", |
| "ADPM2 step loop (5 steps default; each step = 2 diffusion_unet dispatches + RNG noise add)", |
| "Style blend: ref = α · s_pred[:, :128] + (1-α) · ref_s[:, :128]; s = β · s_pred[:, 128:] + (1-β) · ref_s[:, 128:]", |
| "Reference mel: librosa.load(sr=24000) → librosa.effects.trim(top_db=30) → mel(n_fft=2048, hop=300, win=1200, n_mels=80, fmin=0, fmax=8000)", |
| "pred_aln_trg construction from rounded predicted durations (data-dependent)", |
| "en/asr matmul: en = d.transpose(-1,-2) @ pred_aln_trg; asr = t_en @ pred_aln_trg", |
| "HiFi-GAN tail shift: roll asr/en right by one frame, repeat first frame" |
| ], |
| "totals": { |
| "n_stages": 9, |
| "disk_size_mb": 258, |
| "warm_predict_ms_typical": 390, |
| "rtfx_typical": 9.4, |
| "cold_start_s_typical": 13, |
| "cold_start_breakdown": { |
| "anecompiler_first_call": "12s (Apple ANE compilation cache miss)", |
| "fp16_load": "~1s warm" |
| } |
| }, |
| "voices": { |
| "directory": "voices/", |
| "type": "zero-shot reference clips (any 3-10s mono 24 kHz WAV; the model copies timbre + prosody)", |
| "samples": [ |
| {"file": "Yinghao.wav", "lang": "en", "note": "neutral male"}, |
| {"file": "Nima.wav", "lang": "en", "note": "neutral male"}, |
| {"file": "Gavin.wav", "lang": "en", "note": "neutral male"}, |
| {"file": "Vinay.wav", "lang": "en", "note": "neutral male"}, |
| {"file": "amused.wav", "lang": "en", "note": "amused emotion"}, |
| {"file": "anger.wav", "lang": "en", "note": "angry emotion"}, |
| {"file": "disgusted.wav","lang": "en", "note": "disgusted emotion"}, |
| {"file": "sleepy.wav", "lang": "en", "note": "sleepy emotion"}, |
| {"file": "696_92939_000016_000006.wav", "lang": "en", "note": "LibriTTS sample, default reference"}, |
| {"file": "1221-135767-0014.wav", "lang": "en", "note": "LibriTTS sample"}, |
| {"file": "1789_142896_000022_000005.wav","lang":"en", "note": "LibriTTS sample"}, |
| {"file": "4077-13754-0000.wav", "lang": "en", "note": "LibriTTS sample"}, |
| {"file": "5639-40744-0020.wav", "lang": "en", "note": "LibriTTS sample"}, |
| {"file": "908-157963-0027.wav", "lang": "en", "note": "LibriTTS sample"}, |
| {"file": "3.wav", "lang": "en", "note": "misc reference"}, |
| {"file": "4.wav", "lang": "en", "note": "misc reference"}, |
| {"file": "5.wav", "lang": "en", "note": "misc reference"} |
| ] |
| }, |
| "samples": { |
| "directory": "samples/", |
| "files": [ |
| {"file": "sample_swift.wav", "text": "Hello, this is StyleTTS 2.", "voice": "696_92939_000016_000006.wav", "produced_by": "Swift CoreML driver", "duration_s": 3.02}, |
| {"file": "sample_python.wav", "text": "StyleTTS 2 is a text to speech model.", "voice": "696_92939_000016_000006.wav", "produced_by": "Python CoreML pipeline (coreml/inference.py)"} |
| ] |
| }, |
| "platform_requirements": { |
| "macos_min": "14.0", |
| "ios_min": "17.0 (mlprogram macOS15 deployment target — verify on iOS)", |
| "deployment_target": "macOS15", |
| "hardware": "Apple Silicon recommended" |
| } |
| } |
|
|
