{ "model_name": "styletts2-libritts-coreml", "version": "1.0.0", "base_model": "yl4579/StyleTTS2 LibriTTS checkpoint (epoch 20, 2nd-stage)", "sample_rate": 24000, "frame_hop": 300, "phoneme_vocab": "espeak-ng en-us IPA + StyleTTS2 TextCleaner", "limits": { "max_phonemes": 57, "note": "bert and diffusion_unet stages have a fixed token axis of 57 (CoreML CPU MLProgram backend rejects RangeDim on these graphs). Inputs producing more than 57 phonemes will fail until token-bucketed packages are added." }, "stages": [ { "name": "text_encoder", "package": "packages/text_encoder_fp16.mlpackage", "precision": "fp16", "compute_units": "CPU_ONLY", "inputs": [ { "name": "tokens", "shape": [1, "T_token"], "dtype": "int32", "range": [1, 512] }, { "name": "input_lengths", "shape": [1], "dtype": "int32" }, { "name": "text_mask", "shape": [1, "T_token"], "dtype": "float32" } ], "outputs": [ { "name": "t_en", "shape": [1, 512, "T_token"], "dtype": "float32" } ] }, { "name": "bert", "package": "packages/bert_fp16.mlpackage", "precision": "fp16", "compute_units": "CPU_AND_NE", "fixed_token_axis": 57, "inputs": [ { "name": "tokens", "shape": [1, 57], "dtype": "int32" }, { "name": "attention_mask", "shape": [1, 57], "dtype": "int32" } ], "outputs": [ { "name": "bert_dur", "shape": [1, 57, 768] }, { "name": "d_en", "shape": [1, 512, 57] } ] }, { "name": "ref_encoder", "package": "packages/ref_encoder_fp16.mlpackage", "precision": "fp16", "compute_units": "CPU_AND_NE", "inputs": [ { "name": "mel", "shape": [1, 1, 80, "T_mel"], "dtype": "float32", "note": "24 kHz mel spectrogram of reference audio. n_fft=2048, hop=300, win=1200, n_mels=80." } ], "outputs": [ { "name": "ref_s", "shape": [1, 256], "dtype": "float32", "note": "Style embedding. ref_s[:, :128] is reference timbre, ref_s[:, 128:] is reference prosody." } ] }, { "name": "diffusion_unet", "package": "packages/diffusion_unet_fp16.mlpackage", "precision": "fp16", "compute_units": "CPU_AND_NE", "fixed_token_axis": 57, "inputs": [ { "name": "x_noisy", "shape": [1, 1, 256] }, { "name": "sigma", "shape": [1] }, { "name": "embedding", "shape": [1, 57, 768] }, { "name": "features", "shape": [1, 256] } ], "outputs": [ { "name": "x_denoised", "shape": [1, 1, 256] } ], "note": "Called num_steps × 2 dispatches per utterance under ADPM2 sampler. Use Karras sigmas (sigma_min=0.0001, sigma_max=3.0, rho_schedule=9.0). 5 steps default." }, { "name": "duration_predictor", "package": "packages/duration_predictor_fp16.mlpackage", "precision": "fp16", "compute_units": "CPU_ONLY", "inputs": [ { "name": "d_en", "shape": [1, 512, "T_token"] }, { "name": "s", "shape": [1, 128] }, { "name": "text_mask", "shape": [1, "T_token"] } ], "outputs": [ { "name": "d", "shape": [1, "T_token", 640] }, { "name": "duration_logits", "shape": [1, "T_token", 50] } ] }, { "name": "f0n_predictor", "package": "packages/f0n_predictor_fp16.mlpackage", "precision": "fp16", "compute_units": "CPU_AND_NE", "inputs": [ { "name": "en", "shape": [1, 640, "T_frame"] }, { "name": "s", "shape": [1, 128] } ], "outputs": [ { "name": "f0_pred", "shape": [1, "F0_LEN"] }, { "name": "n_pred", "shape": [1, "F0_LEN"] } ], "note": "F0_LEN = 2 * T_frame." }, { "name": "har_source", "package": "packages/har_source.mlpackage", "precision": "fp32", "compute_units": "CPU_AND_GPU", "inputs": [ { "name": "f0", "shape": [1, "F0_LEN"] } ], "outputs": [ { "name": "har", "shape": [1, 1, "HAR_LEN"] } ], "note": "HAR_LEN = 300 * F0_LEN. fp32 required: computes sin(2π · cumsum(f0)) at audio rate; fp16 cumsum drifts ~10 bits over 74400 samples and produces audible phase distortion." }, { "name": "decoder_pre", "package": "packages/decoder_pre_fp16.mlpackage", "precision": "fp16", "compute_units": "CPU_AND_NE", "inputs": [ { "name": "asr", "shape": [1, 512, "T_frame"] }, { "name": "f0_pred", "shape": [1, "F0_LEN"] }, { "name": "n_pred", "shape": [1, "F0_LEN"] }, { "name": "ref", "shape": [1, 128] } ], "outputs": [ { "name": "x_pre", "shape": [1, 512, "T_frame2"] } ], "note": "T_frame2 = 2 * T_frame. Splits the HiFi-GAN decoder: pre-stage (AdaIN encode/decode + F0/N convs) is ANE-clean." }, { "name": "decoder_upsample", "package": "packages/decoder_upsample_fp16.mlpackage", "precision": "fp16", "compute_units": "CPU_ONLY", "inputs": [ { "name": "x_pre", "shape": [1, 512, "T_frame2"] }, { "name": "ref", "shape": [1, 128] }, { "name": "har_source", "shape": [1, 1, "HAR_LEN"] } ], "outputs": [ { "name": "audio", "shape": [1, 1, "AUDIO_LEN"] } ], "note": "HiFi-GAN Generator (ConvTranspose1d ups stack). ANE compile fails (ANECCompile() FAILED), CPU_ONLY is the most predictable. Tail-trim 50 samples." } ], "pipeline_order": [ "text_encoder", "bert", "ref_encoder", "diffusion_unet (×N steps × 2 dispatches under ADPM2)", "duration_predictor", "f0n_predictor", "har_source", "decoder_pre", "decoder_upsample" ], "non_coreml_pipeline_steps": [ "espeak-ng phonemize + StyleTTS2 TextCleaner tokenize", "Karras sigma schedule (CPU)", "ADPM2 step loop (5 steps default; each step = 2 diffusion_unet dispatches + RNG noise add)", "Style blend: ref = α · s_pred[:, :128] + (1-α) · ref_s[:, :128]; s = β · s_pred[:, 128:] + (1-β) · ref_s[:, 128:]", "Reference mel: librosa.load(sr=24000) → librosa.effects.trim(top_db=30) → mel(n_fft=2048, hop=300, win=1200, n_mels=80, fmin=0, fmax=8000)", "pred_aln_trg construction from rounded predicted durations (data-dependent)", "en/asr matmul: en = d.transpose(-1,-2) @ pred_aln_trg; asr = t_en @ pred_aln_trg", "HiFi-GAN tail shift: roll asr/en right by one frame, repeat first frame" ], "totals": { "n_stages": 9, "disk_size_mb": 258, "warm_predict_ms_typical": 390, "rtfx_typical": 9.4, "cold_start_s_typical": 13, "cold_start_breakdown": { "anecompiler_first_call": "12s (Apple ANE compilation cache miss)", "fp16_load": "~1s warm" } }, "voices": { "directory": "voices/", "type": "zero-shot reference clips (any 3-10s mono 24 kHz WAV; the model copies timbre + prosody)", "samples": [ {"file": "Yinghao.wav", "lang": "en", "note": "neutral male"}, {"file": "Nima.wav", "lang": "en", "note": "neutral male"}, {"file": "Gavin.wav", "lang": "en", "note": "neutral male"}, {"file": "Vinay.wav", "lang": "en", "note": "neutral male"}, {"file": "amused.wav", "lang": "en", "note": "amused emotion"}, {"file": "anger.wav", "lang": "en", "note": "angry emotion"}, {"file": "disgusted.wav","lang": "en", "note": "disgusted emotion"}, {"file": "sleepy.wav", "lang": "en", "note": "sleepy emotion"}, {"file": "696_92939_000016_000006.wav", "lang": "en", "note": "LibriTTS sample, default reference"}, {"file": "1221-135767-0014.wav", "lang": "en", "note": "LibriTTS sample"}, {"file": "1789_142896_000022_000005.wav","lang":"en", "note": "LibriTTS sample"}, {"file": "4077-13754-0000.wav", "lang": "en", "note": "LibriTTS sample"}, {"file": "5639-40744-0020.wav", "lang": "en", "note": "LibriTTS sample"}, {"file": "908-157963-0027.wav", "lang": "en", "note": "LibriTTS sample"}, {"file": "3.wav", "lang": "en", "note": "misc reference"}, {"file": "4.wav", "lang": "en", "note": "misc reference"}, {"file": "5.wav", "lang": "en", "note": "misc reference"} ] }, "samples": { "directory": "samples/", "files": [ {"file": "sample_swift.wav", "text": "Hello, this is StyleTTS 2.", "voice": "696_92939_000016_000006.wav", "produced_by": "Swift CoreML driver", "duration_s": 3.02}, {"file": "sample_python.wav", "text": "StyleTTS 2 is a text to speech model.", "voice": "696_92939_000016_000006.wav", "produced_by": "Python CoreML pipeline (coreml/inference.py)"} ] }, "platform_requirements": { "macos_min": "14.0", "ios_min": "17.0 (mlprogram macOS15 deployment target — verify on iOS)", "deployment_target": "macOS15", "hardware": "Apple Silicon recommended" } }