#include "tts.h" #include #include #include #include #include #include #include #include "ggml-alloc.h" #include "ggml-backend.h" #include "ggml-cpu.h" #include "ggml.h" #include "gguf.h" #include "synth.h" namespace { // 64-bit file seek (GGUF files can pass 2 GB in other models) int seek64(FILE* f, int64_t off) { #ifdef _WIN32 return _fseeki64(f, off, SEEK_SET); #else return fseeko(f, off_t(off), SEEK_SET); #endif } struct Hp { int channels, heads, feedforward, kernel, enc_layers, pro_layers, ac_layers, pred_channels, max_tokens, max_frames; int vocab, hop, sample_rate; float f0_min, f0_max, pitch_ref; }; class Engine : public TtsModel { public: ~Engine() override { if (galloc_) ggml_gallocr_free(galloc_); if (buffer_) ggml_backend_buffer_free(buffer_); if (weights_) ggml_free(weights_); if (gguf_) gguf_free(gguf_); if (backend_) ggml_backend_free(backend_); } void load_file(const std::string& path, int threads) { gguf_init_params params{true, &weights_}; gguf_ = gguf_init_from_file(path.c_str(), params); if (!gguf_) throw std::runtime_error("cannot read GGUF " + path); backend_ = ggml_backend_cpu_init(); ggml_backend_cpu_set_n_threads(backend_, threads); buffer_ = ggml_backend_alloc_ctx_tensors(weights_, backend_); FILE* f = std::fopen(path.c_str(), "rb"); if (!f) throw std::runtime_error("cannot open " + path); std::vector tmp; const size_t base = gguf_get_data_offset(gguf_); for (int64_t i = 0; i < gguf_get_n_tensors(gguf_); ++i) { ggml_tensor* t = ggml_get_tensor(weights_, gguf_get_tensor_name(gguf_, i)); const size_t n = ggml_nbytes(t); tmp.resize(n); seek64(f, int64_t(base + gguf_get_tensor_offset(gguf_, i))); if (std::fread(tmp.data(), 1, n, f) != n) { std::fclose(f); throw std::runtime_error("short read"); } ggml_backend_tensor_set(t, tmp.data(), 0, n); weight_bytes_ += n; } std::fclose(f); auto u32 = [&](const char* k) { return int(gguf_get_val_u32(gguf_, key(k))); }; auto f32 = [&](const char* k) { return gguf_get_val_f32(gguf_, key(k)); }; hp_ = {u32("channels"), u32("heads"), u32("feedforward"), u32("kernel"), u32("encoder_layers"), u32("prosody_layers"), u32("acoustic_layers"), u32("predictor_channels"), u32("max_tokens"), u32("max_frames"), u32("vocab_size"), u32("hop"), u32("sample_rate"), f32("f0_min"), f32("f0_max"), f32("pitch_reference_hz")}; styles_ = strings("styles"); symbols_ = strings("symbols"); phone_.assign(hp_.vocab, 0); vowel_.assign(hp_.vocab, 0); for (int id : ints("phone_ids")) phone_.at(id) = 1; for (int id : ints("vowel_ids")) vowel_.at(id) = 1; galloc_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend_)); const int64_t dec = gguf_find_key(gguf_, "tinyvoca.decoder"); flow_ = dec >= 0 && std::string(gguf_get_val_str(gguf_, dec)) == "flow"; if (flow_) { // flow models: front -> flow-matching mel decoder -> Vocos (scripts/tts_hi_flow_to_gguf.py) fp_.layers = u32("flow.layers"); fp_.n_mels = u32("flow.n_mels"); fp_.steps = u32("flow.steps"); fp_.temperature = f32("flow.temperature"); fp_.vocos_layers = u32("vocos.layers"); fp_.n_fft = u32("vocos.n_fft"); fp_.vocos_hop = u32("vocos.hop"); const int64_t g = gguf_find_key(gguf_, "tinyvoca.flow.guidance"); fp_.guidance = g >= 0 ? gguf_get_val_f32(gguf_, g) : 0.0f; const int64_t jk = gguf_find_key(gguf_, "tinyvoca.flow.jumps"); // distilled few-step model fp_.jumps = jk >= 0 ? int(gguf_get_val_u32(gguf_, jk)) : 0; const int64_t pf = gguf_find_key(gguf_, "tinyvoca.prosody_flow.steps"); if (pf >= 0) { fp_.prosody = true; fp_.dur_layers = u32("prosody_flow.duration_layers"); fp_.pe_layers = u32("prosody_flow.frame_layers"); fp_.prosody_steps = u32("prosody_flow.steps"); fp_.prosody_temperature = f32("prosody_flow.temperature"); } return; } sc_.sample_rate = hp_.sample_rate; sc_.hop = hp_.hop; sc_.synth_fft = u32("synth_fft"); sc_.synth_hop = u32("synth_hop"); sc_.noise_fft = u32("noise_fft"); sc_.noise_hop = u32("noise_hop"); sc_.cep_causal = u32("cep_causal"); sc_.cep_anti = u32("cep_anti"); sc_.noise_cep = u32("noise_cep"); sc_.harmonic_limit = float(std::round(double(f32("harmonic_limit")) * 1e6) / 1e6); sc_.causal_lifter = read_vec("vocoder.causal_lifter"); sc_.anti_lifter = read_vec("vocoder.anti_lifter"); sc_.noise_lifter = read_vec("vocoder.noise_lifter"); } const std::vector& styles() const override { return styles_; } const std::vector& symbols() const override { return symbols_; } int sample_rate() const override { return hp_.sample_rate; } int max_tokens() const override { return hp_.max_tokens; } size_t weight_bytes() const override { return weight_bytes_; } std::vector synthesize(const std::vector& ids, const std::vector& stress, const TtsOptions& opt, TtsDebug* dbg) override { const int N = int(ids.size()); if (N < 1 || N > hp_.max_tokens || stress.size() != ids.size()) throw std::runtime_error("invalid token input"); if (opt.style < 0 || opt.style >= int(styles_.size())) throw std::runtime_error("invalid style"); if (!(opt.speed >= 0.25f && opt.speed <= 4.0f)) throw std::runtime_error("speed must be between 0.25 and 4"); const int C = hp_.channels; // ---- stage 1: embeddings -> encoder -> duration predictor std::vector hidden(size_t(C) * N), dur(size_t(2) * N); { Graph g(this); ggml_tensor* t_ids = g.input_i32(N), * t_st = g.input_i32(N), * t_sty = g.input_i32(1), * t_pos = g.input_i32(N); ggml_tensor* x = ggml_add(g.ctx, ggml_get_rows(g.ctx, W("tokens.weight"), t_ids), ggml_get_rows(g.ctx, W("stress.weight"), t_st)); x = ggml_add(g.ctx, x, ggml_get_rows(g.ctx, W("voices.weight"), t_sty)); ggml_tensor* h = stack(g.ctx, x, "encoder", hp_.enc_layers, t_pos, N); ggml_tensor* d = predictor(g.ctx, h, N); g.output(h); g.output(d); g.alloc(); g.set(t_ids, ids.data()); g.set(t_st, stress.data()); g.set(t_sty, &opt.style); set_positions(g, t_pos, N); g.compute(); g.get(h, hidden.data()); g.get(d, dur.data()); } // ---- durations (predict_durations): log(1 + frames) regression + "has frames" logit for pause tokens; // prosody-flow models SAMPLE log(1 + frames) with the duration flow (ProsodyFlows.sample_durations) std::vector log_frames(N); if (flow_ && fp_.prosody) { log_frames = sample_flow("pflow_dur", fp_.dur_layers, 1, hidden, N, fp_.prosody_steps, fp_.prosody_temperature, 0.0f, opt.duration_noise, opt.seed * 3 + 1); } else { for (int i = 0; i < N; ++i) log_frames[i] = dur[size_t(i) * 2]; } std::vector durations(N); int T = 0; const float cap = std::log1p(400.0f); for (int i = 0; i < N; ++i) { const float v = std::min(std::max(log_frames[i], 0.0f), cap); int frames = int(std::nearbyint(std::expm1(v) / opt.speed)); frames = std::max(frames, 1); const bool present = phone_.at(ids[i]) || dur[size_t(i) * 2 + 1] > 0.0f; durations[i] = present ? frames : 0; T += durations[i]; } if (T < 1) throw std::runtime_error("utterance has no frames"); if (T > hp_.max_frames) throw UtteranceTooLong("utterance exceeds max_frames; split the input"); // ---- expand: token index per frame + position-in-token features std::vector owner(T); std::vector feats(size_t(3) * T); { int f = 0; for (int i = 0; i < N; ++i) { for (int k = 0; k < durations[i]; ++k, ++f) { const float owned = float(std::max(durations[i], 1)); const float rel = (float(k) + 0.5f) / owned; owner[f] = i; feats[size_t(f) * 3 + 0] = rel; feats[size_t(f) * 3 + 1] = 1.0f - rel; feats[size_t(f) * 3 + 2] = std::log1p(owned) / 4.0f; } } } // ---- stage 2: frames -> prosody stack -> pitch / voicing / energy std::vector phid(size_t(C) * T), pros(size_t(3) * T); { Graph g(this); ggml_tensor* t_hid = g.input_f32(C, N), * t_idx = g.input_i32(T), * t_feat = g.input_f32(3, T); ggml_tensor* t_sty = g.input_i32(1), * t_pos = g.input_i32(T); ggml_tensor* x = ggml_get_rows(g.ctx, t_hid, t_idx); x = ggml_add(g.ctx, x, linear(g.ctx, t_feat, "frame_position")); x = ggml_add(g.ctx, x, ggml_get_rows(g.ctx, W("frame_voices.weight"), t_sty)); ggml_tensor* ph = stack(g.ctx, x, "prosody_stack", hp_.pro_layers, t_pos, T); ggml_tensor* pr = linear(g.ctx, ph, "prosody_head"); g.output(ph); g.output(pr); g.alloc(); g.set(t_hid, hidden.data()); g.set(t_idx, owner.data()); g.set(t_feat, feats.data()); g.set(t_sty, &opt.style); set_positions(g, t_pos, T); g.compute(); g.get(ph, phid.data()); g.get(pr, pros.data()); } std::vector pros_raw = pros; if (flow_ && fp_.prosody) { // sampled pitch and energy; voicing stays the regression head's const std::vector pe = sample_flow("pflow_pe", fp_.pe_layers, 2, phid, T, fp_.prosody_steps, fp_.prosody_temperature, 0.0f, opt.pitch_energy_noise, opt.seed * 3 + 2); for (int f = 0; f < T; ++f) { pros[size_t(f) * 3] = pe[size_t(f) * 2]; pros[size_t(f) * 3 + 2] = pe[size_t(f) * 2 + 1]; } } // vowels are voiced by definition (voiced_vowels=True) for (int f = 0; f < T; ++f) if (vowel_.at(ids[owner[f]])) pros[size_t(f) * 3 + 1] = std::max(pros[size_t(f) * 3 + 1], 2.0f); // soft octave limiter above the voiced median (torch.nanmedian = lower middle value) if (opt.octave_fix > 0) { std::vector v; for (int f = 0; f < T; ++f) if (pros[size_t(f) * 3 + 1] > 0) v.push_back(pros[size_t(f) * 3]); float median = 0.0f; if (!v.empty()) { std::nth_element(v.begin(), v.begin() + (v.size() - 1) / 2, v.end()); median = v[(v.size() - 1) / 2]; } const float limit = std::log(opt.octave_fix); for (int f = 0; f < T; ++f) { float& p = pros[size_t(f) * 3]; const float excess = p - median; if (excess > 0) p = median + limit * std::tanh(excess / limit); } } // hard prosody (what the decoder is conditioned on) and the oscillator pitch std::vector cond(size_t(3) * T), f0(T); for (int f = 0; f < T; ++f) { const float* p = &pros[size_t(f) * 3]; cond[size_t(f) * 3 + 0] = std::min(std::max(p[0], -2.0f), 1.5f); cond[size_t(f) * 3 + 1] = p[1] > 0 ? 1.0f : 0.0f; cond[size_t(f) * 3 + 2] = std::min(std::max(p[2], -4.5f), 3.0f); f0[f] = std::min(std::max(hp_.pitch_ref * std::exp(p[0]), hp_.f0_min), hp_.f0_max); } if (flow_) { std::vector wave = flow_decode(phid, cond, T, opt, dbg); if (dbg) { dbg->encoder = hidden; dbg->duration_raw = dur; dbg->durations = durations; dbg->frames = T; dbg->prosody_hidden = phid; dbg->prosody_raw = pros_raw; dbg->prosody = pros; dbg->f0 = f0; } return wave; } // ---- stage 3: acoustic stack -> synthesizer heads const int hw = sc_.cep_causal + sc_.cep_anti + hp_.hop / sc_.synth_hop, nw = sc_.noise_cep + hp_.hop / sc_.noise_hop; std::vector ac(size_t(C) * T), harm(size_t(hw) * T), nh(size_t(nw) * T); { Graph g(this); ggml_tensor* t_ph = g.input_f32(C, T), * t_cond = g.input_f32(3, T), * t_pos = g.input_i32(T); ggml_tensor* x = ggml_add(g.ctx, t_ph, linear(g.ctx, t_cond, "prosody_projection")); ggml_tensor* a = stack(g.ctx, x, "acoustic_stack", hp_.ac_layers, t_pos, T); ggml_tensor* hh = linear(g.ctx, a, "vocoder.harmonic"); ggml_tensor* nn = linear(g.ctx, a, "vocoder.noise"); g.output(a); g.output(hh); g.output(nn); g.alloc(); g.set(t_ph, phid.data()); g.set(t_cond, cond.data()); set_positions(g, t_pos, T); g.compute(); g.get(a, ac.data()); g.get(hh, harm.data()); g.get(nn, nh.data()); } // ---- source-filter synthesis std::vector noise; const float* nz = opt.noise; if (!nz) { noise.resize(size_t(T) * hp_.hop); std::mt19937_64 rng(opt.seed); std::normal_distribution normal(0.0f, 1.0f); for (float& s : noise) s = normal(rng); nz = noise.data(); } std::vector wave = synthesize_wave(sc_, harm.data(), nh.data(), f0.data(), T, nz); if (dbg) { dbg->encoder = hidden; dbg->duration_raw = dur; dbg->durations = durations; dbg->frames = T; dbg->prosody_hidden = phid; dbg->prosody_raw = pros_raw; dbg->prosody = pros; dbg->acoustic = ac; dbg->f0 = f0; dbg->harmonic_head = harm; dbg->noise_head = nh; } return wave; } private: struct FlowHp { int layers = 10, n_mels = 100, steps = 16, vocos_layers = 8, n_fft = 1024, vocos_hop = 256; float temperature = 1.0f, guidance = 0.0f; int jumps = 0; // > 0: distilled student, sampled with this many one-pass jumps bool prosody = false; // prosody flows (tiny_voca/prosody_flow.py) int dur_layers = 1, pe_layers = 2, prosody_steps = 8; float prosody_temperature = 0.8f; }; // FlowDecoder.sample (midpoint solver, uniform steps, optional classifier-free guidance) for the decoder stored // under `p` ("flow", "pflow_dur", "pflow_pe"): cond [T][CD] -> denormalised x [T][M]. noise = the starting point // (temperature applied); without it x0 = temperature * N(0, 1) from `seed`. // jumps > 0 (distilled students, FlowDecoder.sample_jumps): `jumps` equal jumps x += d * u(x, t, d), one pass each, // with the jump-size embedding added to the time embedding; steps and guidance are then unused. std::vector sample_flow(const std::string& p, int layers, int M, const std::vector& cond, int T, int steps, float temperature, float guidance, const float* noise, uint64_t seed, int jumps = 0) { if (jumps > 0) guidance = 0.0f; const int CD = int(cond.size() / size_t(T)); const int Wd = int(W(p + ".input.bias")->ne[0]); std::vector x(size_t(M) * T); if (noise) { std::copy(noise, noise + x.size(), x.begin()); } else { std::mt19937_64 rng(seed); std::normal_distribution normal(0.0f, 1.0f); for (float& v : x) v = temperature * normal(rng); } // one velocity graph, built once and run 2 x steps times (x 2 with guidance) with new inputs std::vector v(x.size()), vu(guidance > 0 ? x.size() : 0), xm(x.size()), temb(Wd); const std::vector zeros(guidance > 0 ? cond.size() : 0, 0.0f); { Graph g(this); ggml_tensor* t_x = g.input_f32(M, T), * t_c = g.input_f32(CD, T), * t_t = g.input_f32(Wd, 1), * t_pos = g.input_i32(T); ggml_tensor* h = ggml_add(g.ctx, ggml_add(g.ctx, ggml_mul_mat(g.ctx, W(p + ".input_x.weight"), t_x), ggml_mul_mat(g.ctx, W(p + ".input_c.weight"), t_c)), W(p + ".input.bias")); ggml_tensor* te = linear(g.ctx, ggml_silu(g.ctx, linear(g.ctx, t_t, p + ".time.0")), p + ".time.2"); // [W, 1] ggml_tensor* t_d = nullptr; if (jumps > 0) { t_d = g.input_f32(Wd, 1); te = ggml_add(g.ctx, te, linear(g.ctx, ggml_silu(g.ctx, linear(g.ctx, t_d, p + ".jump.0")), p + ".jump.2")); } for (int i = 0; i < layers; ++i) h = block(g.ctx, ggml_add(g.ctx, h, linear(g.ctx, te, p + ".time_mod." + std::to_string(i))), p + ".blocks." + std::to_string(i), t_pos, T); ggml_tensor* out = linear(g.ctx, layernorm(g.ctx, h, p + ".norm"), p + ".output"); // [M, T] g.output(out); g.alloc(); std::vector demb(jumps > 0 ? Wd : 0); auto embed = [&](float t, std::vector& e) { // FlowDecoder.time_embedding const int half = Wd / 2; for (int i = 0; i < half; ++i) { const float freq = std::exp(-std::log(10000.0f) * float(i) / float(half)); const float angle = 1000.0f * t * freq; e[i] = std::sin(angle); e[half + i] = std::cos(angle); } }; float jump = 0.0f; auto run = [&](const std::vector& xin, const std::vector& cin, float t, std::vector& vout) { embed(t, temb); if (t_d) { embed(jump, demb); g.set(t_d, demb.data()); } g.set(t_x, xin.data()); g.set(t_c, cin.data()); g.set(t_t, temb.data()); set_positions(g, t_pos, T); g.compute(); g.get(out, vout.data()); }; auto velocity = [&](const std::vector& xin, float t, std::vector& vout) { run(xin, cond, t, vout); if (guidance > 0) { // v = v_cond + w (v_cond - v_uncond) run(xin, zeros, t, vu); for (size_t i = 0; i < vout.size(); ++i) vout[i] += guidance * (vout[i] - vu[i]); } }; if (jumps > 0) { const double d = 1.0 / jumps; jump = float(d); for (int k = 0; k < jumps; ++k) { velocity(x, float(k * d), v); for (size_t i = 0; i < x.size(); ++i) x[i] += float(d) * v[i]; } } else { for (int k = 0; k < steps; ++k) { const float t0 = float(k) / float(steps), t1 = float(k + 1) / float(steps); // torch.linspace(0, 1, steps + 1) const double dt = double(t1) - double(t0); velocity(x, t0, v); for (size_t i = 0; i < x.size(); ++i) xm[i] = x[i] + float(0.5 * dt) * v[i]; velocity(xm, float(double(t0) + 0.5 * dt), v); for (size_t i = 0; i < x.size(); ++i) x[i] += float(dt) * v[i]; } } } const std::vector mean = read_vec((p + ".mel_mean").c_str()), sd = read_vec((p + ".mel_std").c_str()); for (int f = 0; f < T; ++f) for (int m = 0; m < M; ++m) x[size_t(f) * M + m] = x[size_t(f) * M + m] * sd[m] + mean[m]; return x; } // The mel flow on [prosody-stack states, hard prosody], then Vocos. std::vector flow_decode(const std::vector& phid, const std::vector& cond3, int T, const TtsOptions& opt, TtsDebug* dbg) { const int C = hp_.channels, M = fp_.n_mels, CD = C + 3; const int steps = opt.flow_steps > 0 ? opt.flow_steps : fp_.steps; const float temperature = opt.temperature >= 0 ? opt.temperature : fp_.temperature; const float guidance = opt.guidance >= 0 ? opt.guidance : fp_.guidance; std::vector fc(size_t(CD) * T); for (int f = 0; f < T; ++f) { std::copy(phid.begin() + size_t(f) * C, phid.begin() + size_t(f + 1) * C, fc.begin() + size_t(f) * CD); std::copy(cond3.begin() + size_t(f) * 3, cond3.begin() + size_t(f + 1) * 3, fc.begin() + size_t(f) * CD + C); } // distilled students: --steps N means N jumps (default: the GGUF's) const int jumps = fp_.jumps > 0 ? (opt.flow_steps > 0 ? opt.flow_steps : fp_.jumps) : 0; const std::vector x = sample_flow("flow", fp_.layers, M, fc, T, steps, temperature, guidance, opt.noise, opt.seed, jumps); // Vocos gets the mel with its last frame repeated (T + 1 frames -> T * hop samples) std::vector mel(size_t(M) * (T + 1)); std::copy(x.begin(), x.end(), mel.begin()); std::copy(mel.begin() + size_t(T - 1) * M, mel.begin() + size_t(T) * M, mel.begin() + size_t(T) * M); if (dbg) { dbg->flow_cond = fc; dbg->mel.assign(mel.begin(), mel.begin() + size_t(M) * T); } const int F = T + 1, bins2 = fp_.n_fft + 2; std::vector head(size_t(bins2) * F); { Graph g(this); ggml_tensor* t_mel = g.input_f32(M, F); ggml_tensor* y = conv(g.ctx, t_mel, "vocos.backbone.embed", 7, F); y = layernorm(g.ctx, y, "vocos.backbone.norm", 1e-6f); for (int i = 0; i < fp_.vocos_layers; ++i) { const std::string p = "vocos.backbone.convnext." + std::to_string(i); ggml_tensor* z = layernorm(g.ctx, depthwise(g.ctx, y, p + ".dwconv", F), p + ".norm", 1e-6f); z = linear(g.ctx, ggml_gelu_erf(g.ctx, linear(g.ctx, z, p + ".pwconv1")), p + ".pwconv2"); y = ggml_add(g.ctx, y, ggml_mul(g.ctx, z, W(p + ".gamma"))); } ggml_tensor* o = linear(g.ctx, layernorm(g.ctx, y, "vocos.backbone.final_layer_norm", 1e-6f), "vocos.head.out"); g.output(o); g.alloc(); g.set(t_mel, mel.data()); g.compute(); g.get(o, head.data()); } return vocos_istft(head.data(), F, fp_.n_fft, fp_.vocos_hop); } // One compute graph: tensors in a metadata-only context, data allocated by the shared gallocr (reused between calls). struct Graph { Engine* e; ggml_context* ctx; ggml_cgraph* gf; std::vector outputs; explicit Graph(Engine* engine) : e(engine) { ggml_init_params p{ggml_tensor_overhead() * 8192 + ggml_graph_overhead_custom(8192, false), nullptr, true}; ctx = ggml_init(p); gf = ggml_new_graph_custom(ctx, 8192, false); } ~Graph() { ggml_free(ctx); } ggml_tensor* input_i32(int n) { auto t = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n); ggml_set_input(t); return t; } ggml_tensor* input_f32(int a, int b) { auto t = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, a, b); ggml_set_input(t); return t; } void output(ggml_tensor* t) { ggml_set_output(t); outputs.push_back(t); } void alloc() { for (auto t : outputs) ggml_build_forward_expand(gf, t); if (!ggml_gallocr_alloc_graph(e->galloc_, gf)) throw std::runtime_error("graph allocation failed"); } void set(ggml_tensor* t, const void* data) { ggml_backend_tensor_set(t, data, 0, ggml_nbytes(t)); } void get(ggml_tensor* t, void* data) { ggml_backend_tensor_get(t, data, 0, ggml_nbytes(t)); } void compute() { if (ggml_backend_graph_compute(e->backend_, gf) != GGML_STATUS_SUCCESS) throw std::runtime_error("compute failed"); } }; int64_t key(const char* k) const { const int64_t id = gguf_find_key(gguf_, (std::string("tinyvoca.") + k).c_str()); if (id < 0) throw std::runtime_error(std::string("missing GGUF key ") + k); return id; } std::vector strings(const char* k) const { const int64_t id = key(k); std::vector out; for (size_t i = 0; i < gguf_get_arr_n(gguf_, id); ++i) out.emplace_back(gguf_get_arr_str(gguf_, id, i)); return out; } std::vector ints(const char* k) const { const int64_t id = key(k); const size_t n = gguf_get_arr_n(gguf_, id); const enum gguf_type type = gguf_get_arr_type(gguf_, id); const void* d = gguf_get_arr_data(gguf_, id); std::vector out(n); for (size_t i = 0; i < n; ++i) { switch (type) { case GGUF_TYPE_INT32: out[i] = static_cast(d)[i]; break; case GGUF_TYPE_UINT32: out[i] = int(static_cast(d)[i]); break; case GGUF_TYPE_INT64: out[i] = int(static_cast(d)[i]); break; case GGUF_TYPE_UINT8: out[i] = static_cast(d)[i]; break; default: throw std::runtime_error(std::string("unexpected array type for ") + k); } } return out; } std::vector read_vec(const char* name) { ggml_tensor* t = W(name); std::vector v(ggml_nelements(t)); ggml_backend_tensor_get(t, v.data(), 0, ggml_nbytes(t)); return v; } ggml_tensor* W(const std::string& name) const { ggml_tensor* t = ggml_get_tensor(weights_, name.c_str()); if (!t) throw std::runtime_error("missing tensor " + name); return t; } void set_positions(Graph& g, ggml_tensor* t, int n) { std::vector pos(n); for (int i = 0; i < n; ++i) pos[i] = i; g.set(t, pos.data()); } ggml_tensor* linear(ggml_context* ctx, ggml_tensor* x, const std::string& p) const { return ggml_add(ctx, ggml_mul_mat(ctx, W(p + ".weight"), x), W(p + ".bias")); } ggml_tensor* layernorm(ggml_context* ctx, ggml_tensor* x, const std::string& p, float eps = 1e-5f) const { return ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, eps), W(p + ".weight")), W(p + ".bias")); } // depthwise Conv1d, kernel K, padding K/2 (cross-correlation as in PyTorch): sum over taps of shifted inputs ggml_tensor* depthwise(ggml_context* ctx, ggml_tensor* y, const std::string& p, int T) const { ggml_tensor* w = W(p + ".weight"); // [C, K]: tap k is row k const int C = int(w->ne[0]), K = int(w->ne[1]), P = K / 2; ggml_tensor* padded = ggml_pad_ext(ctx, y, 0, 0, P, P, 0, 0, 0, 0); ggml_tensor* out = nullptr; for (int k = 0; k < K; ++k) { ggml_tensor* v = ggml_view_2d(ctx, padded, C, T, padded->nb[1], size_t(k) * padded->nb[1]); ggml_tensor* term = ggml_mul(ctx, v, ggml_view_1d(ctx, w, C, size_t(k) * w->nb[1])); out = out ? ggml_add(ctx, out, term) : term; } return ggml_add(ctx, out, W(p + ".bias")); } ggml_tensor* block(ggml_context* ctx, ggml_tensor* x, const std::string& p, ggml_tensor* pos, int T) const { const int C = int(x->ne[0]), H = hp_.heads, D = C / H; // width from the input: flows may be narrower ggml_tensor* qkv = linear(ctx, layernorm(ctx, x, p + ".norm1"), p + ".attention.qkv"); // [3C, T] auto part = [&](int i) { ggml_tensor* v = ggml_view_3d(ctx, qkv, D, H, T, size_t(D) * sizeof(float), qkv->nb[1], size_t(i) * C * sizeof(float)); return ggml_cont(ctx, v); // [D, H, T] }; ggml_tensor* q = ggml_rope_ext(ctx, part(0), pos, nullptr, D, GGML_ROPE_TYPE_NORMAL, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); ggml_tensor* k = ggml_rope_ext(ctx, part(1), pos, nullptr, D, GGML_ROPE_TYPE_NORMAL, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); ggml_tensor* v = part(2); q = ggml_permute(ctx, q, 0, 2, 1, 3); // [D, T, H] k = ggml_permute(ctx, k, 0, 2, 1, 3); ggml_tensor* att; if (flash_) { // flash attention: never builds the T x T score matrix (memory stays linear in T) att = ggml_flash_attn_ext(ctx, q, k, ggml_permute(ctx, v, 0, 2, 1, 3), nullptr, 1.0f / std::sqrt(float(D)), 0.0f, 0.0f); att = ggml_reshape_2d(ctx, att, C, T); // result is [D, H, T] } else { ggml_tensor* kq = ggml_soft_max_ext(ctx, ggml_mul_mat(ctx, k, q), nullptr, 1.0f / std::sqrt(float(D)), 0.0f); // [Tk, Tq, H] ggml_tensor* vt = ggml_cont(ctx, ggml_permute(ctx, v, 1, 2, 0, 3)); // [T, D, H] att = ggml_cont_2d(ctx, ggml_permute(ctx, ggml_mul_mat(ctx, vt, kq), 0, 2, 1, 3), C, T); } att = linear(ctx, att, p + ".attention.out"); x = ggml_add(ctx, x, att); ggml_tensor* y = depthwise(ctx, layernorm(ctx, x, p + ".norm2"), p + ".depthwise", T); y = linear(ctx, ggml_gelu_erf(ctx, linear(ctx, y, p + ".expand")), p + ".project"); return ggml_add(ctx, x, y); } ggml_tensor* stack(ggml_context* ctx, ggml_tensor* x, const std::string& p, int layers, ggml_tensor* pos, int T) const { for (int i = 0; i < layers; ++i) x = block(ctx, x, p + ".blocks." + std::to_string(i), pos, T); return layernorm(ctx, x, p + ".norm"); } // Conv1d kernel 3, padding 1: three taps as matrix multiplies on shifted inputs ggml_tensor* conv3(ggml_context* ctx, ggml_tensor* x, const std::string& p, int T) const { return conv(ctx, x, p, 3, T); } // Conv1d kernel K (odd), padding K/2: K taps "

.weight.k" as matrix multiplies on shifted inputs ggml_tensor* conv(ggml_context* ctx, ggml_tensor* x, const std::string& p, int K, int T) const { ggml_tensor* padded = ggml_pad_ext(ctx, x, 0, 0, K / 2, K / 2, 0, 0, 0, 0); ggml_tensor* out = nullptr; for (int k = 0; k < K; ++k) { ggml_tensor* v = ggml_view_2d(ctx, padded, x->ne[0], T, padded->nb[1], size_t(k) * padded->nb[1]); ggml_tensor* term = ggml_mul_mat(ctx, W(p + ".weight.k" + std::to_string(k)), v); out = out ? ggml_add(ctx, out, term) : term; } return ggml_add(ctx, out, W(p + ".bias")); } ggml_tensor* predictor(ggml_context* ctx, ggml_tensor* h, int N) const { ggml_tensor* x = layernorm(ctx, ggml_gelu_erf(ctx, conv3(ctx, h, "duration.conv1", N)), "duration.norm1"); x = layernorm(ctx, ggml_gelu_erf(ctx, conv3(ctx, x, "duration.conv2", N)), "duration.norm2"); return linear(ctx, x, "duration.out"); // [2, N] } Hp hp_{}; bool flow_ = false; FlowHp fp_; bool flash_ = std::getenv("TTS_NO_FLASH") == nullptr; SynthConfig sc_; std::vector styles_, symbols_; std::vector phone_, vowel_; gguf_context* gguf_ = nullptr; ggml_context* weights_ = nullptr; ggml_backend_t backend_ = nullptr; ggml_backend_buffer_t buffer_ = nullptr; ggml_gallocr_t galloc_ = nullptr; size_t weight_bytes_ = 0; }; } // namespace std::unique_ptr TtsModel::load(const std::string& gguf_path, int threads) { auto e = std::make_unique(); e->load_file(gguf_path, threads); return e; }