Download cpp/src/tts.cpp from RL10x/tiny-voca-hindi: direct link, hf CLI and curl.
- Browser
- Download file 32 kB
-
https://huggingface.co/RL10x/tiny-voca-hindi/resolve/main/cpp/src/tts.cpp
- Command line
-
hf download hf://RL10x/tiny-voca-hindi/cpp/src/tts.cpp
-
curl -L -o tts.cpp https://huggingface.co/RL10x/tiny-voca-hindi/resolve/main/cpp/src/tts.cpp
32 kB
| namespace { | |
| // 64-bit file seek (GGUF files can pass 2 GB in other models) | |
| int seek64(FILE* f, int64_t off) { | |
| return _fseeki64(f, off, SEEK_SET); | |
| return fseeko(f, off_t(off), SEEK_SET); | |
| } | |
| struct Hp { | |
| int channels, heads, feedforward, kernel, enc_layers, pro_layers, ac_layers, pred_channels, max_tokens, max_frames; | |
| int vocab, hop, sample_rate; | |
| float f0_min, f0_max, pitch_ref; | |
| }; | |
| class Engine : public TtsModel { | |
| public: | |
| ~Engine() override { | |
| if (galloc_) ggml_gallocr_free(galloc_); | |
| if (buffer_) ggml_backend_buffer_free(buffer_); | |
| if (weights_) ggml_free(weights_); | |
| if (gguf_) gguf_free(gguf_); | |
| if (backend_) ggml_backend_free(backend_); | |
| } | |
| void load_file(const std::string& path, int threads) { | |
| gguf_init_params params{true, &weights_}; | |
| gguf_ = gguf_init_from_file(path.c_str(), params); | |
| if (!gguf_) throw std::runtime_error("cannot read GGUF " + path); | |
| backend_ = ggml_backend_cpu_init(); | |
| ggml_backend_cpu_set_n_threads(backend_, threads); | |
| buffer_ = ggml_backend_alloc_ctx_tensors(weights_, backend_); | |
| FILE* f = std::fopen(path.c_str(), "rb"); | |
| if (!f) throw std::runtime_error("cannot open " + path); | |
| std::vector<uint8_t> tmp; | |
| const size_t base = gguf_get_data_offset(gguf_); | |
| for (int64_t i = 0; i < gguf_get_n_tensors(gguf_); ++i) { | |
| ggml_tensor* t = ggml_get_tensor(weights_, gguf_get_tensor_name(gguf_, i)); | |
| const size_t n = ggml_nbytes(t); | |
| tmp.resize(n); | |
| seek64(f, int64_t(base + gguf_get_tensor_offset(gguf_, i))); | |
| if (std::fread(tmp.data(), 1, n, f) != n) { std::fclose(f); throw std::runtime_error("short read"); } | |
| ggml_backend_tensor_set(t, tmp.data(), 0, n); | |
| weight_bytes_ += n; | |
| } | |
| std::fclose(f); | |
| auto u32 = [&](const char* k) { return int(gguf_get_val_u32(gguf_, key(k))); }; | |
| auto f32 = [&](const char* k) { return gguf_get_val_f32(gguf_, key(k)); }; | |
| hp_ = {u32("channels"), u32("heads"), u32("feedforward"), u32("kernel"), u32("encoder_layers"), u32("prosody_layers"), | |
| u32("acoustic_layers"), u32("predictor_channels"), u32("max_tokens"), u32("max_frames"), u32("vocab_size"), | |
| u32("hop"), u32("sample_rate"), f32("f0_min"), f32("f0_max"), f32("pitch_reference_hz")}; | |
| styles_ = strings("styles"); | |
| symbols_ = strings("symbols"); | |
| phone_.assign(hp_.vocab, 0); vowel_.assign(hp_.vocab, 0); | |
| for (int id : ints("phone_ids")) phone_.at(id) = 1; | |
| for (int id : ints("vowel_ids")) vowel_.at(id) = 1; | |
| galloc_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend_)); | |
| const int64_t dec = gguf_find_key(gguf_, "tinyvoca.decoder"); | |
| flow_ = dec >= 0 && std::string(gguf_get_val_str(gguf_, dec)) == "flow"; | |
| if (flow_) { | |
| // flow models: front -> flow-matching mel decoder -> Vocos (scripts/tts_hi_flow_to_gguf.py) | |
| fp_.layers = u32("flow.layers"); fp_.n_mels = u32("flow.n_mels"); fp_.steps = u32("flow.steps"); | |
| fp_.temperature = f32("flow.temperature"); fp_.vocos_layers = u32("vocos.layers"); | |
| fp_.n_fft = u32("vocos.n_fft"); fp_.vocos_hop = u32("vocos.hop"); | |
| const int64_t g = gguf_find_key(gguf_, "tinyvoca.flow.guidance"); | |
| fp_.guidance = g >= 0 ? gguf_get_val_f32(gguf_, g) : 0.0f; | |
| const int64_t jk = gguf_find_key(gguf_, "tinyvoca.flow.jumps"); // distilled few-step model | |
| fp_.jumps = jk >= 0 ? int(gguf_get_val_u32(gguf_, jk)) : 0; | |
| const int64_t pf = gguf_find_key(gguf_, "tinyvoca.prosody_flow.steps"); | |
| if (pf >= 0) { | |
| fp_.prosody = true; | |
| fp_.dur_layers = u32("prosody_flow.duration_layers"); fp_.pe_layers = u32("prosody_flow.frame_layers"); | |
| fp_.prosody_steps = u32("prosody_flow.steps"); fp_.prosody_temperature = f32("prosody_flow.temperature"); | |
| } | |
| return; | |
| } | |
| sc_.sample_rate = hp_.sample_rate; sc_.hop = hp_.hop; | |
| sc_.synth_fft = u32("synth_fft"); sc_.synth_hop = u32("synth_hop"); sc_.noise_fft = u32("noise_fft"); | |
| sc_.noise_hop = u32("noise_hop"); sc_.cep_causal = u32("cep_causal"); sc_.cep_anti = u32("cep_anti"); | |
| sc_.noise_cep = u32("noise_cep"); | |
| sc_.harmonic_limit = float(std::round(double(f32("harmonic_limit")) * 1e6) / 1e6); | |
| sc_.causal_lifter = read_vec("vocoder.causal_lifter"); | |
| sc_.anti_lifter = read_vec("vocoder.anti_lifter"); | |
| sc_.noise_lifter = read_vec("vocoder.noise_lifter"); | |
| } | |
| const std::vector<std::string>& styles() const override { return styles_; } | |
| const std::vector<std::string>& symbols() const override { return symbols_; } | |
| int sample_rate() const override { return hp_.sample_rate; } | |
| int max_tokens() const override { return hp_.max_tokens; } | |
| size_t weight_bytes() const override { return weight_bytes_; } | |
| std::vector<float> synthesize(const std::vector<int32_t>& ids, const std::vector<int32_t>& stress, const TtsOptions& opt, | |
| TtsDebug* dbg) override { | |
| const int N = int(ids.size()); | |
| if (N < 1 || N > hp_.max_tokens || stress.size() != ids.size()) throw std::runtime_error("invalid token input"); | |
| if (opt.style < 0 || opt.style >= int(styles_.size())) throw std::runtime_error("invalid style"); | |
| if (!(opt.speed >= 0.25f && opt.speed <= 4.0f)) throw std::runtime_error("speed must be between 0.25 and 4"); | |
| const int C = hp_.channels; | |
| // ---- stage 1: embeddings -> encoder -> duration predictor | |
| std::vector<float> hidden(size_t(C) * N), dur(size_t(2) * N); | |
| { | |
| Graph g(this); | |
| ggml_tensor* t_ids = g.input_i32(N), * t_st = g.input_i32(N), * t_sty = g.input_i32(1), * t_pos = g.input_i32(N); | |
| ggml_tensor* x = ggml_add(g.ctx, ggml_get_rows(g.ctx, W("tokens.weight"), t_ids), ggml_get_rows(g.ctx, W("stress.weight"), t_st)); | |
| x = ggml_add(g.ctx, x, ggml_get_rows(g.ctx, W("voices.weight"), t_sty)); | |
| ggml_tensor* h = stack(g.ctx, x, "encoder", hp_.enc_layers, t_pos, N); | |
| ggml_tensor* d = predictor(g.ctx, h, N); | |
| g.output(h); g.output(d); | |
| g.alloc(); | |
| g.set(t_ids, ids.data()); g.set(t_st, stress.data()); g.set(t_sty, &opt.style); set_positions(g, t_pos, N); | |
| g.compute(); | |
| g.get(h, hidden.data()); g.get(d, dur.data()); | |
| } | |
| // ---- durations (predict_durations): log(1 + frames) regression + "has frames" logit for pause tokens; | |
| // prosody-flow models SAMPLE log(1 + frames) with the duration flow (ProsodyFlows.sample_durations) | |
| std::vector<float> log_frames(N); | |
| if (flow_ && fp_.prosody) { | |
| log_frames = sample_flow("pflow_dur", fp_.dur_layers, 1, hidden, N, fp_.prosody_steps, fp_.prosody_temperature, 0.0f, | |
| opt.duration_noise, opt.seed * 3 + 1); | |
| } else { | |
| for (int i = 0; i < N; ++i) log_frames[i] = dur[size_t(i) * 2]; | |
| } | |
| std::vector<int32_t> durations(N); | |
| int T = 0; | |
| const float cap = std::log1p(400.0f); | |
| for (int i = 0; i < N; ++i) { | |
| const float v = std::min(std::max(log_frames[i], 0.0f), cap); | |
| int frames = int(std::nearbyint(std::expm1(v) / opt.speed)); | |
| frames = std::max(frames, 1); | |
| const bool present = phone_.at(ids[i]) || dur[size_t(i) * 2 + 1] > 0.0f; | |
| durations[i] = present ? frames : 0; | |
| T += durations[i]; | |
| } | |
| if (T < 1) throw std::runtime_error("utterance has no frames"); | |
| if (T > hp_.max_frames) throw UtteranceTooLong("utterance exceeds max_frames; split the input"); | |
| // ---- expand: token index per frame + position-in-token features | |
| std::vector<int32_t> owner(T); | |
| std::vector<float> feats(size_t(3) * T); | |
| { | |
| int f = 0; | |
| for (int i = 0; i < N; ++i) { | |
| for (int k = 0; k < durations[i]; ++k, ++f) { | |
| const float owned = float(std::max(durations[i], 1)); | |
| const float rel = (float(k) + 0.5f) / owned; | |
| owner[f] = i; | |
| feats[size_t(f) * 3 + 0] = rel; | |
| feats[size_t(f) * 3 + 1] = 1.0f - rel; | |
| feats[size_t(f) * 3 + 2] = std::log1p(owned) / 4.0f; | |
| } | |
| } | |
| } | |
| // ---- stage 2: frames -> prosody stack -> pitch / voicing / energy | |
| std::vector<float> phid(size_t(C) * T), pros(size_t(3) * T); | |
| { | |
| Graph g(this); | |
| ggml_tensor* t_hid = g.input_f32(C, N), * t_idx = g.input_i32(T), * t_feat = g.input_f32(3, T); | |
| ggml_tensor* t_sty = g.input_i32(1), * t_pos = g.input_i32(T); | |
| ggml_tensor* x = ggml_get_rows(g.ctx, t_hid, t_idx); | |
| x = ggml_add(g.ctx, x, linear(g.ctx, t_feat, "frame_position")); | |
| x = ggml_add(g.ctx, x, ggml_get_rows(g.ctx, W("frame_voices.weight"), t_sty)); | |
| ggml_tensor* ph = stack(g.ctx, x, "prosody_stack", hp_.pro_layers, t_pos, T); | |
| ggml_tensor* pr = linear(g.ctx, ph, "prosody_head"); | |
| g.output(ph); g.output(pr); | |
| g.alloc(); | |
| g.set(t_hid, hidden.data()); g.set(t_idx, owner.data()); g.set(t_feat, feats.data()); g.set(t_sty, &opt.style); | |
| set_positions(g, t_pos, T); | |
| g.compute(); | |
| g.get(ph, phid.data()); g.get(pr, pros.data()); | |
| } | |
| std::vector<float> pros_raw = pros; | |
| if (flow_ && fp_.prosody) { // sampled pitch and energy; voicing stays the regression head's | |
| const std::vector<float> pe = sample_flow("pflow_pe", fp_.pe_layers, 2, phid, T, fp_.prosody_steps, | |
| fp_.prosody_temperature, 0.0f, opt.pitch_energy_noise, opt.seed * 3 + 2); | |
| for (int f = 0; f < T; ++f) { pros[size_t(f) * 3] = pe[size_t(f) * 2]; pros[size_t(f) * 3 + 2] = pe[size_t(f) * 2 + 1]; } | |
| } | |
| // vowels are voiced by definition (voiced_vowels=True) | |
| for (int f = 0; f < T; ++f) | |
| if (vowel_.at(ids[owner[f]])) pros[size_t(f) * 3 + 1] = std::max(pros[size_t(f) * 3 + 1], 2.0f); | |
| // soft octave limiter above the voiced median (torch.nanmedian = lower middle value) | |
| if (opt.octave_fix > 0) { | |
| std::vector<float> v; | |
| for (int f = 0; f < T; ++f) if (pros[size_t(f) * 3 + 1] > 0) v.push_back(pros[size_t(f) * 3]); | |
| float median = 0.0f; | |
| if (!v.empty()) { | |
| std::nth_element(v.begin(), v.begin() + (v.size() - 1) / 2, v.end()); | |
| median = v[(v.size() - 1) / 2]; | |
| } | |
| const float limit = std::log(opt.octave_fix); | |
| for (int f = 0; f < T; ++f) { | |
| float& p = pros[size_t(f) * 3]; | |
| const float excess = p - median; | |
| if (excess > 0) p = median + limit * std::tanh(excess / limit); | |
| } | |
| } | |
| // hard prosody (what the decoder is conditioned on) and the oscillator pitch | |
| std::vector<float> cond(size_t(3) * T), f0(T); | |
| for (int f = 0; f < T; ++f) { | |
| const float* p = &pros[size_t(f) * 3]; | |
| cond[size_t(f) * 3 + 0] = std::min(std::max(p[0], -2.0f), 1.5f); | |
| cond[size_t(f) * 3 + 1] = p[1] > 0 ? 1.0f : 0.0f; | |
| cond[size_t(f) * 3 + 2] = std::min(std::max(p[2], -4.5f), 3.0f); | |
| f0[f] = std::min(std::max(hp_.pitch_ref * std::exp(p[0]), hp_.f0_min), hp_.f0_max); | |
| } | |
| if (flow_) { | |
| std::vector<float> wave = flow_decode(phid, cond, T, opt, dbg); | |
| if (dbg) { | |
| dbg->encoder = hidden; dbg->duration_raw = dur; dbg->durations = durations; dbg->frames = T; | |
| dbg->prosody_hidden = phid; dbg->prosody_raw = pros_raw; dbg->prosody = pros; dbg->f0 = f0; | |
| } | |
| return wave; | |
| } | |
| // ---- stage 3: acoustic stack -> synthesizer heads | |
| const int hw = sc_.cep_causal + sc_.cep_anti + hp_.hop / sc_.synth_hop, nw = sc_.noise_cep + hp_.hop / sc_.noise_hop; | |
| std::vector<float> ac(size_t(C) * T), harm(size_t(hw) * T), nh(size_t(nw) * T); | |
| { | |
| Graph g(this); | |
| ggml_tensor* t_ph = g.input_f32(C, T), * t_cond = g.input_f32(3, T), * t_pos = g.input_i32(T); | |
| ggml_tensor* x = ggml_add(g.ctx, t_ph, linear(g.ctx, t_cond, "prosody_projection")); | |
| ggml_tensor* a = stack(g.ctx, x, "acoustic_stack", hp_.ac_layers, t_pos, T); | |
| ggml_tensor* hh = linear(g.ctx, a, "vocoder.harmonic"); | |
| ggml_tensor* nn = linear(g.ctx, a, "vocoder.noise"); | |
| g.output(a); g.output(hh); g.output(nn); | |
| g.alloc(); | |
| g.set(t_ph, phid.data()); g.set(t_cond, cond.data()); set_positions(g, t_pos, T); | |
| g.compute(); | |
| g.get(a, ac.data()); g.get(hh, harm.data()); g.get(nn, nh.data()); | |
| } | |
| // ---- source-filter synthesis | |
| std::vector<float> noise; | |
| const float* nz = opt.noise; | |
| if (!nz) { | |
| noise.resize(size_t(T) * hp_.hop); | |
| std::mt19937_64 rng(opt.seed); | |
| std::normal_distribution<float> normal(0.0f, 1.0f); | |
| for (float& s : noise) s = normal(rng); | |
| nz = noise.data(); | |
| } | |
| std::vector<float> wave = synthesize_wave(sc_, harm.data(), nh.data(), f0.data(), T, nz); | |
| if (dbg) { | |
| dbg->encoder = hidden; dbg->duration_raw = dur; dbg->durations = durations; dbg->frames = T; | |
| dbg->prosody_hidden = phid; dbg->prosody_raw = pros_raw; dbg->prosody = pros; dbg->acoustic = ac; dbg->f0 = f0; | |
| dbg->harmonic_head = harm; dbg->noise_head = nh; | |
| } | |
| return wave; | |
| } | |
| private: | |
| struct FlowHp { | |
| int layers = 10, n_mels = 100, steps = 16, vocos_layers = 8, n_fft = 1024, vocos_hop = 256; | |
| float temperature = 1.0f, guidance = 0.0f; | |
| int jumps = 0; // > 0: distilled student, sampled with this many one-pass jumps | |
| bool prosody = false; // prosody flows (tiny_voca/prosody_flow.py) | |
| int dur_layers = 1, pe_layers = 2, prosody_steps = 8; | |
| float prosody_temperature = 0.8f; | |
| }; | |
| // FlowDecoder.sample (midpoint solver, uniform steps, optional classifier-free guidance) for the decoder stored | |
| // under `p` ("flow", "pflow_dur", "pflow_pe"): cond [T][CD] -> denormalised x [T][M]. noise = the starting point | |
| // (temperature applied); without it x0 = temperature * N(0, 1) from `seed`. | |
| // jumps > 0 (distilled students, FlowDecoder.sample_jumps): `jumps` equal jumps x += d * u(x, t, d), one pass each, | |
| // with the jump-size embedding added to the time embedding; steps and guidance are then unused. | |
| std::vector<float> sample_flow(const std::string& p, int layers, int M, const std::vector<float>& cond, int T, int steps, | |
| float temperature, float guidance, const float* noise, uint64_t seed, int jumps = 0) { | |
| if (jumps > 0) guidance = 0.0f; | |
| const int CD = int(cond.size() / size_t(T)); | |
| const int Wd = int(W(p + ".input.bias")->ne[0]); | |
| std::vector<float> x(size_t(M) * T); | |
| if (noise) { | |
| std::copy(noise, noise + x.size(), x.begin()); | |
| } else { | |
| std::mt19937_64 rng(seed); | |
| std::normal_distribution<float> normal(0.0f, 1.0f); | |
| for (float& v : x) v = temperature * normal(rng); | |
| } | |
| // one velocity graph, built once and run 2 x steps times (x 2 with guidance) with new inputs | |
| std::vector<float> v(x.size()), vu(guidance > 0 ? x.size() : 0), xm(x.size()), temb(Wd); | |
| const std::vector<float> zeros(guidance > 0 ? cond.size() : 0, 0.0f); | |
| { | |
| Graph g(this); | |
| ggml_tensor* t_x = g.input_f32(M, T), * t_c = g.input_f32(CD, T), * t_t = g.input_f32(Wd, 1), * t_pos = g.input_i32(T); | |
| ggml_tensor* h = ggml_add(g.ctx, ggml_add(g.ctx, ggml_mul_mat(g.ctx, W(p + ".input_x.weight"), t_x), | |
| ggml_mul_mat(g.ctx, W(p + ".input_c.weight"), t_c)), W(p + ".input.bias")); | |
| ggml_tensor* te = linear(g.ctx, ggml_silu(g.ctx, linear(g.ctx, t_t, p + ".time.0")), p + ".time.2"); // [W, 1] | |
| ggml_tensor* t_d = nullptr; | |
| if (jumps > 0) { | |
| t_d = g.input_f32(Wd, 1); | |
| te = ggml_add(g.ctx, te, linear(g.ctx, ggml_silu(g.ctx, linear(g.ctx, t_d, p + ".jump.0")), p + ".jump.2")); | |
| } | |
| for (int i = 0; i < layers; ++i) | |
| h = block(g.ctx, ggml_add(g.ctx, h, linear(g.ctx, te, p + ".time_mod." + std::to_string(i))), | |
| p + ".blocks." + std::to_string(i), t_pos, T); | |
| ggml_tensor* out = linear(g.ctx, layernorm(g.ctx, h, p + ".norm"), p + ".output"); // [M, T] | |
| g.output(out); | |
| g.alloc(); | |
| std::vector<float> demb(jumps > 0 ? Wd : 0); | |
| auto embed = [&](float t, std::vector<float>& e) { // FlowDecoder.time_embedding | |
| const int half = Wd / 2; | |
| for (int i = 0; i < half; ++i) { | |
| const float freq = std::exp(-std::log(10000.0f) * float(i) / float(half)); | |
| const float angle = 1000.0f * t * freq; | |
| e[i] = std::sin(angle); e[half + i] = std::cos(angle); | |
| } | |
| }; | |
| float jump = 0.0f; | |
| auto run = [&](const std::vector<float>& xin, const std::vector<float>& cin, float t, std::vector<float>& vout) { | |
| embed(t, temb); | |
| if (t_d) { embed(jump, demb); g.set(t_d, demb.data()); } | |
| g.set(t_x, xin.data()); g.set(t_c, cin.data()); g.set(t_t, temb.data()); set_positions(g, t_pos, T); | |
| g.compute(); | |
| g.get(out, vout.data()); | |
| }; | |
| auto velocity = [&](const std::vector<float>& xin, float t, std::vector<float>& vout) { | |
| run(xin, cond, t, vout); | |
| if (guidance > 0) { // v = v_cond + w (v_cond - v_uncond) | |
| run(xin, zeros, t, vu); | |
| for (size_t i = 0; i < vout.size(); ++i) vout[i] += guidance * (vout[i] - vu[i]); | |
| } | |
| }; | |
| if (jumps > 0) { | |
| const double d = 1.0 / jumps; | |
| jump = float(d); | |
| for (int k = 0; k < jumps; ++k) { | |
| velocity(x, float(k * d), v); | |
| for (size_t i = 0; i < x.size(); ++i) x[i] += float(d) * v[i]; | |
| } | |
| } else { | |
| for (int k = 0; k < steps; ++k) { | |
| const float t0 = float(k) / float(steps), t1 = float(k + 1) / float(steps); // torch.linspace(0, 1, steps + 1) | |
| const double dt = double(t1) - double(t0); | |
| velocity(x, t0, v); | |
| for (size_t i = 0; i < x.size(); ++i) xm[i] = x[i] + float(0.5 * dt) * v[i]; | |
| velocity(xm, float(double(t0) + 0.5 * dt), v); | |
| for (size_t i = 0; i < x.size(); ++i) x[i] += float(dt) * v[i]; | |
| } | |
| } | |
| } | |
| const std::vector<float> mean = read_vec((p + ".mel_mean").c_str()), sd = read_vec((p + ".mel_std").c_str()); | |
| for (int f = 0; f < T; ++f) | |
| for (int m = 0; m < M; ++m) x[size_t(f) * M + m] = x[size_t(f) * M + m] * sd[m] + mean[m]; | |
| return x; | |
| } | |
| // The mel flow on [prosody-stack states, hard prosody], then Vocos. | |
| std::vector<float> flow_decode(const std::vector<float>& phid, const std::vector<float>& cond3, int T, const TtsOptions& opt, | |
| TtsDebug* dbg) { | |
| const int C = hp_.channels, M = fp_.n_mels, CD = C + 3; | |
| const int steps = opt.flow_steps > 0 ? opt.flow_steps : fp_.steps; | |
| const float temperature = opt.temperature >= 0 ? opt.temperature : fp_.temperature; | |
| const float guidance = opt.guidance >= 0 ? opt.guidance : fp_.guidance; | |
| std::vector<float> fc(size_t(CD) * T); | |
| for (int f = 0; f < T; ++f) { | |
| std::copy(phid.begin() + size_t(f) * C, phid.begin() + size_t(f + 1) * C, fc.begin() + size_t(f) * CD); | |
| std::copy(cond3.begin() + size_t(f) * 3, cond3.begin() + size_t(f + 1) * 3, fc.begin() + size_t(f) * CD + C); | |
| } | |
| // distilled students: --steps N means N jumps (default: the GGUF's) | |
| const int jumps = fp_.jumps > 0 ? (opt.flow_steps > 0 ? opt.flow_steps : fp_.jumps) : 0; | |
| const std::vector<float> x = sample_flow("flow", fp_.layers, M, fc, T, steps, temperature, guidance, opt.noise, opt.seed, jumps); | |
| // Vocos gets the mel with its last frame repeated (T + 1 frames -> T * hop samples) | |
| std::vector<float> mel(size_t(M) * (T + 1)); | |
| std::copy(x.begin(), x.end(), mel.begin()); | |
| std::copy(mel.begin() + size_t(T - 1) * M, mel.begin() + size_t(T) * M, mel.begin() + size_t(T) * M); | |
| if (dbg) { dbg->flow_cond = fc; dbg->mel.assign(mel.begin(), mel.begin() + size_t(M) * T); } | |
| const int F = T + 1, bins2 = fp_.n_fft + 2; | |
| std::vector<float> head(size_t(bins2) * F); | |
| { | |
| Graph g(this); | |
| ggml_tensor* t_mel = g.input_f32(M, F); | |
| ggml_tensor* y = conv(g.ctx, t_mel, "vocos.backbone.embed", 7, F); | |
| y = layernorm(g.ctx, y, "vocos.backbone.norm", 1e-6f); | |
| for (int i = 0; i < fp_.vocos_layers; ++i) { | |
| const std::string p = "vocos.backbone.convnext." + std::to_string(i); | |
| ggml_tensor* z = layernorm(g.ctx, depthwise(g.ctx, y, p + ".dwconv", F), p + ".norm", 1e-6f); | |
| z = linear(g.ctx, ggml_gelu_erf(g.ctx, linear(g.ctx, z, p + ".pwconv1")), p + ".pwconv2"); | |
| y = ggml_add(g.ctx, y, ggml_mul(g.ctx, z, W(p + ".gamma"))); | |
| } | |
| ggml_tensor* o = linear(g.ctx, layernorm(g.ctx, y, "vocos.backbone.final_layer_norm", 1e-6f), "vocos.head.out"); | |
| g.output(o); | |
| g.alloc(); | |
| g.set(t_mel, mel.data()); | |
| g.compute(); | |
| g.get(o, head.data()); | |
| } | |
| return vocos_istft(head.data(), F, fp_.n_fft, fp_.vocos_hop); | |
| } | |
| // One compute graph: tensors in a metadata-only context, data allocated by the shared gallocr (reused between calls). | |
| struct Graph { | |
| Engine* e; | |
| ggml_context* ctx; | |
| ggml_cgraph* gf; | |
| std::vector<ggml_tensor*> outputs; | |
| explicit Graph(Engine* engine) : e(engine) { | |
| ggml_init_params p{ggml_tensor_overhead() * 8192 + ggml_graph_overhead_custom(8192, false), nullptr, true}; | |
| ctx = ggml_init(p); | |
| gf = ggml_new_graph_custom(ctx, 8192, false); | |
| } | |
| ~Graph() { ggml_free(ctx); } | |
| ggml_tensor* input_i32(int n) { auto t = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n); ggml_set_input(t); return t; } | |
| ggml_tensor* input_f32(int a, int b) { auto t = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, a, b); ggml_set_input(t); return t; } | |
| void output(ggml_tensor* t) { ggml_set_output(t); outputs.push_back(t); } | |
| void alloc() { | |
| for (auto t : outputs) ggml_build_forward_expand(gf, t); | |
| if (!ggml_gallocr_alloc_graph(e->galloc_, gf)) throw std::runtime_error("graph allocation failed"); | |
| } | |
| void set(ggml_tensor* t, const void* data) { ggml_backend_tensor_set(t, data, 0, ggml_nbytes(t)); } | |
| void get(ggml_tensor* t, void* data) { ggml_backend_tensor_get(t, data, 0, ggml_nbytes(t)); } | |
| void compute() { | |
| if (ggml_backend_graph_compute(e->backend_, gf) != GGML_STATUS_SUCCESS) throw std::runtime_error("compute failed"); | |
| } | |
| }; | |
| int64_t key(const char* k) const { | |
| const int64_t id = gguf_find_key(gguf_, (std::string("tinyvoca.") + k).c_str()); | |
| if (id < 0) throw std::runtime_error(std::string("missing GGUF key ") + k); | |
| return id; | |
| } | |
| std::vector<std::string> strings(const char* k) const { | |
| const int64_t id = key(k); | |
| std::vector<std::string> out; | |
| for (size_t i = 0; i < gguf_get_arr_n(gguf_, id); ++i) out.emplace_back(gguf_get_arr_str(gguf_, id, i)); | |
| return out; | |
| } | |
| std::vector<int> ints(const char* k) const { | |
| const int64_t id = key(k); | |
| const size_t n = gguf_get_arr_n(gguf_, id); | |
| const enum gguf_type type = gguf_get_arr_type(gguf_, id); | |
| const void* d = gguf_get_arr_data(gguf_, id); | |
| std::vector<int> out(n); | |
| for (size_t i = 0; i < n; ++i) { | |
| switch (type) { | |
| case GGUF_TYPE_INT32: out[i] = static_cast<const int32_t*>(d)[i]; break; | |
| case GGUF_TYPE_UINT32: out[i] = int(static_cast<const uint32_t*>(d)[i]); break; | |
| case GGUF_TYPE_INT64: out[i] = int(static_cast<const int64_t*>(d)[i]); break; | |
| case GGUF_TYPE_UINT8: out[i] = static_cast<const uint8_t*>(d)[i]; break; | |
| default: throw std::runtime_error(std::string("unexpected array type for ") + k); | |
| } | |
| } | |
| return out; | |
| } | |
| std::vector<float> read_vec(const char* name) { | |
| ggml_tensor* t = W(name); | |
| std::vector<float> v(ggml_nelements(t)); | |
| ggml_backend_tensor_get(t, v.data(), 0, ggml_nbytes(t)); | |
| return v; | |
| } | |
| ggml_tensor* W(const std::string& name) const { | |
| ggml_tensor* t = ggml_get_tensor(weights_, name.c_str()); | |
| if (!t) throw std::runtime_error("missing tensor " + name); | |
| return t; | |
| } | |
| void set_positions(Graph& g, ggml_tensor* t, int n) { | |
| std::vector<int32_t> pos(n); | |
| for (int i = 0; i < n; ++i) pos[i] = i; | |
| g.set(t, pos.data()); | |
| } | |
| ggml_tensor* linear(ggml_context* ctx, ggml_tensor* x, const std::string& p) const { | |
| return ggml_add(ctx, ggml_mul_mat(ctx, W(p + ".weight"), x), W(p + ".bias")); | |
| } | |
| ggml_tensor* layernorm(ggml_context* ctx, ggml_tensor* x, const std::string& p, float eps = 1e-5f) const { | |
| return ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, eps), W(p + ".weight")), W(p + ".bias")); | |
| } | |
| // depthwise Conv1d, kernel K, padding K/2 (cross-correlation as in PyTorch): sum over taps of shifted inputs | |
| ggml_tensor* depthwise(ggml_context* ctx, ggml_tensor* y, const std::string& p, int T) const { | |
| ggml_tensor* w = W(p + ".weight"); // [C, K]: tap k is row k | |
| const int C = int(w->ne[0]), K = int(w->ne[1]), P = K / 2; | |
| ggml_tensor* padded = ggml_pad_ext(ctx, y, 0, 0, P, P, 0, 0, 0, 0); | |
| ggml_tensor* out = nullptr; | |
| for (int k = 0; k < K; ++k) { | |
| ggml_tensor* v = ggml_view_2d(ctx, padded, C, T, padded->nb[1], size_t(k) * padded->nb[1]); | |
| ggml_tensor* term = ggml_mul(ctx, v, ggml_view_1d(ctx, w, C, size_t(k) * w->nb[1])); | |
| out = out ? ggml_add(ctx, out, term) : term; | |
| } | |
| return ggml_add(ctx, out, W(p + ".bias")); | |
| } | |
| ggml_tensor* block(ggml_context* ctx, ggml_tensor* x, const std::string& p, ggml_tensor* pos, int T) const { | |
| const int C = int(x->ne[0]), H = hp_.heads, D = C / H; // width from the input: flows may be narrower | |
| ggml_tensor* qkv = linear(ctx, layernorm(ctx, x, p + ".norm1"), p + ".attention.qkv"); // [3C, T] | |
| auto part = [&](int i) { | |
| ggml_tensor* v = ggml_view_3d(ctx, qkv, D, H, T, size_t(D) * sizeof(float), qkv->nb[1], size_t(i) * C * sizeof(float)); | |
| return ggml_cont(ctx, v); // [D, H, T] | |
| }; | |
| ggml_tensor* q = ggml_rope_ext(ctx, part(0), pos, nullptr, D, GGML_ROPE_TYPE_NORMAL, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); | |
| ggml_tensor* k = ggml_rope_ext(ctx, part(1), pos, nullptr, D, GGML_ROPE_TYPE_NORMAL, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); | |
| ggml_tensor* v = part(2); | |
| q = ggml_permute(ctx, q, 0, 2, 1, 3); // [D, T, H] | |
| k = ggml_permute(ctx, k, 0, 2, 1, 3); | |
| ggml_tensor* att; | |
| if (flash_) { | |
| // flash attention: never builds the T x T score matrix (memory stays linear in T) | |
| att = ggml_flash_attn_ext(ctx, q, k, ggml_permute(ctx, v, 0, 2, 1, 3), nullptr, 1.0f / std::sqrt(float(D)), 0.0f, 0.0f); | |
| att = ggml_reshape_2d(ctx, att, C, T); // result is [D, H, T] | |
| } else { | |
| ggml_tensor* kq = ggml_soft_max_ext(ctx, ggml_mul_mat(ctx, k, q), nullptr, 1.0f / std::sqrt(float(D)), 0.0f); // [Tk, Tq, H] | |
| ggml_tensor* vt = ggml_cont(ctx, ggml_permute(ctx, v, 1, 2, 0, 3)); // [T, D, H] | |
| att = ggml_cont_2d(ctx, ggml_permute(ctx, ggml_mul_mat(ctx, vt, kq), 0, 2, 1, 3), C, T); | |
| } | |
| att = linear(ctx, att, p + ".attention.out"); | |
| x = ggml_add(ctx, x, att); | |
| ggml_tensor* y = depthwise(ctx, layernorm(ctx, x, p + ".norm2"), p + ".depthwise", T); | |
| y = linear(ctx, ggml_gelu_erf(ctx, linear(ctx, y, p + ".expand")), p + ".project"); | |
| return ggml_add(ctx, x, y); | |
| } | |
| ggml_tensor* stack(ggml_context* ctx, ggml_tensor* x, const std::string& p, int layers, ggml_tensor* pos, int T) const { | |
| for (int i = 0; i < layers; ++i) x = block(ctx, x, p + ".blocks." + std::to_string(i), pos, T); | |
| return layernorm(ctx, x, p + ".norm"); | |
| } | |
| // Conv1d kernel 3, padding 1: three taps as matrix multiplies on shifted inputs | |
| ggml_tensor* conv3(ggml_context* ctx, ggml_tensor* x, const std::string& p, int T) const { return conv(ctx, x, p, 3, T); } | |
| // Conv1d kernel K (odd), padding K/2: K taps "<p>.weight.k<i>" as matrix multiplies on shifted inputs | |
| ggml_tensor* conv(ggml_context* ctx, ggml_tensor* x, const std::string& p, int K, int T) const { | |
| ggml_tensor* padded = ggml_pad_ext(ctx, x, 0, 0, K / 2, K / 2, 0, 0, 0, 0); | |
| ggml_tensor* out = nullptr; | |
| for (int k = 0; k < K; ++k) { | |
| ggml_tensor* v = ggml_view_2d(ctx, padded, x->ne[0], T, padded->nb[1], size_t(k) * padded->nb[1]); | |
| ggml_tensor* term = ggml_mul_mat(ctx, W(p + ".weight.k" + std::to_string(k)), v); | |
| out = out ? ggml_add(ctx, out, term) : term; | |
| } | |
| return ggml_add(ctx, out, W(p + ".bias")); | |
| } | |
| ggml_tensor* predictor(ggml_context* ctx, ggml_tensor* h, int N) const { | |
| ggml_tensor* x = layernorm(ctx, ggml_gelu_erf(ctx, conv3(ctx, h, "duration.conv1", N)), "duration.norm1"); | |
| x = layernorm(ctx, ggml_gelu_erf(ctx, conv3(ctx, x, "duration.conv2", N)), "duration.norm2"); | |
| return linear(ctx, x, "duration.out"); // [2, N] | |
| } | |
| Hp hp_{}; | |
| bool flow_ = false; | |
| FlowHp fp_; | |
| bool flash_ = std::getenv("TTS_NO_FLASH") == nullptr; | |
| SynthConfig sc_; | |
| std::vector<std::string> styles_, symbols_; | |
| std::vector<char> phone_, vowel_; | |
| gguf_context* gguf_ = nullptr; | |
| ggml_context* weights_ = nullptr; | |
| ggml_backend_t backend_ = nullptr; | |
| ggml_backend_buffer_t buffer_ = nullptr; | |
| ggml_gallocr_t galloc_ = nullptr; | |
| size_t weight_bytes_ = 0; | |
| }; | |
| } // namespace | |
| std::unique_ptr<TtsModel> TtsModel::load(const std::string& gguf_path, int threads) { | |
| auto e = std::make_unique<Engine>(); | |
| e->load_file(gguf_path, threads); | |
| return e; | |
| } | |