tiny-voca-hindi / cpp /src /tts.cpp
RL10x's picture
Add inference code (Python + C++ engine + web page)
ab1b068 verified
Raw History Blame Contribute Delete
32 kB
#include "tts.h"
#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <random>
#include <stdexcept>
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "ggml-cpu.h"
#include "ggml.h"
#include "gguf.h"
#include "synth.h"
namespace {
// 64-bit file seek (GGUF files can pass 2 GB in other models)
int seek64(FILE* f, int64_t off) {
#ifdef _WIN32
return _fseeki64(f, off, SEEK_SET);
#else
return fseeko(f, off_t(off), SEEK_SET);
#endif
}
struct Hp {
int channels, heads, feedforward, kernel, enc_layers, pro_layers, ac_layers, pred_channels, max_tokens, max_frames;
int vocab, hop, sample_rate;
float f0_min, f0_max, pitch_ref;
};
class Engine : public TtsModel {
public:
~Engine() override {
if (galloc_) ggml_gallocr_free(galloc_);
if (buffer_) ggml_backend_buffer_free(buffer_);
if (weights_) ggml_free(weights_);
if (gguf_) gguf_free(gguf_);
if (backend_) ggml_backend_free(backend_);
}
void load_file(const std::string& path, int threads) {
gguf_init_params params{true, &weights_};
gguf_ = gguf_init_from_file(path.c_str(), params);
if (!gguf_) throw std::runtime_error("cannot read GGUF " + path);
backend_ = ggml_backend_cpu_init();
ggml_backend_cpu_set_n_threads(backend_, threads);
buffer_ = ggml_backend_alloc_ctx_tensors(weights_, backend_);
FILE* f = std::fopen(path.c_str(), "rb");
if (!f) throw std::runtime_error("cannot open " + path);
std::vector<uint8_t> tmp;
const size_t base = gguf_get_data_offset(gguf_);
for (int64_t i = 0; i < gguf_get_n_tensors(gguf_); ++i) {
ggml_tensor* t = ggml_get_tensor(weights_, gguf_get_tensor_name(gguf_, i));
const size_t n = ggml_nbytes(t);
tmp.resize(n);
seek64(f, int64_t(base + gguf_get_tensor_offset(gguf_, i)));
if (std::fread(tmp.data(), 1, n, f) != n) { std::fclose(f); throw std::runtime_error("short read"); }
ggml_backend_tensor_set(t, tmp.data(), 0, n);
weight_bytes_ += n;
}
std::fclose(f);
auto u32 = [&](const char* k) { return int(gguf_get_val_u32(gguf_, key(k))); };
auto f32 = [&](const char* k) { return gguf_get_val_f32(gguf_, key(k)); };
hp_ = {u32("channels"), u32("heads"), u32("feedforward"), u32("kernel"), u32("encoder_layers"), u32("prosody_layers"),
u32("acoustic_layers"), u32("predictor_channels"), u32("max_tokens"), u32("max_frames"), u32("vocab_size"),
u32("hop"), u32("sample_rate"), f32("f0_min"), f32("f0_max"), f32("pitch_reference_hz")};
styles_ = strings("styles");
symbols_ = strings("symbols");
phone_.assign(hp_.vocab, 0); vowel_.assign(hp_.vocab, 0);
for (int id : ints("phone_ids")) phone_.at(id) = 1;
for (int id : ints("vowel_ids")) vowel_.at(id) = 1;
galloc_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend_));
const int64_t dec = gguf_find_key(gguf_, "tinyvoca.decoder");
flow_ = dec >= 0 && std::string(gguf_get_val_str(gguf_, dec)) == "flow";
if (flow_) {
// flow models: front -> flow-matching mel decoder -> Vocos (scripts/tts_hi_flow_to_gguf.py)
fp_.layers = u32("flow.layers"); fp_.n_mels = u32("flow.n_mels"); fp_.steps = u32("flow.steps");
fp_.temperature = f32("flow.temperature"); fp_.vocos_layers = u32("vocos.layers");
fp_.n_fft = u32("vocos.n_fft"); fp_.vocos_hop = u32("vocos.hop");
const int64_t g = gguf_find_key(gguf_, "tinyvoca.flow.guidance");
fp_.guidance = g >= 0 ? gguf_get_val_f32(gguf_, g) : 0.0f;
const int64_t jk = gguf_find_key(gguf_, "tinyvoca.flow.jumps"); // distilled few-step model
fp_.jumps = jk >= 0 ? int(gguf_get_val_u32(gguf_, jk)) : 0;
const int64_t pf = gguf_find_key(gguf_, "tinyvoca.prosody_flow.steps");
if (pf >= 0) {
fp_.prosody = true;
fp_.dur_layers = u32("prosody_flow.duration_layers"); fp_.pe_layers = u32("prosody_flow.frame_layers");
fp_.prosody_steps = u32("prosody_flow.steps"); fp_.prosody_temperature = f32("prosody_flow.temperature");
}
return;
}
sc_.sample_rate = hp_.sample_rate; sc_.hop = hp_.hop;
sc_.synth_fft = u32("synth_fft"); sc_.synth_hop = u32("synth_hop"); sc_.noise_fft = u32("noise_fft");
sc_.noise_hop = u32("noise_hop"); sc_.cep_causal = u32("cep_causal"); sc_.cep_anti = u32("cep_anti");
sc_.noise_cep = u32("noise_cep");
sc_.harmonic_limit = float(std::round(double(f32("harmonic_limit")) * 1e6) / 1e6);
sc_.causal_lifter = read_vec("vocoder.causal_lifter");
sc_.anti_lifter = read_vec("vocoder.anti_lifter");
sc_.noise_lifter = read_vec("vocoder.noise_lifter");
}
const std::vector<std::string>& styles() const override { return styles_; }
const std::vector<std::string>& symbols() const override { return symbols_; }
int sample_rate() const override { return hp_.sample_rate; }
int max_tokens() const override { return hp_.max_tokens; }
size_t weight_bytes() const override { return weight_bytes_; }
std::vector<float> synthesize(const std::vector<int32_t>& ids, const std::vector<int32_t>& stress, const TtsOptions& opt,
TtsDebug* dbg) override {
const int N = int(ids.size());
if (N < 1 || N > hp_.max_tokens || stress.size() != ids.size()) throw std::runtime_error("invalid token input");
if (opt.style < 0 || opt.style >= int(styles_.size())) throw std::runtime_error("invalid style");
if (!(opt.speed >= 0.25f && opt.speed <= 4.0f)) throw std::runtime_error("speed must be between 0.25 and 4");
const int C = hp_.channels;
// ---- stage 1: embeddings -> encoder -> duration predictor
std::vector<float> hidden(size_t(C) * N), dur(size_t(2) * N);
{
Graph g(this);
ggml_tensor* t_ids = g.input_i32(N), * t_st = g.input_i32(N), * t_sty = g.input_i32(1), * t_pos = g.input_i32(N);
ggml_tensor* x = ggml_add(g.ctx, ggml_get_rows(g.ctx, W("tokens.weight"), t_ids), ggml_get_rows(g.ctx, W("stress.weight"), t_st));
x = ggml_add(g.ctx, x, ggml_get_rows(g.ctx, W("voices.weight"), t_sty));
ggml_tensor* h = stack(g.ctx, x, "encoder", hp_.enc_layers, t_pos, N);
ggml_tensor* d = predictor(g.ctx, h, N);
g.output(h); g.output(d);
g.alloc();
g.set(t_ids, ids.data()); g.set(t_st, stress.data()); g.set(t_sty, &opt.style); set_positions(g, t_pos, N);
g.compute();
g.get(h, hidden.data()); g.get(d, dur.data());
}
// ---- durations (predict_durations): log(1 + frames) regression + "has frames" logit for pause tokens;
// prosody-flow models SAMPLE log(1 + frames) with the duration flow (ProsodyFlows.sample_durations)
std::vector<float> log_frames(N);
if (flow_ && fp_.prosody) {
log_frames = sample_flow("pflow_dur", fp_.dur_layers, 1, hidden, N, fp_.prosody_steps, fp_.prosody_temperature, 0.0f,
opt.duration_noise, opt.seed * 3 + 1);
} else {
for (int i = 0; i < N; ++i) log_frames[i] = dur[size_t(i) * 2];
}
std::vector<int32_t> durations(N);
int T = 0;
const float cap = std::log1p(400.0f);
for (int i = 0; i < N; ++i) {
const float v = std::min(std::max(log_frames[i], 0.0f), cap);
int frames = int(std::nearbyint(std::expm1(v) / opt.speed));
frames = std::max(frames, 1);
const bool present = phone_.at(ids[i]) || dur[size_t(i) * 2 + 1] > 0.0f;
durations[i] = present ? frames : 0;
T += durations[i];
}
if (T < 1) throw std::runtime_error("utterance has no frames");
if (T > hp_.max_frames) throw UtteranceTooLong("utterance exceeds max_frames; split the input");
// ---- expand: token index per frame + position-in-token features
std::vector<int32_t> owner(T);
std::vector<float> feats(size_t(3) * T);
{
int f = 0;
for (int i = 0; i < N; ++i) {
for (int k = 0; k < durations[i]; ++k, ++f) {
const float owned = float(std::max(durations[i], 1));
const float rel = (float(k) + 0.5f) / owned;
owner[f] = i;
feats[size_t(f) * 3 + 0] = rel;
feats[size_t(f) * 3 + 1] = 1.0f - rel;
feats[size_t(f) * 3 + 2] = std::log1p(owned) / 4.0f;
}
}
}
// ---- stage 2: frames -> prosody stack -> pitch / voicing / energy
std::vector<float> phid(size_t(C) * T), pros(size_t(3) * T);
{
Graph g(this);
ggml_tensor* t_hid = g.input_f32(C, N), * t_idx = g.input_i32(T), * t_feat = g.input_f32(3, T);
ggml_tensor* t_sty = g.input_i32(1), * t_pos = g.input_i32(T);
ggml_tensor* x = ggml_get_rows(g.ctx, t_hid, t_idx);
x = ggml_add(g.ctx, x, linear(g.ctx, t_feat, "frame_position"));
x = ggml_add(g.ctx, x, ggml_get_rows(g.ctx, W("frame_voices.weight"), t_sty));
ggml_tensor* ph = stack(g.ctx, x, "prosody_stack", hp_.pro_layers, t_pos, T);
ggml_tensor* pr = linear(g.ctx, ph, "prosody_head");
g.output(ph); g.output(pr);
g.alloc();
g.set(t_hid, hidden.data()); g.set(t_idx, owner.data()); g.set(t_feat, feats.data()); g.set(t_sty, &opt.style);
set_positions(g, t_pos, T);
g.compute();
g.get(ph, phid.data()); g.get(pr, pros.data());
}
std::vector<float> pros_raw = pros;
if (flow_ && fp_.prosody) { // sampled pitch and energy; voicing stays the regression head's
const std::vector<float> pe = sample_flow("pflow_pe", fp_.pe_layers, 2, phid, T, fp_.prosody_steps,
fp_.prosody_temperature, 0.0f, opt.pitch_energy_noise, opt.seed * 3 + 2);
for (int f = 0; f < T; ++f) { pros[size_t(f) * 3] = pe[size_t(f) * 2]; pros[size_t(f) * 3 + 2] = pe[size_t(f) * 2 + 1]; }
}
// vowels are voiced by definition (voiced_vowels=True)
for (int f = 0; f < T; ++f)
if (vowel_.at(ids[owner[f]])) pros[size_t(f) * 3 + 1] = std::max(pros[size_t(f) * 3 + 1], 2.0f);
// soft octave limiter above the voiced median (torch.nanmedian = lower middle value)
if (opt.octave_fix > 0) {
std::vector<float> v;
for (int f = 0; f < T; ++f) if (pros[size_t(f) * 3 + 1] > 0) v.push_back(pros[size_t(f) * 3]);
float median = 0.0f;
if (!v.empty()) {
std::nth_element(v.begin(), v.begin() + (v.size() - 1) / 2, v.end());
median = v[(v.size() - 1) / 2];
}
const float limit = std::log(opt.octave_fix);
for (int f = 0; f < T; ++f) {
float& p = pros[size_t(f) * 3];
const float excess = p - median;
if (excess > 0) p = median + limit * std::tanh(excess / limit);
}
}
// hard prosody (what the decoder is conditioned on) and the oscillator pitch
std::vector<float> cond(size_t(3) * T), f0(T);
for (int f = 0; f < T; ++f) {
const float* p = &pros[size_t(f) * 3];
cond[size_t(f) * 3 + 0] = std::min(std::max(p[0], -2.0f), 1.5f);
cond[size_t(f) * 3 + 1] = p[1] > 0 ? 1.0f : 0.0f;
cond[size_t(f) * 3 + 2] = std::min(std::max(p[2], -4.5f), 3.0f);
f0[f] = std::min(std::max(hp_.pitch_ref * std::exp(p[0]), hp_.f0_min), hp_.f0_max);
}
if (flow_) {
std::vector<float> wave = flow_decode(phid, cond, T, opt, dbg);
if (dbg) {
dbg->encoder = hidden; dbg->duration_raw = dur; dbg->durations = durations; dbg->frames = T;
dbg->prosody_hidden = phid; dbg->prosody_raw = pros_raw; dbg->prosody = pros; dbg->f0 = f0;
}
return wave;
}
// ---- stage 3: acoustic stack -> synthesizer heads
const int hw = sc_.cep_causal + sc_.cep_anti + hp_.hop / sc_.synth_hop, nw = sc_.noise_cep + hp_.hop / sc_.noise_hop;
std::vector<float> ac(size_t(C) * T), harm(size_t(hw) * T), nh(size_t(nw) * T);
{
Graph g(this);
ggml_tensor* t_ph = g.input_f32(C, T), * t_cond = g.input_f32(3, T), * t_pos = g.input_i32(T);
ggml_tensor* x = ggml_add(g.ctx, t_ph, linear(g.ctx, t_cond, "prosody_projection"));
ggml_tensor* a = stack(g.ctx, x, "acoustic_stack", hp_.ac_layers, t_pos, T);
ggml_tensor* hh = linear(g.ctx, a, "vocoder.harmonic");
ggml_tensor* nn = linear(g.ctx, a, "vocoder.noise");
g.output(a); g.output(hh); g.output(nn);
g.alloc();
g.set(t_ph, phid.data()); g.set(t_cond, cond.data()); set_positions(g, t_pos, T);
g.compute();
g.get(a, ac.data()); g.get(hh, harm.data()); g.get(nn, nh.data());
}
// ---- source-filter synthesis
std::vector<float> noise;
const float* nz = opt.noise;
if (!nz) {
noise.resize(size_t(T) * hp_.hop);
std::mt19937_64 rng(opt.seed);
std::normal_distribution<float> normal(0.0f, 1.0f);
for (float& s : noise) s = normal(rng);
nz = noise.data();
}
std::vector<float> wave = synthesize_wave(sc_, harm.data(), nh.data(), f0.data(), T, nz);
if (dbg) {
dbg->encoder = hidden; dbg->duration_raw = dur; dbg->durations = durations; dbg->frames = T;
dbg->prosody_hidden = phid; dbg->prosody_raw = pros_raw; dbg->prosody = pros; dbg->acoustic = ac; dbg->f0 = f0;
dbg->harmonic_head = harm; dbg->noise_head = nh;
}
return wave;
}
private:
struct FlowHp {
int layers = 10, n_mels = 100, steps = 16, vocos_layers = 8, n_fft = 1024, vocos_hop = 256;
float temperature = 1.0f, guidance = 0.0f;
int jumps = 0; // > 0: distilled student, sampled with this many one-pass jumps
bool prosody = false; // prosody flows (tiny_voca/prosody_flow.py)
int dur_layers = 1, pe_layers = 2, prosody_steps = 8;
float prosody_temperature = 0.8f;
};
// FlowDecoder.sample (midpoint solver, uniform steps, optional classifier-free guidance) for the decoder stored
// under `p` ("flow", "pflow_dur", "pflow_pe"): cond [T][CD] -> denormalised x [T][M]. noise = the starting point
// (temperature applied); without it x0 = temperature * N(0, 1) from `seed`.
// jumps > 0 (distilled students, FlowDecoder.sample_jumps): `jumps` equal jumps x += d * u(x, t, d), one pass each,
// with the jump-size embedding added to the time embedding; steps and guidance are then unused.
std::vector<float> sample_flow(const std::string& p, int layers, int M, const std::vector<float>& cond, int T, int steps,
float temperature, float guidance, const float* noise, uint64_t seed, int jumps = 0) {
if (jumps > 0) guidance = 0.0f;
const int CD = int(cond.size() / size_t(T));
const int Wd = int(W(p + ".input.bias")->ne[0]);
std::vector<float> x(size_t(M) * T);
if (noise) {
std::copy(noise, noise + x.size(), x.begin());
} else {
std::mt19937_64 rng(seed);
std::normal_distribution<float> normal(0.0f, 1.0f);
for (float& v : x) v = temperature * normal(rng);
}
// one velocity graph, built once and run 2 x steps times (x 2 with guidance) with new inputs
std::vector<float> v(x.size()), vu(guidance > 0 ? x.size() : 0), xm(x.size()), temb(Wd);
const std::vector<float> zeros(guidance > 0 ? cond.size() : 0, 0.0f);
{
Graph g(this);
ggml_tensor* t_x = g.input_f32(M, T), * t_c = g.input_f32(CD, T), * t_t = g.input_f32(Wd, 1), * t_pos = g.input_i32(T);
ggml_tensor* h = ggml_add(g.ctx, ggml_add(g.ctx, ggml_mul_mat(g.ctx, W(p + ".input_x.weight"), t_x),
ggml_mul_mat(g.ctx, W(p + ".input_c.weight"), t_c)), W(p + ".input.bias"));
ggml_tensor* te = linear(g.ctx, ggml_silu(g.ctx, linear(g.ctx, t_t, p + ".time.0")), p + ".time.2"); // [W, 1]
ggml_tensor* t_d = nullptr;
if (jumps > 0) {
t_d = g.input_f32(Wd, 1);
te = ggml_add(g.ctx, te, linear(g.ctx, ggml_silu(g.ctx, linear(g.ctx, t_d, p + ".jump.0")), p + ".jump.2"));
}
for (int i = 0; i < layers; ++i)
h = block(g.ctx, ggml_add(g.ctx, h, linear(g.ctx, te, p + ".time_mod." + std::to_string(i))),
p + ".blocks." + std::to_string(i), t_pos, T);
ggml_tensor* out = linear(g.ctx, layernorm(g.ctx, h, p + ".norm"), p + ".output"); // [M, T]
g.output(out);
g.alloc();
std::vector<float> demb(jumps > 0 ? Wd : 0);
auto embed = [&](float t, std::vector<float>& e) { // FlowDecoder.time_embedding
const int half = Wd / 2;
for (int i = 0; i < half; ++i) {
const float freq = std::exp(-std::log(10000.0f) * float(i) / float(half));
const float angle = 1000.0f * t * freq;
e[i] = std::sin(angle); e[half + i] = std::cos(angle);
}
};
float jump = 0.0f;
auto run = [&](const std::vector<float>& xin, const std::vector<float>& cin, float t, std::vector<float>& vout) {
embed(t, temb);
if (t_d) { embed(jump, demb); g.set(t_d, demb.data()); }
g.set(t_x, xin.data()); g.set(t_c, cin.data()); g.set(t_t, temb.data()); set_positions(g, t_pos, T);
g.compute();
g.get(out, vout.data());
};
auto velocity = [&](const std::vector<float>& xin, float t, std::vector<float>& vout) {
run(xin, cond, t, vout);
if (guidance > 0) { // v = v_cond + w (v_cond - v_uncond)
run(xin, zeros, t, vu);
for (size_t i = 0; i < vout.size(); ++i) vout[i] += guidance * (vout[i] - vu[i]);
}
};
if (jumps > 0) {
const double d = 1.0 / jumps;
jump = float(d);
for (int k = 0; k < jumps; ++k) {
velocity(x, float(k * d), v);
for (size_t i = 0; i < x.size(); ++i) x[i] += float(d) * v[i];
}
} else {
for (int k = 0; k < steps; ++k) {
const float t0 = float(k) / float(steps), t1 = float(k + 1) / float(steps); // torch.linspace(0, 1, steps + 1)
const double dt = double(t1) - double(t0);
velocity(x, t0, v);
for (size_t i = 0; i < x.size(); ++i) xm[i] = x[i] + float(0.5 * dt) * v[i];
velocity(xm, float(double(t0) + 0.5 * dt), v);
for (size_t i = 0; i < x.size(); ++i) x[i] += float(dt) * v[i];
}
}
}
const std::vector<float> mean = read_vec((p + ".mel_mean").c_str()), sd = read_vec((p + ".mel_std").c_str());
for (int f = 0; f < T; ++f)
for (int m = 0; m < M; ++m) x[size_t(f) * M + m] = x[size_t(f) * M + m] * sd[m] + mean[m];
return x;
}
// The mel flow on [prosody-stack states, hard prosody], then Vocos.
std::vector<float> flow_decode(const std::vector<float>& phid, const std::vector<float>& cond3, int T, const TtsOptions& opt,
TtsDebug* dbg) {
const int C = hp_.channels, M = fp_.n_mels, CD = C + 3;
const int steps = opt.flow_steps > 0 ? opt.flow_steps : fp_.steps;
const float temperature = opt.temperature >= 0 ? opt.temperature : fp_.temperature;
const float guidance = opt.guidance >= 0 ? opt.guidance : fp_.guidance;
std::vector<float> fc(size_t(CD) * T);
for (int f = 0; f < T; ++f) {
std::copy(phid.begin() + size_t(f) * C, phid.begin() + size_t(f + 1) * C, fc.begin() + size_t(f) * CD);
std::copy(cond3.begin() + size_t(f) * 3, cond3.begin() + size_t(f + 1) * 3, fc.begin() + size_t(f) * CD + C);
}
// distilled students: --steps N means N jumps (default: the GGUF's)
const int jumps = fp_.jumps > 0 ? (opt.flow_steps > 0 ? opt.flow_steps : fp_.jumps) : 0;
const std::vector<float> x = sample_flow("flow", fp_.layers, M, fc, T, steps, temperature, guidance, opt.noise, opt.seed, jumps);
// Vocos gets the mel with its last frame repeated (T + 1 frames -> T * hop samples)
std::vector<float> mel(size_t(M) * (T + 1));
std::copy(x.begin(), x.end(), mel.begin());
std::copy(mel.begin() + size_t(T - 1) * M, mel.begin() + size_t(T) * M, mel.begin() + size_t(T) * M);
if (dbg) { dbg->flow_cond = fc; dbg->mel.assign(mel.begin(), mel.begin() + size_t(M) * T); }
const int F = T + 1, bins2 = fp_.n_fft + 2;
std::vector<float> head(size_t(bins2) * F);
{
Graph g(this);
ggml_tensor* t_mel = g.input_f32(M, F);
ggml_tensor* y = conv(g.ctx, t_mel, "vocos.backbone.embed", 7, F);
y = layernorm(g.ctx, y, "vocos.backbone.norm", 1e-6f);
for (int i = 0; i < fp_.vocos_layers; ++i) {
const std::string p = "vocos.backbone.convnext." + std::to_string(i);
ggml_tensor* z = layernorm(g.ctx, depthwise(g.ctx, y, p + ".dwconv", F), p + ".norm", 1e-6f);
z = linear(g.ctx, ggml_gelu_erf(g.ctx, linear(g.ctx, z, p + ".pwconv1")), p + ".pwconv2");
y = ggml_add(g.ctx, y, ggml_mul(g.ctx, z, W(p + ".gamma")));
}
ggml_tensor* o = linear(g.ctx, layernorm(g.ctx, y, "vocos.backbone.final_layer_norm", 1e-6f), "vocos.head.out");
g.output(o);
g.alloc();
g.set(t_mel, mel.data());
g.compute();
g.get(o, head.data());
}
return vocos_istft(head.data(), F, fp_.n_fft, fp_.vocos_hop);
}
// One compute graph: tensors in a metadata-only context, data allocated by the shared gallocr (reused between calls).
struct Graph {
Engine* e;
ggml_context* ctx;
ggml_cgraph* gf;
std::vector<ggml_tensor*> outputs;
explicit Graph(Engine* engine) : e(engine) {
ggml_init_params p{ggml_tensor_overhead() * 8192 + ggml_graph_overhead_custom(8192, false), nullptr, true};
ctx = ggml_init(p);
gf = ggml_new_graph_custom(ctx, 8192, false);
}
~Graph() { ggml_free(ctx); }
ggml_tensor* input_i32(int n) { auto t = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n); ggml_set_input(t); return t; }
ggml_tensor* input_f32(int a, int b) { auto t = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, a, b); ggml_set_input(t); return t; }
void output(ggml_tensor* t) { ggml_set_output(t); outputs.push_back(t); }
void alloc() {
for (auto t : outputs) ggml_build_forward_expand(gf, t);
if (!ggml_gallocr_alloc_graph(e->galloc_, gf)) throw std::runtime_error("graph allocation failed");
}
void set(ggml_tensor* t, const void* data) { ggml_backend_tensor_set(t, data, 0, ggml_nbytes(t)); }
void get(ggml_tensor* t, void* data) { ggml_backend_tensor_get(t, data, 0, ggml_nbytes(t)); }
void compute() {
if (ggml_backend_graph_compute(e->backend_, gf) != GGML_STATUS_SUCCESS) throw std::runtime_error("compute failed");
}
};
int64_t key(const char* k) const {
const int64_t id = gguf_find_key(gguf_, (std::string("tinyvoca.") + k).c_str());
if (id < 0) throw std::runtime_error(std::string("missing GGUF key ") + k);
return id;
}
std::vector<std::string> strings(const char* k) const {
const int64_t id = key(k);
std::vector<std::string> out;
for (size_t i = 0; i < gguf_get_arr_n(gguf_, id); ++i) out.emplace_back(gguf_get_arr_str(gguf_, id, i));
return out;
}
std::vector<int> ints(const char* k) const {
const int64_t id = key(k);
const size_t n = gguf_get_arr_n(gguf_, id);
const enum gguf_type type = gguf_get_arr_type(gguf_, id);
const void* d = gguf_get_arr_data(gguf_, id);
std::vector<int> out(n);
for (size_t i = 0; i < n; ++i) {
switch (type) {
case GGUF_TYPE_INT32: out[i] = static_cast<const int32_t*>(d)[i]; break;
case GGUF_TYPE_UINT32: out[i] = int(static_cast<const uint32_t*>(d)[i]); break;
case GGUF_TYPE_INT64: out[i] = int(static_cast<const int64_t*>(d)[i]); break;
case GGUF_TYPE_UINT8: out[i] = static_cast<const uint8_t*>(d)[i]; break;
default: throw std::runtime_error(std::string("unexpected array type for ") + k);
}
}
return out;
}
std::vector<float> read_vec(const char* name) {
ggml_tensor* t = W(name);
std::vector<float> v(ggml_nelements(t));
ggml_backend_tensor_get(t, v.data(), 0, ggml_nbytes(t));
return v;
}
ggml_tensor* W(const std::string& name) const {
ggml_tensor* t = ggml_get_tensor(weights_, name.c_str());
if (!t) throw std::runtime_error("missing tensor " + name);
return t;
}
void set_positions(Graph& g, ggml_tensor* t, int n) {
std::vector<int32_t> pos(n);
for (int i = 0; i < n; ++i) pos[i] = i;
g.set(t, pos.data());
}
ggml_tensor* linear(ggml_context* ctx, ggml_tensor* x, const std::string& p) const {
return ggml_add(ctx, ggml_mul_mat(ctx, W(p + ".weight"), x), W(p + ".bias"));
}
ggml_tensor* layernorm(ggml_context* ctx, ggml_tensor* x, const std::string& p, float eps = 1e-5f) const {
return ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, eps), W(p + ".weight")), W(p + ".bias"));
}
// depthwise Conv1d, kernel K, padding K/2 (cross-correlation as in PyTorch): sum over taps of shifted inputs
ggml_tensor* depthwise(ggml_context* ctx, ggml_tensor* y, const std::string& p, int T) const {
ggml_tensor* w = W(p + ".weight"); // [C, K]: tap k is row k
const int C = int(w->ne[0]), K = int(w->ne[1]), P = K / 2;
ggml_tensor* padded = ggml_pad_ext(ctx, y, 0, 0, P, P, 0, 0, 0, 0);
ggml_tensor* out = nullptr;
for (int k = 0; k < K; ++k) {
ggml_tensor* v = ggml_view_2d(ctx, padded, C, T, padded->nb[1], size_t(k) * padded->nb[1]);
ggml_tensor* term = ggml_mul(ctx, v, ggml_view_1d(ctx, w, C, size_t(k) * w->nb[1]));
out = out ? ggml_add(ctx, out, term) : term;
}
return ggml_add(ctx, out, W(p + ".bias"));
}
ggml_tensor* block(ggml_context* ctx, ggml_tensor* x, const std::string& p, ggml_tensor* pos, int T) const {
const int C = int(x->ne[0]), H = hp_.heads, D = C / H; // width from the input: flows may be narrower
ggml_tensor* qkv = linear(ctx, layernorm(ctx, x, p + ".norm1"), p + ".attention.qkv"); // [3C, T]
auto part = [&](int i) {
ggml_tensor* v = ggml_view_3d(ctx, qkv, D, H, T, size_t(D) * sizeof(float), qkv->nb[1], size_t(i) * C * sizeof(float));
return ggml_cont(ctx, v); // [D, H, T]
};
ggml_tensor* q = ggml_rope_ext(ctx, part(0), pos, nullptr, D, GGML_ROPE_TYPE_NORMAL, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
ggml_tensor* k = ggml_rope_ext(ctx, part(1), pos, nullptr, D, GGML_ROPE_TYPE_NORMAL, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
ggml_tensor* v = part(2);
q = ggml_permute(ctx, q, 0, 2, 1, 3); // [D, T, H]
k = ggml_permute(ctx, k, 0, 2, 1, 3);
ggml_tensor* att;
if (flash_) {
// flash attention: never builds the T x T score matrix (memory stays linear in T)
att = ggml_flash_attn_ext(ctx, q, k, ggml_permute(ctx, v, 0, 2, 1, 3), nullptr, 1.0f / std::sqrt(float(D)), 0.0f, 0.0f);
att = ggml_reshape_2d(ctx, att, C, T); // result is [D, H, T]
} else {
ggml_tensor* kq = ggml_soft_max_ext(ctx, ggml_mul_mat(ctx, k, q), nullptr, 1.0f / std::sqrt(float(D)), 0.0f); // [Tk, Tq, H]
ggml_tensor* vt = ggml_cont(ctx, ggml_permute(ctx, v, 1, 2, 0, 3)); // [T, D, H]
att = ggml_cont_2d(ctx, ggml_permute(ctx, ggml_mul_mat(ctx, vt, kq), 0, 2, 1, 3), C, T);
}
att = linear(ctx, att, p + ".attention.out");
x = ggml_add(ctx, x, att);
ggml_tensor* y = depthwise(ctx, layernorm(ctx, x, p + ".norm2"), p + ".depthwise", T);
y = linear(ctx, ggml_gelu_erf(ctx, linear(ctx, y, p + ".expand")), p + ".project");
return ggml_add(ctx, x, y);
}
ggml_tensor* stack(ggml_context* ctx, ggml_tensor* x, const std::string& p, int layers, ggml_tensor* pos, int T) const {
for (int i = 0; i < layers; ++i) x = block(ctx, x, p + ".blocks." + std::to_string(i), pos, T);
return layernorm(ctx, x, p + ".norm");
}
// Conv1d kernel 3, padding 1: three taps as matrix multiplies on shifted inputs
ggml_tensor* conv3(ggml_context* ctx, ggml_tensor* x, const std::string& p, int T) const { return conv(ctx, x, p, 3, T); }
// Conv1d kernel K (odd), padding K/2: K taps "<p>.weight.k<i>" as matrix multiplies on shifted inputs
ggml_tensor* conv(ggml_context* ctx, ggml_tensor* x, const std::string& p, int K, int T) const {
ggml_tensor* padded = ggml_pad_ext(ctx, x, 0, 0, K / 2, K / 2, 0, 0, 0, 0);
ggml_tensor* out = nullptr;
for (int k = 0; k < K; ++k) {
ggml_tensor* v = ggml_view_2d(ctx, padded, x->ne[0], T, padded->nb[1], size_t(k) * padded->nb[1]);
ggml_tensor* term = ggml_mul_mat(ctx, W(p + ".weight.k" + std::to_string(k)), v);
out = out ? ggml_add(ctx, out, term) : term;
}
return ggml_add(ctx, out, W(p + ".bias"));
}
ggml_tensor* predictor(ggml_context* ctx, ggml_tensor* h, int N) const {
ggml_tensor* x = layernorm(ctx, ggml_gelu_erf(ctx, conv3(ctx, h, "duration.conv1", N)), "duration.norm1");
x = layernorm(ctx, ggml_gelu_erf(ctx, conv3(ctx, x, "duration.conv2", N)), "duration.norm2");
return linear(ctx, x, "duration.out"); // [2, N]
}
Hp hp_{};
bool flow_ = false;
FlowHp fp_;
bool flash_ = std::getenv("TTS_NO_FLASH") == nullptr;
SynthConfig sc_;
std::vector<std::string> styles_, symbols_;
std::vector<char> phone_, vowel_;
gguf_context* gguf_ = nullptr;
ggml_context* weights_ = nullptr;
ggml_backend_t backend_ = nullptr;
ggml_backend_buffer_t buffer_ = nullptr;
ggml_gallocr_t galloc_ = nullptr;
size_t weight_bytes_ = 0;
};
} // namespace
std::unique_ptr<TtsModel> TtsModel::load(const std::string& gguf_path, int threads) {
auto e = std::make_unique<Engine>();
e->load_file(gguf_path, threads);
return e;
}