File size: 2,537 Bytes
ab1b068
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
// Hindi tiny-voca TTS engine on ggml: phoneme ids -> waveform (24 kHz). Port of tiny_voca.model.TinyVoca.synthesize
// (voiced_vowels=True, soft octave limiter) and the source-filter synthesizer (synth.h).
#pragma once
#include <cstdint>
#include <memory>
#include <stdexcept>
#include <string>
#include <vector>

struct TtsOptions {
    int style = 0;              // index into TtsModel::styles()
    float speed = 1.0f;         // 0.5 .. 2
    float octave_fix = 1.6f;    // soft pitch limiter (0 = off), infer.OCTAVE_FIX
    uint64_t seed = 0;          // noise seed
    // optional external noise, for checking against PyTorch: synthesizer models T*hop samples; flow models the starting
    // point x0 of the flow, T*n_mels values with the temperature already applied
    const float* noise = nullptr;
    int flow_steps = 0;         // flow models: midpoint steps (0 = the GGUF's default, 16)
    float temperature = -1.0f;  // flow models: starting-noise scale (< 0 = the GGUF's default, 1.0)
    float guidance = -1.0f;     // flow models: classifier-free guidance strength (< 0 = the GGUF's default)
    // prosody-flow models, for checking against PyTorch: the duration flow's start [N] and the pitch/energy flow's
    // start [T * 2], temperature applied
    const float* duration_noise = nullptr;
    const float* pitch_energy_noise = nullptr;
};

// Intermediates, filled when a pointer is passed (for checking against scripts/tts_hi_reference.py and
// scripts/tts_hi_flow_reference.py).
struct TtsDebug {
    std::vector<float> encoder, duration_raw, prosody_hidden, prosody_raw, prosody, acoustic, f0, harmonic_head, noise_head;
    std::vector<float> flow_cond, mel;     // flow models: [T][259] conditioning and [T][n_mels] flow output
    std::vector<int32_t> durations;
    int frames = 0;
};

class TtsModel {
public:
    static std::unique_ptr<TtsModel> load(const std::string& gguf_path, int threads);
    virtual ~TtsModel() = default;
    virtual std::vector<float> synthesize(const std::vector<int32_t>& ids, const std::vector<int32_t>& stress,
                                          const TtsOptions& opt, TtsDebug* dbg = nullptr) = 0;
    virtual const std::vector<std::string>& styles() const = 0;
    virtual const std::vector<std::string>& symbols() const = 0;
    virtual int sample_rate() const = 0;
    virtual int max_tokens() const = 0;
    virtual size_t weight_bytes() const = 0;
};

class UtteranceTooLong : public std::runtime_error {
public:
    using std::runtime_error::runtime_error;
};