File size: 11,291 Bytes
a775b3f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
"""Portable phoneme-only FastMicro 24K runtime. Requires PyTorch and packaged code."""
import hashlib
import importlib
import json
import math
from pathlib import Path
import sys
import threading

SCHEMA = 'inflect-fastmicro24k-phonemes-v1'
VOICES = ('hope', 'sophia', 'cassidy', 'james', 'jasper', 'mark')
DONORS = ('af_heart', 'af_heart', 'af_sarah', 'am_michael', 'am_michael', 'am_michael')
CODE_FILES = ('native_mini_inference.py', 'native26_text_model.py', 'native33_text_model.py',
              'native33_text_fast.py', 'inflect_kokoro_student27.py', 'native27_nativegen_bridge.py',
              'qualify_native27_native_generator_budget.py', 'student.py', 'student_decoder.py',
              'kokoro/__init__.py', 'kokoro/model.py', 'kokoro/modules.py', 'kokoro/istftnet.py',
              'kokoro/custom_stft.py', 'vhead.py', 'fastmicro_inference_20261029.py')
PAYLOAD_KEYS = {'schema', 'text_config', 'text_state', 'joint_state', 'styles', 'vocab',
                'max_frames', 'sample_rate', 'provenance'}
_LOCK = threading.RLock()


def sha(path):
    h = hashlib.sha256()
    with open(path, 'rb') as f:
        for chunk in iter(lambda: f.read(1024 * 1024), b''):
            h.update(chunk)
    return h.hexdigest()


def verify_manifest(directory):
    root = Path(directory).resolve(strict=True)
    if (root / 'manifest.json').is_symlink():
        raise ValueError('Symlink manifest is not allowed')
    manifest = json.loads((root / 'manifest.json').read_text(encoding='utf-8'))
    files = manifest.get('files')
    required = {'weights.pt'} | {'code/' + name for name in CODE_FILES}
    if manifest.get('schema') != SCHEMA or not isinstance(files, dict) or not required <= set(files):
        raise ValueError('Invalid export manifest or missing required files')
    for name, digest in files.items():
        if not isinstance(name, str) or '\\' in name or not isinstance(digest, str):
            raise ValueError('Invalid manifest entry')
        parts = name.split('/')
        if any(p in ('', '.', '..') for p in parts) or name.startswith('/') or ':' in name:
            raise ValueError('Unsafe export path')
        path = root.joinpath(*parts)
        if any(root.joinpath(*parts[:i]).is_symlink() for i in range(1, len(parts) + 1)):
            raise ValueError('Symlink in export path')
        if not path.is_file() or not path.resolve().is_relative_to(root) or sha(path) != digest:
            raise ValueError('Export file hash mismatch: ' + name)
    return root, manifest


def _code_path(root):
    code = root / 'code'
    # Avoid silently importing a different model's identically named modules.
    names = {Path(name).parts[0].removesuffix('.py') for name in CODE_FILES}
    for name, module in tuple(sys.modules.items()):
        if name.split('.')[0] not in names or name == __name__:
            continue
        origin = getattr(module, '__file__', None)
        if not origin or not Path(origin).resolve().is_relative_to(code):
            raise RuntimeError('Conflicting preloaded module: ' + name)
    if str(code) in sys.path:
        sys.path.remove(str(code))
    sys.path.insert(0, str(code))


def _make_joint(torch, mini, vhead, max_frames):
    nn = torch.nn
    class ConditionedHead(nn.Module):
        def __init__(self):
            super().__init__()
            self.head = vhead.SourceFilterHead2(in_ch=128, dim=320, blocks=7)
            self.head.blocks[-1] = vhead.ConvNeXt(320, mult=2)
            self.style_affine = nn.Linear(128, 256)

        def forward(self, hidden, style, f0):
            scale, shift = self.style_affine(style).chunk(2, -1)
            conditioned = hidden * (1 + 0.1 * torch.tanh(scale[:, :, None])) + shift[:, :, None]
            return self.head(conditioned, f0, c0=None)

    class Joint(nn.Module):
        def __init__(self):
            super().__init__()
            self.bridge = mini.build_bridge(max_frames)
            del self.bridge.projection
            self.head = ConditionedHead()

        def forward(self, asr, f0, noise, style, sid):
            bridge, trunk = self.bridge, self.bridge.trunk
            speakers = torch.full((asr.shape[0],), sid, device=asr.device, dtype=torch.long)
            voice_style = bridge.speaker_embedding(speakers)
            f0_conv, n_conv = trunk.F0_conv(f0[:, None]), trunk.N_conv(noise[:, None])
            x = trunk.encode(torch.cat((asr, f0_conv, n_conv), 1), voice_style)
            residual, use_residual = trunk.asr_res(asr), True
            for block in trunk.decode:
                if use_residual:
                    x = torch.cat((x, residual, f0_conv, n_conv), 1)
                x = block(x, voice_style)
                if block.upsample_type != 'none':
                    use_residual = False
            if x.shape != (asr.shape[0], 128, f0.shape[1]):
                raise ValueError('Unexpected trunk output shape')
            return self.head(x, style, f0)
    return Joint()


class FastMicro:
    def __init__(self, payload, root, device='cpu', allow_cuda=False):
        import torch
        if set(payload) != PAYLOAD_KEYS or payload['schema'] != SCHEMA:
            raise ValueError('Invalid payload schema')
        dev = torch.device(device)
        if dev.type != 'cpu' and not (dev.type == 'cuda' and allow_cuda):
            raise ValueError('Only CPU or explicitly allowed CUDA is supported')
        if payload['sample_rate'] != 24000 or type(payload['max_frames']) is not int or not 2 <= payload['max_frames'] <= 1200:
            raise ValueError('Invalid sample rate or frame bound')
        _code_path(root)
        mini = importlib.import_module('native_mini_inference')
        factory = importlib.import_module('native26_text_model')
        vhead = importlib.import_module('vhead')
        if payload['text_config'] != factory.default_config():
            raise ValueError('Text configuration differs')
        with torch.random.fork_rng(devices=[]):
            self.text = factory.build_model(payload['text_config'])
            self.joint = _make_joint(torch, mini, vhead, payload['max_frames'])
        for model, state in ((self.text, payload['text_state']), (self.joint, payload['joint_state'])):
            mini.tensor_state(state)
            mini.strict_load(model, state)
        styles, vocab = payload['styles'], payload['vocab']
        if not isinstance(styles, dict) or set(styles) != {'banks', 'offsets', 'strengths'}:
            raise ValueError('Invalid style schema')
        if set(styles['banks']) != set(DONORS) or any(not torch.is_tensor(x) or x.shape != (510, 128) or x.dtype != torch.float32 or not bool(torch.isfinite(x).all()) for x in styles['banks'].values()):
            raise ValueError('Invalid style banks')
        offsets, strengths = styles['offsets'], styles['strengths']
        if not torch.is_tensor(offsets) or offsets.shape != (6, 128) or offsets.dtype != torch.float32 or not bool(torch.isfinite(offsets).all()) or not isinstance(strengths, list) or len(strengths) != 6 or not all(type(x) in (int, float) and math.isfinite(x) and 0 <= x <= 1 for x in strengths):
            raise ValueError('Invalid voice offsets or strengths')
        if not isinstance(vocab, dict) or not vocab or not all(isinstance(k, str) and len(k) == 1 and type(v) is int and 0 < v < 178 for k, v in vocab.items()) or len(set(vocab.values())) != len(vocab):
            raise ValueError('Invalid vocabulary')
        self.text.to(dev).eval().requires_grad_(False)
        self.joint.to(dev).eval().requires_grad_(False)
        self.styles, self.vocab, self.max_frames, self.device = styles, vocab, payload['max_frames'], dev
        self.parameter_counts = {'text': sum(p.numel() for p in self.text.parameters()),
                                 'joint': sum(p.numel() for p in self.joint.parameters())}

    def synthesize_phonemes(self, phones, voice, speed=1., seed=42):
        import torch
        if not isinstance(phones, str) or not 1 <= len(phones) <= 510 or any(c not in self.vocab for c in phones):
            raise ValueError('Expected 1..510 known phoneme characters')
        if voice not in VOICES or type(seed) is not int or not 0 <= seed < 2**32:
            raise ValueError('Invalid voice or seed')
        if type(speed) not in (int, float) or not math.isfinite(speed) or not 0.75 <= speed <= 1.25:
            raise ValueError('Speed must be finite and within 0.75..1.25')
        sid = VOICES.index(voice)
        ids = [0] + [self.vocab[c] for c in phones] + [0]
        tokens = torch.tensor([ids], device=self.device, dtype=torch.long)
        lengths = torch.tensor([len(ids)], device=self.device, dtype=torch.long)
        speakers = torch.tensor([sid], device=self.device, dtype=torch.long)
        devices = [self.device.index if self.device.index is not None else torch.cuda.current_device()] if self.device.type == 'cuda' else []
        with _LOCK, torch.no_grad():
            durations, _, _, _ = self.text(tokens, lengths, speaker_ids=speakers)
            if durations.shape != tokens.shape or not bool(torch.isfinite(durations).all()):
                raise ValueError('Invalid predicted duration')
            scaled = durations / speed
            if not bool(torch.isfinite(scaled).all()) or not bool((scaled.abs() < 1e7).all()):
                raise ValueError('Predicted duration overflow')
            counts = scaled.round().clamp(min=1)
            expected_frames = max(2, int(counts.sum()))
            if expected_frames > self.max_frames:
                raise ValueError('Predicted frame limit exceeded')
            result = self.text.infer(tokens, lengths, speakers, speed=speed)
            frames = result['asr'].shape[-1]
            if not torch.equal(result['dur'], durations) or frames != expected_frames or int(result['frame_lengths'][0]) != frames:
                raise ValueError('Unexpected predicted frames')
            if result['asr'].shape != (1, 512, frames) or result['F0'].shape != (1, 2 * frames) or result['N'].shape != (1, 2 * frames) or not all(bool(torch.isfinite(result[k]).all()) for k in ('asr', 'F0', 'N')):
                raise ValueError('Invalid predicted feature shapes or values')
            style = (self.styles['banks'][DONORS[sid]][len(ids) - 3].to(self.device) +
                     self.styles['offsets'][sid].to(self.device) * self.styles['strengths'][sid])[None]
            with torch.random.fork_rng(devices=devices):
                torch.manual_seed(seed)
                audio = self.joint(result['asr'], result['F0'], result['N'], style, sid)[0]
            if audio.ndim != 1 or len(audio) != 600 * frames or not bool(torch.isfinite(audio).all()):
                raise ValueError('Invalid generated waveform')
            audio = audio.detach().to(device='cpu', dtype=torch.float32)
        return {'audio': audio, 'sample_rate': 24000, 'metadata': {'voice': voice, 'seed': seed,
                'speed': speed, 'frames': frames, 'parameter_counts': dict(self.parameter_counts)}}


def load_export(directory, device='cpu', allow_cuda=False):
    import torch
    root, _ = verify_manifest(directory)
    payload = torch.load(root / 'weights.pt', map_location='cpu', weights_only=True)
    return FastMicro(payload, root, device=device, allow_cuda=allow_cuda)