File size: 11,291 Bytes
a775b3f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 | """Portable phoneme-only FastMicro 24K runtime. Requires PyTorch and packaged code."""
import hashlib
import importlib
import json
import math
from pathlib import Path
import sys
import threading
SCHEMA = 'inflect-fastmicro24k-phonemes-v1'
VOICES = ('hope', 'sophia', 'cassidy', 'james', 'jasper', 'mark')
DONORS = ('af_heart', 'af_heart', 'af_sarah', 'am_michael', 'am_michael', 'am_michael')
CODE_FILES = ('native_mini_inference.py', 'native26_text_model.py', 'native33_text_model.py',
'native33_text_fast.py', 'inflect_kokoro_student27.py', 'native27_nativegen_bridge.py',
'qualify_native27_native_generator_budget.py', 'student.py', 'student_decoder.py',
'kokoro/__init__.py', 'kokoro/model.py', 'kokoro/modules.py', 'kokoro/istftnet.py',
'kokoro/custom_stft.py', 'vhead.py', 'fastmicro_inference_20261029.py')
PAYLOAD_KEYS = {'schema', 'text_config', 'text_state', 'joint_state', 'styles', 'vocab',
'max_frames', 'sample_rate', 'provenance'}
_LOCK = threading.RLock()
def sha(path):
h = hashlib.sha256()
with open(path, 'rb') as f:
for chunk in iter(lambda: f.read(1024 * 1024), b''):
h.update(chunk)
return h.hexdigest()
def verify_manifest(directory):
root = Path(directory).resolve(strict=True)
if (root / 'manifest.json').is_symlink():
raise ValueError('Symlink manifest is not allowed')
manifest = json.loads((root / 'manifest.json').read_text(encoding='utf-8'))
files = manifest.get('files')
required = {'weights.pt'} | {'code/' + name for name in CODE_FILES}
if manifest.get('schema') != SCHEMA or not isinstance(files, dict) or not required <= set(files):
raise ValueError('Invalid export manifest or missing required files')
for name, digest in files.items():
if not isinstance(name, str) or '\\' in name or not isinstance(digest, str):
raise ValueError('Invalid manifest entry')
parts = name.split('/')
if any(p in ('', '.', '..') for p in parts) or name.startswith('/') or ':' in name:
raise ValueError('Unsafe export path')
path = root.joinpath(*parts)
if any(root.joinpath(*parts[:i]).is_symlink() for i in range(1, len(parts) + 1)):
raise ValueError('Symlink in export path')
if not path.is_file() or not path.resolve().is_relative_to(root) or sha(path) != digest:
raise ValueError('Export file hash mismatch: ' + name)
return root, manifest
def _code_path(root):
code = root / 'code'
# Avoid silently importing a different model's identically named modules.
names = {Path(name).parts[0].removesuffix('.py') for name in CODE_FILES}
for name, module in tuple(sys.modules.items()):
if name.split('.')[0] not in names or name == __name__:
continue
origin = getattr(module, '__file__', None)
if not origin or not Path(origin).resolve().is_relative_to(code):
raise RuntimeError('Conflicting preloaded module: ' + name)
if str(code) in sys.path:
sys.path.remove(str(code))
sys.path.insert(0, str(code))
def _make_joint(torch, mini, vhead, max_frames):
nn = torch.nn
class ConditionedHead(nn.Module):
def __init__(self):
super().__init__()
self.head = vhead.SourceFilterHead2(in_ch=128, dim=320, blocks=7)
self.head.blocks[-1] = vhead.ConvNeXt(320, mult=2)
self.style_affine = nn.Linear(128, 256)
def forward(self, hidden, style, f0):
scale, shift = self.style_affine(style).chunk(2, -1)
conditioned = hidden * (1 + 0.1 * torch.tanh(scale[:, :, None])) + shift[:, :, None]
return self.head(conditioned, f0, c0=None)
class Joint(nn.Module):
def __init__(self):
super().__init__()
self.bridge = mini.build_bridge(max_frames)
del self.bridge.projection
self.head = ConditionedHead()
def forward(self, asr, f0, noise, style, sid):
bridge, trunk = self.bridge, self.bridge.trunk
speakers = torch.full((asr.shape[0],), sid, device=asr.device, dtype=torch.long)
voice_style = bridge.speaker_embedding(speakers)
f0_conv, n_conv = trunk.F0_conv(f0[:, None]), trunk.N_conv(noise[:, None])
x = trunk.encode(torch.cat((asr, f0_conv, n_conv), 1), voice_style)
residual, use_residual = trunk.asr_res(asr), True
for block in trunk.decode:
if use_residual:
x = torch.cat((x, residual, f0_conv, n_conv), 1)
x = block(x, voice_style)
if block.upsample_type != 'none':
use_residual = False
if x.shape != (asr.shape[0], 128, f0.shape[1]):
raise ValueError('Unexpected trunk output shape')
return self.head(x, style, f0)
return Joint()
class FastMicro:
def __init__(self, payload, root, device='cpu', allow_cuda=False):
import torch
if set(payload) != PAYLOAD_KEYS or payload['schema'] != SCHEMA:
raise ValueError('Invalid payload schema')
dev = torch.device(device)
if dev.type != 'cpu' and not (dev.type == 'cuda' and allow_cuda):
raise ValueError('Only CPU or explicitly allowed CUDA is supported')
if payload['sample_rate'] != 24000 or type(payload['max_frames']) is not int or not 2 <= payload['max_frames'] <= 1200:
raise ValueError('Invalid sample rate or frame bound')
_code_path(root)
mini = importlib.import_module('native_mini_inference')
factory = importlib.import_module('native26_text_model')
vhead = importlib.import_module('vhead')
if payload['text_config'] != factory.default_config():
raise ValueError('Text configuration differs')
with torch.random.fork_rng(devices=[]):
self.text = factory.build_model(payload['text_config'])
self.joint = _make_joint(torch, mini, vhead, payload['max_frames'])
for model, state in ((self.text, payload['text_state']), (self.joint, payload['joint_state'])):
mini.tensor_state(state)
mini.strict_load(model, state)
styles, vocab = payload['styles'], payload['vocab']
if not isinstance(styles, dict) or set(styles) != {'banks', 'offsets', 'strengths'}:
raise ValueError('Invalid style schema')
if set(styles['banks']) != set(DONORS) or any(not torch.is_tensor(x) or x.shape != (510, 128) or x.dtype != torch.float32 or not bool(torch.isfinite(x).all()) for x in styles['banks'].values()):
raise ValueError('Invalid style banks')
offsets, strengths = styles['offsets'], styles['strengths']
if not torch.is_tensor(offsets) or offsets.shape != (6, 128) or offsets.dtype != torch.float32 or not bool(torch.isfinite(offsets).all()) or not isinstance(strengths, list) or len(strengths) != 6 or not all(type(x) in (int, float) and math.isfinite(x) and 0 <= x <= 1 for x in strengths):
raise ValueError('Invalid voice offsets or strengths')
if not isinstance(vocab, dict) or not vocab or not all(isinstance(k, str) and len(k) == 1 and type(v) is int and 0 < v < 178 for k, v in vocab.items()) or len(set(vocab.values())) != len(vocab):
raise ValueError('Invalid vocabulary')
self.text.to(dev).eval().requires_grad_(False)
self.joint.to(dev).eval().requires_grad_(False)
self.styles, self.vocab, self.max_frames, self.device = styles, vocab, payload['max_frames'], dev
self.parameter_counts = {'text': sum(p.numel() for p in self.text.parameters()),
'joint': sum(p.numel() for p in self.joint.parameters())}
def synthesize_phonemes(self, phones, voice, speed=1., seed=42):
import torch
if not isinstance(phones, str) or not 1 <= len(phones) <= 510 or any(c not in self.vocab for c in phones):
raise ValueError('Expected 1..510 known phoneme characters')
if voice not in VOICES or type(seed) is not int or not 0 <= seed < 2**32:
raise ValueError('Invalid voice or seed')
if type(speed) not in (int, float) or not math.isfinite(speed) or not 0.75 <= speed <= 1.25:
raise ValueError('Speed must be finite and within 0.75..1.25')
sid = VOICES.index(voice)
ids = [0] + [self.vocab[c] for c in phones] + [0]
tokens = torch.tensor([ids], device=self.device, dtype=torch.long)
lengths = torch.tensor([len(ids)], device=self.device, dtype=torch.long)
speakers = torch.tensor([sid], device=self.device, dtype=torch.long)
devices = [self.device.index if self.device.index is not None else torch.cuda.current_device()] if self.device.type == 'cuda' else []
with _LOCK, torch.no_grad():
durations, _, _, _ = self.text(tokens, lengths, speaker_ids=speakers)
if durations.shape != tokens.shape or not bool(torch.isfinite(durations).all()):
raise ValueError('Invalid predicted duration')
scaled = durations / speed
if not bool(torch.isfinite(scaled).all()) or not bool((scaled.abs() < 1e7).all()):
raise ValueError('Predicted duration overflow')
counts = scaled.round().clamp(min=1)
expected_frames = max(2, int(counts.sum()))
if expected_frames > self.max_frames:
raise ValueError('Predicted frame limit exceeded')
result = self.text.infer(tokens, lengths, speakers, speed=speed)
frames = result['asr'].shape[-1]
if not torch.equal(result['dur'], durations) or frames != expected_frames or int(result['frame_lengths'][0]) != frames:
raise ValueError('Unexpected predicted frames')
if result['asr'].shape != (1, 512, frames) or result['F0'].shape != (1, 2 * frames) or result['N'].shape != (1, 2 * frames) or not all(bool(torch.isfinite(result[k]).all()) for k in ('asr', 'F0', 'N')):
raise ValueError('Invalid predicted feature shapes or values')
style = (self.styles['banks'][DONORS[sid]][len(ids) - 3].to(self.device) +
self.styles['offsets'][sid].to(self.device) * self.styles['strengths'][sid])[None]
with torch.random.fork_rng(devices=devices):
torch.manual_seed(seed)
audio = self.joint(result['asr'], result['F0'], result['N'], style, sid)[0]
if audio.ndim != 1 or len(audio) != 600 * frames or not bool(torch.isfinite(audio).all()):
raise ValueError('Invalid generated waveform')
audio = audio.detach().to(device='cpu', dtype=torch.float32)
return {'audio': audio, 'sample_rate': 24000, 'metadata': {'voice': voice, 'seed': seed,
'speed': speed, 'frames': frames, 'parameter_counts': dict(self.parameter_counts)}}
def load_export(directory, device='cpu', allow_cuda=False):
import torch
root, _ = verify_manifest(directory)
payload = torch.load(root / 'weights.pt', map_location='cpu', weights_only=True)
return FastMicro(payload, root, device=device, allow_cuda=allow_cuda)
|