File size: 13,481 Bytes
a775b3f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 | """Pretrained Paradee-A trunk -> native512-channel hidden bridge.
No trainer/CLI, downloads, caches, or jobs. Use full utterances (no local crop or
padding): InstanceNorm makes the teacher hidden target context-dependent.
Hope/James are the initial active IDs; all six16D lookup rows are allocated.
The optional supplied native generator receives EXACT external128D styles.
"""
from pathlib import Path
import time
import torch
from torch import nn
import qualify_native27_native_generator_budget as budget
def require(ok, message):
if not ok:
raise ValueError(message)
def _native_tensors(module):
return {**{'parameter:'+k: v for k, v in module.named_parameters(remove_duplicate=False)},
**{'buffer:'+k: v for k, v in module.named_buffers(remove_duplicate=False)}}
def build_bridge(config):
"""Strict train/render factory: bridge-only state, never attaches nativeG."""
allowed = {'checkpoint', 'paradee_source', 'kokoro_source', 'active_speaker_ids', 'max_frames'}
require(isinstance(config, dict) and all(isinstance(k, str) for k in config), 'Factory config must be a string-keyed dict')
require(not set(config)-allowed, 'Unknown bridge factory config keys: '+','.join(sorted(set(config)-allowed)))
for key in ('checkpoint', 'paradee_source', 'kokoro_source'):
require(isinstance(config.get(key), (str, Path)) and str(config[key]).strip(), 'Explicit factory path required: '+key)
active = config.get('active_speaker_ids', (0, 3))
require(isinstance(active, (list, tuple)) and active
and all(type(sid) is int and 0 <= sid < 6 for sid in active)
and len(set(active)) == len(active), 'Invalid factory active_speaker_ids')
frames = config.get('max_frames', 1200)
require(type(frames) is int and 2 <= frames <= 4096, 'Invalid factory max_frames')
return NativeGeneratorBridge.from_pretrained(
Path(config['checkpoint']), Path(config['paradee_source']), Path(config['kokoro_source']),
native_generator=None, active_speaker_ids=tuple(active), max_frames=frames)
class NativeGeneratorBridge(nn.Module):
"""Consumes a strictly loaded donor decoder; removes its generator/style.
The supplied native generator is owned exclusively, frozen/eval, and retains
its original source and forward method. State guards include NONpersistent
buffers; CPU reference copies are diagnostic overhead, not deployed weights.
No generator affine hooks are needed for the initial Hope/James pilot.
"""
def __init__(self, donor_decoder, native_generator=None, *, active_speaker_ids=(0, 3),
max_frames=1200):
super().__init__()
require(isinstance(donor_decoder.style, nn.Parameter)
and donor_decoder.style.shape == (1, 16), 'Expected released16D donor style')
require(len(donor_decoder.decode) == 4
and donor_decoder.decode[-1].conv2.out_channels == 128
and donor_decoder.generator.ups[0].in_channels == 128,
'Expected Paradee-A128-channel trunk')
require(type(max_frames) is int and 2 <= max_frames <= 4096, 'Bounded full-utterance frame limit required')
active = tuple(active_speaker_ids)
require(active and len(set(active)) == len(active)
and all(type(i) is int and 0 <= i < 6 for i in active), 'Explicit valid active speaker IDs required')
style = donor_decoder.style.detach().clone()
require(style.device.type == 'cpu' and bool(torch.isfinite(style).all()), 'Finite CPU donor required')
del donor_decoder.generator
del donor_decoder.style
self.trunk = donor_decoder.requires_grad_(True)
self.speaker_embedding = nn.Embedding(6, 16)
with torch.no_grad():
self.speaker_embedding.weight.copy_(style.expand(6, -1))
self.projection = nn.Conv1d(128, 512, kernel_size=1, bias=True)
self.active_speaker_ids, self.max_frames = active, max_frames
self.native_generator = native_generator
self.provenance = dict(status='untrained-full-utterance-hidden-KD-prototype',
voices=list(budget.VOICES), active_speaker_ids=list(active),
text_model_included=False, teacher_style_approximated=False,
teacher_hidden='actual adapted native decoder pre-generator hidden[B,512,2T]',
input_context='FULLutterance; never crop/pad/reinterpret phoneme ASR as hidden')
self._native_snapshot = {}
self._native_source = None
self._native_forward_override = None
if native_generator is not None:
require(isinstance(native_generator, nn.Module), 'Native generator module required')
native_generator.eval().requires_grad_(False)
self._native_snapshot = {k: v.detach().cpu().clone() for k, v in _native_tensors(native_generator).items()}
require(all(bool(torch.isfinite(v).all()) for v in self._native_snapshot.values()), 'Nonfinite native state')
object.__setattr__(self, '_native_source', getattr(native_generator, 'm_source', None))
self._native_forward_override = native_generator.__dict__.get('forward')
@classmethod
def from_pretrained(cls, checkpoint, paradee_source, kokoro_source, native_generator=None, **kwargs):
"""Strict trusted-local CPU loader; no teacher model or inference call."""
require(str(torch.get_default_device()) == 'cpu', 'Load donor in a CPU-default process')
started = time.monotonic()
def deadline():
require(time.monotonic()-started < 120, 'CPU pretrained-load deadline exceeded')
paradee_source, kokoro_source = Path(paradee_source).resolve(), Path(kokoro_source).resolve()
pins = {}
for root, entries in ((paradee_source, budget.SOURCE_PINS), (kokoro_source/'kokoro', budget.KOKORO_PINS)):
for name, digest in entries.items():
pins[str(budget.pinned(root/name, digest, deadline))] = digest
path = budget.pinned(checkpoint, budget.PARADEE_PINS['pytorch/decoder.pt'], deadline)
pins[str(path)] = budget.PARADEE_PINS['pytorch/decoder.pt']
_, decoder_cls, _ = budget.load_types(paradee_source, kokoro_source)
state = torch.load(path, map_location='cpu', weights_only=True)
require(isinstance(state, dict) and set(state) == {'model'}, 'Unexpected released decoder checkpoint keys')
budget.tensor_state(torch, state['model'])
donor = decoder_cls('A')
donor.load_state_dict(state['model'], strict=True)
require(sum(p.numel() for p in donor.parameters()) == 3_848_578, 'Released A parameter count differs')
result = cls(donor, native_generator, **kwargs)
for filename, digest in pins.items():
require(budget.sha(filename) == digest, 'Pinned input changed during donor load')
deadline()
result.provenance.update(donor_checkpoint_sha256=pins[str(path)], input_sha256=pins,
helper_sha256=budget.sha(__file__))
return result
def train(self, mode=True):
super().train(mode)
if self.native_generator is not None:
self.native_generator.eval().requires_grad_(False)
return self
def trainable_parameters(self):
"""Optimizer input excludes the entire frozen native generator."""
return (p for p in self.parameters() if p.requires_grad)
def _features(self, asr, f0, noise):
require(isinstance(asr, torch.Tensor) and asr.ndim == 3
and asr.shape[0] > 0 and asr.shape[1] == 512
and 2 <= asr.shape[2] <= self.max_frames, 'Expected full asr[B,512,T] within frame bound')
require(isinstance(f0, torch.Tensor) and isinstance(noise, torch.Tensor)
and f0.shape == noise.shape == (asr.shape[0], 2*asr.shape[2]), 'F0/N must be[B,2T], exact original clock')
require(asr.dtype in (torch.float32, torch.float64)
and all(x.dtype == asr.dtype and x.device == asr.device for x in (f0, noise))
and self.projection.weight.device == asr.device
and self.projection.weight.dtype == asr.dtype, 'Matched model/input dtype/device required')
require(all(bool(torch.isfinite(x).all()) for x in (asr, f0, noise)), 'Nonfinite input features')
def forward_hidden(self, asr, f0, noise, speaker_ids):
"""Original decoder trunk clock/operations, then biased128->512.
No hidden teacher target is accepted here. KD compares this result to
separately cached REAL native pre-G hidden on the same FULL utterance.
Caller must not batch unequal utterances through padding.
"""
self._features(asr, f0, noise)
require(isinstance(speaker_ids, torch.Tensor) and speaker_ids.dtype == torch.long
and speaker_ids.shape == (asr.shape[0],) and speaker_ids.device == asr.device,
'Explicit LongTensor speaker_ids[B] on input device required')
require(bool(((speaker_ids >= 0) & (speaker_ids < 6)).all()), 'Speaker IDs outside six-row lookup')
allowed = torch.zeros_like(speaker_ids, dtype=torch.bool)
for sid in self.active_speaker_ids:
allowed |= speaker_ids == sid
require(bool(allowed.all()), 'Speaker not qualified for current two-voice prototype')
styles = self.speaker_embedding(speaker_ids)
F0 = self.trunk.F0_conv(f0.unsqueeze(1))
N = self.trunk.N_conv(noise.unsqueeze(1))
hidden = self.trunk.encode(torch.cat((asr, F0, N), dim=1), styles)
residual = self.trunk.asr_res(asr)
use_residual = True
for block in self.trunk.decode:
if use_residual:
hidden = torch.cat((hidden, residual, F0, N), dim=1)
hidden = block(hidden, styles)
if block.upsample_type != 'none':
use_residual = False
require(hidden.shape == (asr.shape[0],128,2*asr.shape[2]), 'Donor trunk clock/shape mismatch')
projected = self.projection(hidden)
require(projected.shape == (asr.shape[0],512,2*asr.shape[2])
and bool(torch.isfinite(projected).all()), 'Nonfinite/wrong projected hidden')
return projected
def forward(self, asr, f0, noise, speaker_ids):
return self.forward_hidden(asr, f0, noise, speaker_ids)
def assert_native_unchanged(self):
require(self.native_generator is not None, 'No supplied native generator')
module = self.native_generator
require(getattr(module, 'm_source', None) is self._native_source
and module.__dict__.get('forward') is self._native_forward_override,
'Native source/forward substituted')
require(not any(m.training for m in module.modules()) and not any(p.requires_grad for p in module.parameters()),
'Native generator must stay frozen/eval')
current = _native_tensors(module)
require(set(current) == set(self._native_snapshot), 'Native state keys changed')
changed = [k for k, value in current.items() if value.shape != self._native_snapshot[k].shape
or value.dtype != self._native_snapshot[k].dtype
or not torch.equal(value.detach().cpu(), self._native_snapshot[k])]
if changed:
# Fail closed, restoring only value mutations to owned native tensors.
with torch.no_grad():
for key in changed:
require(current[key].shape == self._native_snapshot[key].shape, 'Native state shape changed')
current[key].copy_(self._native_snapshot[key])
raise ValueError('Native state mutated (restored); synthesis rejected: '+','.join(changed))
return True
def synthesize(self, hidden, styles, f0):
"""Optional nativeG(hidden, EXACT cached style128, original f0).
Preserves gradients to hidden when requested, never to G parameters.
No gain/pitch/source changes, style lookup approximation, or hooks.
State guards are deliberately expensive; this is a qualification API.
"""
require(isinstance(hidden, torch.Tensor) and hidden.ndim == 3 and hidden.shape[0] > 0
and hidden.shape[1] == 512 and 4 <= hidden.shape[2] <= 2*self.max_frames
and hidden.shape[2] % 2 == 0, 'Expected projected hidden[B,512,2T]')
require(isinstance(styles, torch.Tensor) and isinstance(f0, torch.Tensor)
and styles.shape == (hidden.shape[0],128) and f0.shape == (hidden.shape[0],hidden.shape[2]),
'EXACT native styles[B,128] and F0[B,2T] required')
require(hidden.dtype in (torch.float32, torch.float64)
and all(x.dtype == hidden.dtype and x.device == hidden.device
and bool(torch.isfinite(x).all()) for x in (hidden, styles, f0)),
'Finite matched hidden/native styles/F0 required')
self.assert_native_unchanged()
try:
result = self.native_generator(hidden, styles, f0)
finally:
self.assert_native_unchanged()
require(isinstance(result, torch.Tensor) and result.shape[0] == hidden.shape[0]
and bool(torch.isfinite(result).all()), 'Invalid native synthesis output')
return result
|