Inflect-Micro-v3 / models /micro /code /native27_nativegen_bridge.py
owensong's picture
Stage verified private early-access package
a775b3f verified
Raw History Blame Contribute Delete
13.5 kB
"""Pretrained Paradee-A trunk -> native512-channel hidden bridge.
No trainer/CLI, downloads, caches, or jobs. Use full utterances (no local crop or
padding): InstanceNorm makes the teacher hidden target context-dependent.
Hope/James are the initial active IDs; all six16D lookup rows are allocated.
The optional supplied native generator receives EXACT external128D styles.
"""
from pathlib import Path
import time
import torch
from torch import nn
import qualify_native27_native_generator_budget as budget
def require(ok, message):
if not ok:
raise ValueError(message)
def _native_tensors(module):
return {**{'parameter:'+k: v for k, v in module.named_parameters(remove_duplicate=False)},
**{'buffer:'+k: v for k, v in module.named_buffers(remove_duplicate=False)}}
def build_bridge(config):
"""Strict train/render factory: bridge-only state, never attaches nativeG."""
allowed = {'checkpoint', 'paradee_source', 'kokoro_source', 'active_speaker_ids', 'max_frames'}
require(isinstance(config, dict) and all(isinstance(k, str) for k in config), 'Factory config must be a string-keyed dict')
require(not set(config)-allowed, 'Unknown bridge factory config keys: '+','.join(sorted(set(config)-allowed)))
for key in ('checkpoint', 'paradee_source', 'kokoro_source'):
require(isinstance(config.get(key), (str, Path)) and str(config[key]).strip(), 'Explicit factory path required: '+key)
active = config.get('active_speaker_ids', (0, 3))
require(isinstance(active, (list, tuple)) and active
and all(type(sid) is int and 0 <= sid < 6 for sid in active)
and len(set(active)) == len(active), 'Invalid factory active_speaker_ids')
frames = config.get('max_frames', 1200)
require(type(frames) is int and 2 <= frames <= 4096, 'Invalid factory max_frames')
return NativeGeneratorBridge.from_pretrained(
Path(config['checkpoint']), Path(config['paradee_source']), Path(config['kokoro_source']),
native_generator=None, active_speaker_ids=tuple(active), max_frames=frames)
class NativeGeneratorBridge(nn.Module):
"""Consumes a strictly loaded donor decoder; removes its generator/style.
The supplied native generator is owned exclusively, frozen/eval, and retains
its original source and forward method. State guards include NONpersistent
buffers; CPU reference copies are diagnostic overhead, not deployed weights.
No generator affine hooks are needed for the initial Hope/James pilot.
"""
def __init__(self, donor_decoder, native_generator=None, *, active_speaker_ids=(0, 3),
max_frames=1200):
super().__init__()
require(isinstance(donor_decoder.style, nn.Parameter)
and donor_decoder.style.shape == (1, 16), 'Expected released16D donor style')
require(len(donor_decoder.decode) == 4
and donor_decoder.decode[-1].conv2.out_channels == 128
and donor_decoder.generator.ups[0].in_channels == 128,
'Expected Paradee-A128-channel trunk')
require(type(max_frames) is int and 2 <= max_frames <= 4096, 'Bounded full-utterance frame limit required')
active = tuple(active_speaker_ids)
require(active and len(set(active)) == len(active)
and all(type(i) is int and 0 <= i < 6 for i in active), 'Explicit valid active speaker IDs required')
style = donor_decoder.style.detach().clone()
require(style.device.type == 'cpu' and bool(torch.isfinite(style).all()), 'Finite CPU donor required')
del donor_decoder.generator
del donor_decoder.style
self.trunk = donor_decoder.requires_grad_(True)
self.speaker_embedding = nn.Embedding(6, 16)
with torch.no_grad():
self.speaker_embedding.weight.copy_(style.expand(6, -1))
self.projection = nn.Conv1d(128, 512, kernel_size=1, bias=True)
self.active_speaker_ids, self.max_frames = active, max_frames
self.native_generator = native_generator
self.provenance = dict(status='untrained-full-utterance-hidden-KD-prototype',
voices=list(budget.VOICES), active_speaker_ids=list(active),
text_model_included=False, teacher_style_approximated=False,
teacher_hidden='actual adapted native decoder pre-generator hidden[B,512,2T]',
input_context='FULLutterance; never crop/pad/reinterpret phoneme ASR as hidden')
self._native_snapshot = {}
self._native_source = None
self._native_forward_override = None
if native_generator is not None:
require(isinstance(native_generator, nn.Module), 'Native generator module required')
native_generator.eval().requires_grad_(False)
self._native_snapshot = {k: v.detach().cpu().clone() for k, v in _native_tensors(native_generator).items()}
require(all(bool(torch.isfinite(v).all()) for v in self._native_snapshot.values()), 'Nonfinite native state')
object.__setattr__(self, '_native_source', getattr(native_generator, 'm_source', None))
self._native_forward_override = native_generator.__dict__.get('forward')
@classmethod
def from_pretrained(cls, checkpoint, paradee_source, kokoro_source, native_generator=None, **kwargs):
"""Strict trusted-local CPU loader; no teacher model or inference call."""
require(str(torch.get_default_device()) == 'cpu', 'Load donor in a CPU-default process')
started = time.monotonic()
def deadline():
require(time.monotonic()-started < 120, 'CPU pretrained-load deadline exceeded')
paradee_source, kokoro_source = Path(paradee_source).resolve(), Path(kokoro_source).resolve()
pins = {}
for root, entries in ((paradee_source, budget.SOURCE_PINS), (kokoro_source/'kokoro', budget.KOKORO_PINS)):
for name, digest in entries.items():
pins[str(budget.pinned(root/name, digest, deadline))] = digest
path = budget.pinned(checkpoint, budget.PARADEE_PINS['pytorch/decoder.pt'], deadline)
pins[str(path)] = budget.PARADEE_PINS['pytorch/decoder.pt']
_, decoder_cls, _ = budget.load_types(paradee_source, kokoro_source)
state = torch.load(path, map_location='cpu', weights_only=True)
require(isinstance(state, dict) and set(state) == {'model'}, 'Unexpected released decoder checkpoint keys')
budget.tensor_state(torch, state['model'])
donor = decoder_cls('A')
donor.load_state_dict(state['model'], strict=True)
require(sum(p.numel() for p in donor.parameters()) == 3_848_578, 'Released A parameter count differs')
result = cls(donor, native_generator, **kwargs)
for filename, digest in pins.items():
require(budget.sha(filename) == digest, 'Pinned input changed during donor load')
deadline()
result.provenance.update(donor_checkpoint_sha256=pins[str(path)], input_sha256=pins,
helper_sha256=budget.sha(__file__))
return result
def train(self, mode=True):
super().train(mode)
if self.native_generator is not None:
self.native_generator.eval().requires_grad_(False)
return self
def trainable_parameters(self):
"""Optimizer input excludes the entire frozen native generator."""
return (p for p in self.parameters() if p.requires_grad)
def _features(self, asr, f0, noise):
require(isinstance(asr, torch.Tensor) and asr.ndim == 3
and asr.shape[0] > 0 and asr.shape[1] == 512
and 2 <= asr.shape[2] <= self.max_frames, 'Expected full asr[B,512,T] within frame bound')
require(isinstance(f0, torch.Tensor) and isinstance(noise, torch.Tensor)
and f0.shape == noise.shape == (asr.shape[0], 2*asr.shape[2]), 'F0/N must be[B,2T], exact original clock')
require(asr.dtype in (torch.float32, torch.float64)
and all(x.dtype == asr.dtype and x.device == asr.device for x in (f0, noise))
and self.projection.weight.device == asr.device
and self.projection.weight.dtype == asr.dtype, 'Matched model/input dtype/device required')
require(all(bool(torch.isfinite(x).all()) for x in (asr, f0, noise)), 'Nonfinite input features')
def forward_hidden(self, asr, f0, noise, speaker_ids):
"""Original decoder trunk clock/operations, then biased128->512.
No hidden teacher target is accepted here. KD compares this result to
separately cached REAL native pre-G hidden on the same FULL utterance.
Caller must not batch unequal utterances through padding.
"""
self._features(asr, f0, noise)
require(isinstance(speaker_ids, torch.Tensor) and speaker_ids.dtype == torch.long
and speaker_ids.shape == (asr.shape[0],) and speaker_ids.device == asr.device,
'Explicit LongTensor speaker_ids[B] on input device required')
require(bool(((speaker_ids >= 0) & (speaker_ids < 6)).all()), 'Speaker IDs outside six-row lookup')
allowed = torch.zeros_like(speaker_ids, dtype=torch.bool)
for sid in self.active_speaker_ids:
allowed |= speaker_ids == sid
require(bool(allowed.all()), 'Speaker not qualified for current two-voice prototype')
styles = self.speaker_embedding(speaker_ids)
F0 = self.trunk.F0_conv(f0.unsqueeze(1))
N = self.trunk.N_conv(noise.unsqueeze(1))
hidden = self.trunk.encode(torch.cat((asr, F0, N), dim=1), styles)
residual = self.trunk.asr_res(asr)
use_residual = True
for block in self.trunk.decode:
if use_residual:
hidden = torch.cat((hidden, residual, F0, N), dim=1)
hidden = block(hidden, styles)
if block.upsample_type != 'none':
use_residual = False
require(hidden.shape == (asr.shape[0],128,2*asr.shape[2]), 'Donor trunk clock/shape mismatch')
projected = self.projection(hidden)
require(projected.shape == (asr.shape[0],512,2*asr.shape[2])
and bool(torch.isfinite(projected).all()), 'Nonfinite/wrong projected hidden')
return projected
def forward(self, asr, f0, noise, speaker_ids):
return self.forward_hidden(asr, f0, noise, speaker_ids)
def assert_native_unchanged(self):
require(self.native_generator is not None, 'No supplied native generator')
module = self.native_generator
require(getattr(module, 'm_source', None) is self._native_source
and module.__dict__.get('forward') is self._native_forward_override,
'Native source/forward substituted')
require(not any(m.training for m in module.modules()) and not any(p.requires_grad for p in module.parameters()),
'Native generator must stay frozen/eval')
current = _native_tensors(module)
require(set(current) == set(self._native_snapshot), 'Native state keys changed')
changed = [k for k, value in current.items() if value.shape != self._native_snapshot[k].shape
or value.dtype != self._native_snapshot[k].dtype
or not torch.equal(value.detach().cpu(), self._native_snapshot[k])]
if changed:
# Fail closed, restoring only value mutations to owned native tensors.
with torch.no_grad():
for key in changed:
require(current[key].shape == self._native_snapshot[key].shape, 'Native state shape changed')
current[key].copy_(self._native_snapshot[key])
raise ValueError('Native state mutated (restored); synthesis rejected: '+','.join(changed))
return True
def synthesize(self, hidden, styles, f0):
"""Optional nativeG(hidden, EXACT cached style128, original f0).
Preserves gradients to hidden when requested, never to G parameters.
No gain/pitch/source changes, style lookup approximation, or hooks.
State guards are deliberately expensive; this is a qualification API.
"""
require(isinstance(hidden, torch.Tensor) and hidden.ndim == 3 and hidden.shape[0] > 0
and hidden.shape[1] == 512 and 4 <= hidden.shape[2] <= 2*self.max_frames
and hidden.shape[2] % 2 == 0, 'Expected projected hidden[B,512,2T]')
require(isinstance(styles, torch.Tensor) and isinstance(f0, torch.Tensor)
and styles.shape == (hidden.shape[0],128) and f0.shape == (hidden.shape[0],hidden.shape[2]),
'EXACT native styles[B,128] and F0[B,2T] required')
require(hidden.dtype in (torch.float32, torch.float64)
and all(x.dtype == hidden.dtype and x.device == hidden.device
and bool(torch.isfinite(x).all()) for x in (hidden, styles, f0)),
'Finite matched hidden/native styles/F0 required')
self.assert_native_unchanged()
try:
result = self.native_generator(hidden, styles, f0)
finally:
self.assert_native_unchanged()
require(isinstance(result, torch.Tensor) and result.shape[0] == hidden.shape[0]
and bool(torch.isfinite(result).all()), 'Invalid native synthesis output')
return result