File size: 13,481 Bytes
a775b3f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
"""Pretrained Paradee-A trunk -> native512-channel hidden bridge.

No trainer/CLI, downloads, caches, or jobs. Use full utterances (no local crop or
padding): InstanceNorm makes the teacher hidden target context-dependent.
Hope/James are the initial active IDs; all six16D lookup rows are allocated.
The optional supplied native generator receives EXACT external128D styles.
"""
from pathlib import Path
import time

import torch
from torch import nn

import qualify_native27_native_generator_budget as budget


def require(ok, message):
    if not ok:
        raise ValueError(message)


def _native_tensors(module):
    return {**{'parameter:'+k: v for k, v in module.named_parameters(remove_duplicate=False)},
            **{'buffer:'+k: v for k, v in module.named_buffers(remove_duplicate=False)}}


def build_bridge(config):
    """Strict train/render factory: bridge-only state, never attaches nativeG."""
    allowed = {'checkpoint', 'paradee_source', 'kokoro_source', 'active_speaker_ids', 'max_frames'}
    require(isinstance(config, dict) and all(isinstance(k, str) for k in config), 'Factory config must be a string-keyed dict')
    require(not set(config)-allowed, 'Unknown bridge factory config keys: '+','.join(sorted(set(config)-allowed)))
    for key in ('checkpoint', 'paradee_source', 'kokoro_source'):
        require(isinstance(config.get(key), (str, Path)) and str(config[key]).strip(), 'Explicit factory path required: '+key)
    active = config.get('active_speaker_ids', (0, 3))
    require(isinstance(active, (list, tuple)) and active
            and all(type(sid) is int and 0 <= sid < 6 for sid in active)
            and len(set(active)) == len(active), 'Invalid factory active_speaker_ids')
    frames = config.get('max_frames', 1200)
    require(type(frames) is int and 2 <= frames <= 4096, 'Invalid factory max_frames')
    return NativeGeneratorBridge.from_pretrained(
        Path(config['checkpoint']), Path(config['paradee_source']), Path(config['kokoro_source']),
        native_generator=None, active_speaker_ids=tuple(active), max_frames=frames)


class NativeGeneratorBridge(nn.Module):
    """Consumes a strictly loaded donor decoder; removes its generator/style.

    The supplied native generator is owned exclusively, frozen/eval, and retains
    its original source and forward method. State guards include NONpersistent
    buffers; CPU reference copies are diagnostic overhead, not deployed weights.
    No generator affine hooks are needed for the initial Hope/James pilot.
    """

    def __init__(self, donor_decoder, native_generator=None, *, active_speaker_ids=(0, 3),
                 max_frames=1200):
        super().__init__()
        require(isinstance(donor_decoder.style, nn.Parameter)
                and donor_decoder.style.shape == (1, 16), 'Expected released16D donor style')
        require(len(donor_decoder.decode) == 4
                and donor_decoder.decode[-1].conv2.out_channels == 128
                and donor_decoder.generator.ups[0].in_channels == 128,
                'Expected Paradee-A128-channel trunk')
        require(type(max_frames) is int and 2 <= max_frames <= 4096, 'Bounded full-utterance frame limit required')
        active = tuple(active_speaker_ids)
        require(active and len(set(active)) == len(active)
                and all(type(i) is int and 0 <= i < 6 for i in active), 'Explicit valid active speaker IDs required')
        style = donor_decoder.style.detach().clone()
        require(style.device.type == 'cpu' and bool(torch.isfinite(style).all()), 'Finite CPU donor required')
        del donor_decoder.generator
        del donor_decoder.style
        self.trunk = donor_decoder.requires_grad_(True)
        self.speaker_embedding = nn.Embedding(6, 16)
        with torch.no_grad():
            self.speaker_embedding.weight.copy_(style.expand(6, -1))
        self.projection = nn.Conv1d(128, 512, kernel_size=1, bias=True)
        self.active_speaker_ids, self.max_frames = active, max_frames
        self.native_generator = native_generator
        self.provenance = dict(status='untrained-full-utterance-hidden-KD-prototype',
                               voices=list(budget.VOICES), active_speaker_ids=list(active),
                               text_model_included=False, teacher_style_approximated=False,
                               teacher_hidden='actual adapted native decoder pre-generator hidden[B,512,2T]',
                               input_context='FULLutterance; never crop/pad/reinterpret phoneme ASR as hidden')
        self._native_snapshot = {}
        self._native_source = None
        self._native_forward_override = None
        if native_generator is not None:
            require(isinstance(native_generator, nn.Module), 'Native generator module required')
            native_generator.eval().requires_grad_(False)
            self._native_snapshot = {k: v.detach().cpu().clone() for k, v in _native_tensors(native_generator).items()}
            require(all(bool(torch.isfinite(v).all()) for v in self._native_snapshot.values()), 'Nonfinite native state')
            object.__setattr__(self, '_native_source', getattr(native_generator, 'm_source', None))
            self._native_forward_override = native_generator.__dict__.get('forward')

    @classmethod
    def from_pretrained(cls, checkpoint, paradee_source, kokoro_source, native_generator=None, **kwargs):
        """Strict trusted-local CPU loader; no teacher model or inference call."""
        require(str(torch.get_default_device()) == 'cpu', 'Load donor in a CPU-default process')
        started = time.monotonic()

        def deadline():
            require(time.monotonic()-started < 120, 'CPU pretrained-load deadline exceeded')

        paradee_source, kokoro_source = Path(paradee_source).resolve(), Path(kokoro_source).resolve()
        pins = {}
        for root, entries in ((paradee_source, budget.SOURCE_PINS), (kokoro_source/'kokoro', budget.KOKORO_PINS)):
            for name, digest in entries.items():
                pins[str(budget.pinned(root/name, digest, deadline))] = digest
        path = budget.pinned(checkpoint, budget.PARADEE_PINS['pytorch/decoder.pt'], deadline)
        pins[str(path)] = budget.PARADEE_PINS['pytorch/decoder.pt']
        _, decoder_cls, _ = budget.load_types(paradee_source, kokoro_source)
        state = torch.load(path, map_location='cpu', weights_only=True)
        require(isinstance(state, dict) and set(state) == {'model'}, 'Unexpected released decoder checkpoint keys')
        budget.tensor_state(torch, state['model'])
        donor = decoder_cls('A')
        donor.load_state_dict(state['model'], strict=True)
        require(sum(p.numel() for p in donor.parameters()) == 3_848_578, 'Released A parameter count differs')
        result = cls(donor, native_generator, **kwargs)
        for filename, digest in pins.items():
            require(budget.sha(filename) == digest, 'Pinned input changed during donor load')
        deadline()
        result.provenance.update(donor_checkpoint_sha256=pins[str(path)], input_sha256=pins,
                                 helper_sha256=budget.sha(__file__))
        return result

    def train(self, mode=True):
        super().train(mode)
        if self.native_generator is not None:
            self.native_generator.eval().requires_grad_(False)
        return self

    def trainable_parameters(self):
        """Optimizer input excludes the entire frozen native generator."""
        return (p for p in self.parameters() if p.requires_grad)

    def _features(self, asr, f0, noise):
        require(isinstance(asr, torch.Tensor) and asr.ndim == 3
                and asr.shape[0] > 0 and asr.shape[1] == 512
                and 2 <= asr.shape[2] <= self.max_frames, 'Expected full asr[B,512,T] within frame bound')
        require(isinstance(f0, torch.Tensor) and isinstance(noise, torch.Tensor)
                and f0.shape == noise.shape == (asr.shape[0], 2*asr.shape[2]), 'F0/N must be[B,2T], exact original clock')
        require(asr.dtype in (torch.float32, torch.float64)
                and all(x.dtype == asr.dtype and x.device == asr.device for x in (f0, noise))
                and self.projection.weight.device == asr.device
                and self.projection.weight.dtype == asr.dtype, 'Matched model/input dtype/device required')
        require(all(bool(torch.isfinite(x).all()) for x in (asr, f0, noise)), 'Nonfinite input features')

    def forward_hidden(self, asr, f0, noise, speaker_ids):
        """Original decoder trunk clock/operations, then biased128->512.

        No hidden teacher target is accepted here. KD compares this result to
        separately cached REAL native pre-G hidden on the same FULL utterance.
        Caller must not batch unequal utterances through padding.
        """
        self._features(asr, f0, noise)
        require(isinstance(speaker_ids, torch.Tensor) and speaker_ids.dtype == torch.long
                and speaker_ids.shape == (asr.shape[0],) and speaker_ids.device == asr.device,
                'Explicit LongTensor speaker_ids[B] on input device required')
        require(bool(((speaker_ids >= 0) & (speaker_ids < 6)).all()), 'Speaker IDs outside six-row lookup')
        allowed = torch.zeros_like(speaker_ids, dtype=torch.bool)
        for sid in self.active_speaker_ids:
            allowed |= speaker_ids == sid
        require(bool(allowed.all()), 'Speaker not qualified for current two-voice prototype')
        styles = self.speaker_embedding(speaker_ids)
        F0 = self.trunk.F0_conv(f0.unsqueeze(1))
        N = self.trunk.N_conv(noise.unsqueeze(1))
        hidden = self.trunk.encode(torch.cat((asr, F0, N), dim=1), styles)
        residual = self.trunk.asr_res(asr)
        use_residual = True
        for block in self.trunk.decode:
            if use_residual:
                hidden = torch.cat((hidden, residual, F0, N), dim=1)
            hidden = block(hidden, styles)
            if block.upsample_type != 'none':
                use_residual = False
        require(hidden.shape == (asr.shape[0],128,2*asr.shape[2]), 'Donor trunk clock/shape mismatch')
        projected = self.projection(hidden)
        require(projected.shape == (asr.shape[0],512,2*asr.shape[2])
                and bool(torch.isfinite(projected).all()), 'Nonfinite/wrong projected hidden')
        return projected

    def forward(self, asr, f0, noise, speaker_ids):
        return self.forward_hidden(asr, f0, noise, speaker_ids)

    def assert_native_unchanged(self):
        require(self.native_generator is not None, 'No supplied native generator')
        module = self.native_generator
        require(getattr(module, 'm_source', None) is self._native_source
                and module.__dict__.get('forward') is self._native_forward_override,
                'Native source/forward substituted')
        require(not any(m.training for m in module.modules()) and not any(p.requires_grad for p in module.parameters()),
                'Native generator must stay frozen/eval')
        current = _native_tensors(module)
        require(set(current) == set(self._native_snapshot), 'Native state keys changed')
        changed = [k for k, value in current.items() if value.shape != self._native_snapshot[k].shape
                   or value.dtype != self._native_snapshot[k].dtype
                   or not torch.equal(value.detach().cpu(), self._native_snapshot[k])]
        if changed:
            # Fail closed, restoring only value mutations to owned native tensors.
            with torch.no_grad():
                for key in changed:
                    require(current[key].shape == self._native_snapshot[key].shape, 'Native state shape changed')
                    current[key].copy_(self._native_snapshot[key])
            raise ValueError('Native state mutated (restored); synthesis rejected: '+','.join(changed))
        return True

    def synthesize(self, hidden, styles, f0):
        """Optional nativeG(hidden, EXACT cached style128, original f0).

        Preserves gradients to hidden when requested, never to G parameters.
        No gain/pitch/source changes, style lookup approximation, or hooks.
        State guards are deliberately expensive; this is a qualification API.
        """
        require(isinstance(hidden, torch.Tensor) and hidden.ndim == 3 and hidden.shape[0] > 0
                and hidden.shape[1] == 512 and 4 <= hidden.shape[2] <= 2*self.max_frames
                and hidden.shape[2] % 2 == 0, 'Expected projected hidden[B,512,2T]')
        require(isinstance(styles, torch.Tensor) and isinstance(f0, torch.Tensor)
                and styles.shape == (hidden.shape[0],128) and f0.shape == (hidden.shape[0],hidden.shape[2]),
                'EXACT native styles[B,128] and F0[B,2T] required')
        require(hidden.dtype in (torch.float32, torch.float64)
                and all(x.dtype == hidden.dtype and x.device == hidden.device
                        and bool(torch.isfinite(x).all()) for x in (hidden, styles, f0)),
                'Finite matched hidden/native styles/F0 required')
        self.assert_native_unchanged()
        try:
            result = self.native_generator(hidden, styles, f0)
        finally:
            self.assert_native_unchanged()
        require(isinstance(result, torch.Tensor) and result.shape[0] == hidden.shape[0]
                and bool(torch.isfinite(result).all()), 'Invalid native synthesis output')
        return result