File size: 7,887 Bytes
07b13d3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
"""Private selected Mini delivery and optional native48 runtime API."""
import importlib.util
import json
from pathlib import Path

import torch


def load_module(path,name):
    spec=importlib.util.spec_from_file_location(name,path)
    module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
    return module


class MiniDelivery:
    """Cassidy Q1500 only; other voices keep exact original pitch and timing."""
    def __init__(self,base,package,*,polish=True,runtime='canonical'):
        self.base=base;self.root=Path(package);self.vocab=base.vocab;self.polish=polish
        self.runtime=runtime;self.pitch=None;self.cuda_generator=None
        self.single=load_module(self.root/'runtimes/mini_singlepass_20261027.py','_selected_singlepass')
        self.pitch_code=load_module(self.root/'components/pitch/pitch.py','_selected_pitch')
        self.device=next(base.parameters()).device
        if runtime=='cuda-source':
            if self.device.type!='cuda':raise ValueError('cuda-source runtime requires explicit CUDA')
            self.cuda_code=load_module(self.root/'runtimes/mini_cuda_source_20261027.py','_selected_cuda_source')
            self.cuda_generator=self.cuda_code.build_candidate(base,self.root/'baseline/models/mini')
        elif runtime!='canonical':raise ValueError('Unknown Mini runtime')

    def _load_pitch(self):
        if self.pitch is None:
            # Construction must not disturb a caller's noise seeds.
            with torch.random.fork_rng(devices=[]):
                self.pitch=self.pitch_code.FocusPitch(self.base.text.predictor)
            self.pitch.load_state_dict(torch.load(self.root/'components/pitch/weights.pt',map_location='cpu',weights_only=True),strict=True)
            self.pitch.to(self.device).eval().requires_grad_(False)
            assert sum(p.numel() for p in self.pitch.parameters())==3801977

    def synthesize_ids(self,ids,voice='cassidy',*,seed=42):
        import native_mini_inference as native
        sid=native.VOICES.index(voice) if isinstance(voice,str) and voice in native.VOICES else voice
        self.single.validate_request(ids,sid,1.,seed,native.VOICES)
        policy=dict(selected='original',cassidy_only=True,clipped_fraction=0.,fallback_reason=None,
                    runtime=self.runtime,source_audio_at_inference=False)
        hook=None;context=[]
        with self.base._inference_lock,native._RNG_LOCK,torch.inference_mode():
            try:
                observer=None
                if self.polish and sid==2:
                    self._load_pitch()
                    def capture(_module,_inputs,output):context.append(output.transpose(-1,-2))
                    hook=self.base.text.bert_encoder.register_forward_hook(capture)
                    def observer(predicted):
                        if len(context)!=1:raise RuntimeError('Expected one exact text-context pass')
                        tokens=torch.tensor([ids],device=self.device);speakers=torch.tensor([sid],device=self.device)
                        raw=self.pitch(context[0],self.base.text.speaker_embedding(speakers),predicted['aln'],tokens)
                        original=predicted['F0'];voiced=(original>60)&(original<600)
                        if not bool(torch.isfinite(raw).all()):
                            policy['fallback_reason']='nonfinite preferred-pitch prediction';return
                        bad=voiced&((raw<50)|(raw>600))
                        clipped=float(bad.sum()/voiced.sum().clamp_min(1));policy['clipped_fraction']=clipped
                        if clipped>=.01:
                            policy['fallback_reason']='preferred-pitch physical clipping fraction >=0.01';return
                        value=torch.where(voiced,raw.clamp(50,600),original)
                        if value.shape!=original.shape:raise RuntimeError('Pitch clock mismatch')
                        # The helper's local F0 aliases this tensor; weights and native durations are untouched.
                        original.copy_(value);policy['selected']='cassidy-Q1500'
                if self.runtime=='cuda-source':
                    with self.cuda_code.select_generator(self.base,self.cuda_generator,True):
                        result=self.single.synthesize_ids(self.base,ids,sid,seed=seed,_feature_observer=observer)
                else:result=self.single.synthesize_ids(self.base,ids,sid,seed=seed,_feature_observer=observer)
            finally:
                if hook is not None:hook.remove()
        result['metadata']=dict(result['metadata'],delivery=policy)
        return result

    def synthesize_phonemes(self,phones,voice='cassidy',*,seed=42):
        return self.synthesize_ids(self.base.phoneme_ids(phones),voice,seed=seed)


class Native48Enhancer:
    """Selected GAN1000 only. No teacher, discriminator, denoiser or remote assets."""
    def __init__(self,package,device='cpu'):
        self.root=Path(package);self.device=torch.device(device)
        if self.device.type not in ('cpu','cuda'):raise ValueError('CPU or CUDA only')
        code=load_module(self.root/'components/enhancer/enhancer.py','_selected_enhancer')
        state=torch.load(self.root/'components/enhancer/weights.pt',map_location='cpu',weights_only=True)
        base={name[5:]:value for name,value in state.items() if name.startswith('base.')}
        with torch.random.fork_rng(devices=[]):self.model=code.DetailStudent(base)
        self.model.load_state_dict(state,strict=True);self.model.to(self.device).eval().requires_grad_(False)
        if sum(p.numel() for p in self.model.parameters())!=2846722:raise ValueError('Enhancer parameter count mismatch')

    def enhance(self,source24,sample_rate=24000):
        if sample_rate!=24000:raise ValueError('Actual native 24000 Hz source required')
        if not isinstance(source24,torch.Tensor) or source24.ndim!=1 or source24.dtype!=torch.float32:raise ValueError('One float32 mono vector required')
        if not 65<=source24.numel()<=720000:raise ValueError('Enhancer supports 65..720000 source samples; split text before synthesis')
        if not bool(torch.isfinite(source24).all()) or float(source24.abs().max())>=1:raise ValueError('Nonfinite or clipped source; no normalization repair')
        with torch.inference_mode(),torch.autocast(device_type=self.device.type,enabled=False):
            x=source24.to(self.device)[None,None]
            y,reference,_=self.model(x,True)
            if y.shape!=(1,1,2*len(source24)) or not bool(torch.isfinite(y).all()) or float(y.abs().max())>=1:raise ValueError('Enhancer native48 finite/length/clipping check failed')
            baseline=self.model.stft(self.model.baseline(x));actual=self.model.stft(y);ref=self.model.stft(reference)
            quiet=self.model.frame_rms(baseline)<1e-4
            if bool(((self.model.frame_rms(actual)>.001)&quiet).any()):raise ValueError('Source-quiet preservation failed')
            low=torch.fft.rfftfreq(1024,1/48000,device=self.device)<=8000
            p=actual[:,low].to(torch.complex128);r=ref[:,low].to(torch.complex128)
            pe=p.abs().square().sum();re=r.abs().square().sum()
            cosine=1. if float(pe)<1e-18 and float(re)<1e-18 else float((p*r.conj()).real.sum()/(pe*re).sqrt().clamp_min(1e-20))
            db=float(10*torch.log10((pe+1e-20)/(re+1e-20)))
            if cosine<=.999 or abs(db)>=.05:raise ValueError('Selected-study low-band preservation check failed')
            if not bool(torch.count_nonzero(x)) and bool(torch.count_nonzero(y)):raise ValueError('Exact zero silence failed')
            return y[0,0].detach().cpu(),dict(sample_rate=48000,parameters=2846722,cosine_to_previous731k=cosine,lowband_db_to_previous731k=db,
                normalized=False,teacher_at_inference=False,runtime_checks='finite/length/clipping/source-quiet/lowband; not teacher-dependent quality metrics')