File size: 2,526 Bytes
268e727 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | """Expose VoiceMem-Jev through Studio's routing lifecycle."""
from .cuda_runtime import BUCKETS, CompiledCudaRouter
from .adapter import VoiceDepthRouter
class CompiledVoiceDepthRouter(VoiceDepthRouter):
"""Keep host cancellation, caching and GPU admission around compiled calls."""
def _load(self):
import torch
from voicemem.engine import cpu_thread_budget
threads = cpu_thread_budget()
if torch.get_num_threads() != threads:
torch.set_num_threads(threads)
print(f'[jev] Torch CPU thread budget fixed before compilation: {threads}', flush=True)
runtime = CompiledCudaRouter(self.path, device=self.device)
runtime._load()
self._model, self._tokenizer = runtime.model, runtime.tokenizer
with torch.inference_mode():
for bucket in BUCKETS:
ids = torch.full((2, bucket), self._tokenizer.pad_token_id,
dtype=torch.long, device=self.device)
ids[:, 0] = self._tokenizer.cls_token_id
ids[:, -1] = self._tokenizer.sep_token_id
mask = torch.ones_like(ids)
observed = torch.zeros((1, 7), dtype=torch.float32, device=self.device)
for _ in range(3):
self._model(ids, mask, observed).float().cpu().numpy().copy()
if self._model.execution_mode == 'compiled-cuda':
print('[jev] VoiceMem-Jev: compiled CUDA runtime ready; all five buckets warm', flush=True)
else:
print('[jev] VoiceMem-Jev: same-weight eager CUDA fallback ready', flush=True)
def create(*, model, base_url='', api_key='', options=None):
"""Select the existing eager/ONNX adapter or opt into compiled CUDA."""
options = dict(options or {})
compiled = options.pop('compiled', False)
if type(compiled) is not bool:
raise ValueError('compiled must be a boolean')
if compiled and (options.get('backend', 'onnx') != 'eager'
or not str(options.get('device', 'cpu')).startswith('cuda')):
raise ValueError('Compiled inference requires eager CUDA')
from .adapter import create as ordinary
adapter = ordinary(model=model, base_url=base_url, api_key=api_key, options=options)
if not compiled:
return adapter
if adapter.backend != 'eager' or not adapter.device.startswith('cuda'):
raise ValueError('Compiled inference requires eager CUDA')
return CompiledVoiceDepthRouter(model, **options)
|