scripts/omni_o_terminal.py

import argparse, os, sys, io, time, torch, threading, queue, logging, contextlib, warnings
import numpy as np

warnings.filterwarnings('ignore')
logging.getLogger().setLevel(logging.ERROR)

sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
from models.vam import VAM, VAMConfig
from serve.realtime import SileroVAD

SAMPLE_RATE = 16000
AUDIO_SR = 24000


class RealtimeRecorder:
    def __init__(self, vad, threshold=0.5, min_speech_ms=128, min_silence_ms=800, mic=None):
        self.vad = vad
        self.threshold = threshold
        self.min_speech = int(SAMPLE_RATE * min_speech_ms / 1000)
        self.min_silence = int(SAMPLE_RATE * min_silence_ms / 1000)
        self.mic = mic
        self.q = queue.Queue()
        self.lock = threading.Lock()
        self.reset()

    def reset(self):
        self.state = 'idle'
        self.buffer = []
        self.ring = []
        self.speaking = False
        self.speech_samples = 0
        self.silence_samples = 0
        self.tail_silence = 0
        self.interrupt = False

    def _feed(self, w):
        prob = self.vad(w, SAMPLE_RATE)
        with self.lock:
            if prob > self.threshold:
                self.silence_samples = self.tail_silence = 0
                self.speech_samples += len(w)
                self.buffer.append(w)
                if self.speech_samples >= self.min_speech and not self.speaking:
                    self.speaking = True
                    self.buffer = self.ring + self.buffer
                    self.ring = []
                if self.speaking and self.state in ('processing', 'playing'):
                    self.interrupt = True
            elif self.speaking:
                self.silence_samples += len(w)
                self.tail_silence += 1
                self.buffer.append(w)
                if self.silence_samples >= self.min_silence:
                    if self.tail_silence > 1:
                        del self.buffer[-(self.tail_silence - 1):]
                    audio = np.concatenate(self.buffer)
                    self.buffer.clear()
                    self.speaking = False
                    self.speech_samples = self.silence_samples = self.tail_silence = 0
                    self.q.put(audio)
            else:
                if self.speech_samples > 0:
                    self.buffer.clear()
                self.speech_samples = 0
                self.ring = [w]

    def start(self):
        import sounddevice as sd
        def _run():
            with sd.InputStream(samplerate=SAMPLE_RATE, channels=1, dtype='float32',
                                blocksize=512, device=self.mic) as stream:
                while getattr(self, '_running', True):
                    chunk, _ = stream.read(512)
                    self._feed(chunk.flatten())
        self._running = True
        self.thread = threading.Thread(target=_run, daemon=True)
        self.thread.start()

    def stop(self):
        self._running = False


def asr_run(model, samples):
    from funasr.utils.postprocess_utils import rich_transcription_postprocess
    r = model.generate(input=samples, cache={}, language='auto', use_itn=True)
    return rich_transcription_postprocess(r[0]['text']).strip() if r else ''


def init_model(args):
    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    print('Loading ASR...')
    with contextlib.redirect_stdout(io.StringIO()):
        from funasr import AutoModel
        asr = AutoModel(model=os.path.join(root, args.sensevoice_dir),
                        trust_remote_code=True, device=args.device,
                        disable_update=True, batch_size=1)

    print('Loading model...')
    config = VAMConfig(
        hidden_size=args.hidden_size,
        num_hidden_layers=args.num_hidden_layers,
        num_attention_heads=args.hidden_size // 96,
        num_key_value_heads=args.hidden_size // 192,
        use_moe=args.use_moe,
    )
    ckpt_dir = os.path.join(root, args.load_from)
    weight = args.weight
    if not weight.endswith('.pth'):
        if args.use_moe and not weight.endswith('_moe'):
            weight = f'{weight}_moe.pth'
        else:
            weight = f'{weight}.pth'
    ckpt_path = os.path.join(ckpt_dir, weight)

    model = VAM(config,
                audio_encoder_path=os.path.join(root, args.sensevoice_dir),
                vision_model_path=os.path.join(root, args.siglip_dir))
    state = torch.load(ckpt_path, map_location='cpu', weights_only=True)
    model.load_state_dict(state, strict=False)
    if model.audio_encoder is not None:
        model.audio_encoder.to(args.device)
    model = model.half().eval().to(args.device)

    from transformers import AutoTokenizer
    tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, args.tokenizer_dir))
    params = sum(p.numel() for p in model.parameters()) / 1e6
    print(f'  {args.weight}: {params:.2f}M')

    print('Loading Mimi...')
    from transformers import MimiModel
    mimi = MimiModel.from_pretrained(os.path.join(root, args.mimi_dir)).eval().to(args.device)
    if args.device != 'cpu':
        mimi = mimi.half()

    print('Loading VAD...')
    vad = SileroVAD()
    return model, tokenizer, asr, mimi, vad


def mimi_decode(model, codes_2d, device):
    codes = codes_2d.T.unsqueeze(0).to(device)
    codes = torch.where(codes >= 2049, torch.zeros_like(codes), codes)
    with torch.no_grad():
        audio = model.decode(codes).audio_values.squeeze().float().cpu().numpy()
    return audio


def generate_response(recorder, model, tokenizer, mimi, x, device, max_new_tokens=512):
    audio_frames = []
    text_out = ''
    interrupted = False
    with torch.no_grad():
        for y, af in model.generate(
            x, tokenizer.eos_token_id, stream=True, return_audio_codes=True,
            max_new_tokens=max_new_tokens, temperature=0.7, top_p=0.85,
        ):
            with recorder.lock:
                if recorder.interrupt:
                    interrupted = True
                    break
            if y is not None:
                ans = tokenizer.decode(y[0].tolist(), skip_special_tokens=True)
                new_text = ans[len(text_out):]
                if new_text:
                    print(new_text, end='', flush=True)
                    text_out = ans
            if af:
                audio_frames.append(af)
    print()
    if interrupted:
        print('  [interrupted]')
    if audio_frames and not interrupted:
        codes = [f for f in audio_frames if f and len(f) == 8]
        if codes:
            codes_t = torch.tensor(codes, dtype=torch.long)
            pcm = mimi_decode(mimi, codes_t, device)
            return text_out, pcm
    return text_out, None


def warmup(model, mimi, device):
    with torch.no_grad():
        ids = torch.tensor([[1, 2, 3]], device=device)
        au = torch.full((1, 8, 3), 2049, dtype=torch.long, device=device)
        model.forward(torch.cat((au, ids.unsqueeze(1)), dim=1))
        if model.audio_encoder is not None:
            try:
                model.audio_encoder.model(
                    torch.zeros(1, 100, 560, device=device),
                    torch.tensor([100], device=device))
            except Exception:
                pass
        if mimi is not None:
            mimi.decode(torch.zeros(1, 8, 1, dtype=torch.long, device=device))


def build_prompt(tokenizer, history, text):
    msgs = history + [{"role": "user", "content": text}]
    t = tokenizer.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
    return torch.tensor(tokenizer(t)['input_ids'], dtype=torch.long, device='cpu')[None, ...]


def main():
    parser = argparse.ArgumentParser(description='Omni-O Terminal Voice Chat (with interrupt)')
    parser.add_argument('--load_from', default='checkpoint/omni-o')
    parser.add_argument('--weight', default='omni-o')
    parser.add_argument('--tokenizer_dir', default='checkpoint/omni/native_hf')
    parser.add_argument('--sensevoice_dir', default='checkpoint/sensevoice')
    parser.add_argument('--siglip_dir', default='checkpoint/siglip')
    parser.add_argument('--mimi_dir', default='checkpoint/mimi')
    parser.add_argument('--hidden_size', default=768, type=int)
    parser.add_argument('--num_hidden_layers', default=8, type=int)
    parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1])
    parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu')
    parser.add_argument('--max_new_tokens', default=256, type=int)
    parser.add_argument('--vad_threshold', default=0.5, type=float)
    parser.add_argument('--min_speech_ms', default=128, type=int)
    parser.add_argument('--min_silence_ms', default=600, type=int)
    parser.add_argument('--mic', default=None, type=int, help='Microphone device index')
    args = parser.parse_args()

    model, tokenizer, asr, mimi, vad = init_model(args)
    device = args.device

    print('Warmup...')
    warmup(model, mimi, device)
    print('Warmup done!\n')

    import sounddevice as sd
    recorder = RealtimeRecorder(vad, args.vad_threshold, args.min_speech_ms,
                                args.min_silence_ms, args.mic)
    recorder.start()

    history = []
    mic_name = sd.query_devices(args.mic, 'input')['name'] if args.mic is not None else 'default'
    print('=== Omni-O Terminal Voice Chat (interruptible) ===')
    print(f'Mic: {mic_name}')
    print('Speak to start — silence >=600ms = end of turn')
    print('Speak during playback to interrupt')
    print()

    try:
        while True:
            audio = recorder.q.get()
            if len(audio) < SAMPLE_RATE * 0.1:
                continue

            seconds = len(audio) / SAMPLE_RATE
            print(f'\r  {seconds:.1f}s audio  ASR...', end=' ', flush=True)
            st = time.time()
            text = asr_run(asr, audio)
            print(f'"{text}" ({time.time() - st:.1f}s)')
            if not text.strip():
                continue

            history.append({"role": "user", "content": text})

            with recorder.lock:
                recorder.state = 'processing'

            x = build_prompt(tokenizer, history[:-1], text).to(device)
            print('  ', end='', flush=True)
            st = time.time()
            text_resp, pcm = generate_response(recorder, model, tokenizer, mimi,
                                                x, device, args.max_new_tokens)

            with recorder.lock:
                interrupted = recorder.interrupt
                recorder.interrupt = False
                recorder.state = 'playing' if not interrupted else 'idle'

            if text_resp:
                if not interrupted:
                    history.append({"role": "assistant", "content": text_resp})

            if pcm is not None and len(pcm) > 0:
                print(f'  Playing... ({time.time() - st:.1f}s gen)', end=' ', flush=True)
                sd.play(pcm, AUDIO_SR)
                # Poll playback with interrupt check
                while sd.get_stream().active:
                    with recorder.lock:
                        if recorder.interrupt:
                            sd.stop()
                            print('[interrupted]', end=' ')
                            with recorder.lock:
                                recorder.interrupt = False
                                recorder.state = 'idle'
                            break
                    time.sleep(0.05)
                print('done')

            with recorder.lock:
                recorder.state = 'idle'

    except KeyboardInterrupt:
        print('\nBye!')
    finally:
        recorder.stop()


if __name__ == '__main__':
    main()