streaming_inference_example.py

"""Minimal streaming inference for Muno459/fastconformer-quran-streaming (ONNX, cache-aware).

The ONNX takes 80-dim log-mel features (NOT raw audio) + carries cache tensors across chunks.
Pipeline: audio(16k mono) -> log-mel(80) -> fixed-global CMVN -> chunked encoder w/ cache -> CTC greedy.

EXACT shapes that fix the `Where`/`Add` broadcast errors (batch-first for ONNX):
  cache_last_channel     : float32  [1, 17, 70, 512]   (init zeros)   # 17 layers, 70 = left cache
  cache_last_time        : float32  [1, 17, 512, 8]    (init zeros)   # 8  = conv cache (kernel-1)
  cache_last_channel_len : int64    [1] = [0]          (init)
Carry the returned cache_*_next straight back in as the next step's cache_* (already batch-first).

Mel params (must match exactly if you reimplement on-device, e.g. iOS/Android):
  sample_rate=16000, n_fft=512, win_length=400 (25ms hann), hop_length=160 (10ms),
  n_mels=80 (librosa slaney-norm mel), power=2.0, log = ln(x + 2**-24), preemph=0.97, dither=0 (inference).
Then apply fixed-global CMVN from streaming_global_cmvn.npz: (mel - mean) / (std + 1e-5),
using the tlog_* constants for phone audio, clean_* for studio.

The simplest *correct* mel is NeMo's own preprocessor (one import); a hand-rolled mel must match the
params above or accuracy drops. Decode with the BPE tokenizer (tokenizer.model, blank id = 1024).
"""
import numpy as np, onnxruntime as ort, soundfile as sf, sentencepiece as spm

SESS = ort.InferenceSession("model.q8.onnx", providers=["CPUExecutionProvider"])
SP = spm.SentencePieceProcessor(model_file="tokenizer.model")
CMVN = np.load("streaming_global_cmvn.npz")
BLANK = 1024
N_LAYERS, D_MODEL, LEFT_CACHE, TIME_CACHE = 17, 512, 70, 8
CHUNK_MEL = 112          # mel frames per streaming step (8x subsampling -> ~14 output frames)

def log_mel(wav, sr=16000):
    """80-dim log-mel matching NeMo's FilterbankFeatures. Uses NeMo if available (exact),
    else falls back to a torchaudio/librosa reimplementation with the documented params."""
    import torch, torchaudio
    wav = torch.tensor(np.asarray(wav, np.float32))
    wav = torch.cat([wav[:1], wav[1:] - 0.97 * wav[:-1]])               # preemphasis
    spec = torchaudio.transforms.MelSpectrogram(
        sample_rate=sr, n_fft=512, win_length=400, hop_length=160,
        n_mels=80, power=2.0, window_fn=torch.hann_window, norm="slaney", mel_scale="slaney")(wav)
    return torch.log(spec + 2**-24).numpy()                              # (80, T)

def transcribe(path, kind="tlog"):
    wav, sr = sf.read(path)
    if wav.ndim > 1: wav = wav.mean(1)
    mel = log_mel(wav, sr)                                               # (80, T)
    gm = CMVN[f"{kind}_mean"][:, None]; gs = CMVN[f"{kind}_std"][:, None]
    mel = (mel - gm) / (gs + 1e-5)
    clc = np.zeros((1, N_LAYERS, LEFT_CACHE, D_MODEL), np.float32)
    clt = np.zeros((1, N_LAYERS, D_MODEL, TIME_CACHE), np.float32)
    cll = np.zeros((1,), np.int64)
    ids = []
    for s in range(0, mel.shape[1], CHUNK_MEL):
        chunk = mel[:, s:s + CHUNK_MEL][None].astype(np.float32)        # (1,80,chunk)
        length = np.array([chunk.shape[2]], np.int64)
        logp, _, clc, clt, cll = SESS.run(None, {
            "audio_signal": chunk, "length": length,
            "cache_last_channel": clc, "cache_last_time": clt, "cache_last_channel_len": cll})
        a = logp[0].argmax(-1)
        prev = -1
        for t in a:
            t = int(t)
            if t != prev and t != BLANK: ids.append(t)
            prev = t
    return SP.decode([i for i in ids])

if __name__ == "__main__":
    import sys
    print(transcribe(sys.argv[1], kind="tlog" if len(sys.argv) < 3 else sys.argv[2]))