Upload folder using huggingface_hub

README.md

@@ -1,3 +1,16 @@
----
-license: apache-2.0
----
+---
+license: apache-2.0
+tags:
+  - voice-activity-detection
+  - onnx
+  - streaming
+  - firered-vad
+  - dfsmn
+language:
+  - en
+  - zh
+base_model: FireRedTeam/FireRedVAD
+pipeline_tag: voice-activity-detection
+---
+
+# FireRedVAD VAD (ONNX), do not contain vad_streaming & aed

cmvn.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b020eb6a57b01993c7aa032fbb0e33d257359ef1bdcb4b66e3dc360f11b42d4e
+size 644

cmvn.json

@@ -0,0 +1 @@
+{"dim": 80, "means": [10.422951698303223, 10.86209774017334, 11.764544486999512, 12.490164756774902, 13.259830474853516, 13.895943641662598, 14.364940643310547, 14.593948364257812, 14.749723434448242, 14.668314933776855, 14.730796813964844, 14.775052070617676, 14.989051818847656, 15.17800521850586, 15.253520011901855, 15.32863712310791, 15.33401870727539, 15.288641929626465, 15.427661895751953, 15.24626636505127, 15.092574119567871, 15.290421485900879, 15.075750350952148, 15.186773300170898, 15.08867359161377, 15.170797348022461, 15.070178031921387, 15.150794982910156, 15.108532905578613, 15.115345001220703, 15.141280174255371, 15.131832122802734, 15.145195960998535, 15.191518783569336, 15.235478401184082, 15.30636978149414, 15.373021125793457, 15.416394233703613, 15.459856986999512, 15.391432762145996, 15.463576316833496, 15.39966106414795, 15.462907791137695, 15.441629409790039, 15.484969139099121, 15.552401542663574, 15.638092041015625, 15.705489158630371, 15.767008781433105, 15.855123519897461, 15.867269515991211, 15.8915376663208, 15.923145294189453, 15.97838306427002, 16.014801025390625, 16.048675537109375, 16.082029342651367, 16.096799850463867, 16.09373664855957, 16.072479248046875, 16.075510025024414, 16.02227020263672, 15.976761817932129, 15.89786434173584, 15.81274127960205, 15.71120548248291, 15.604198455810547, 15.553519248962402, 15.510252952575684, 15.460023880004883, 15.415684700012207, 15.376028060913086, 15.328349113464355, 15.295371055603027, 15.185470581054688, 15.017045021057129, 14.90507984161377, 14.623806953430176, 14.138093948364258, 13.313870429992676], "inverse_std_variances": [0.24949808418750763, 0.23563234508037567, 0.23145152628421783, 0.23322339355945587, 0.23182660341262817, 0.2285335659980774, 0.22434869408607483, 0.21898920834064484, 0.21832437813282013, 0.2208259254693985, 0.22296735644340515, 0.2228841632604599, 0.22234810888767242, 0.22100642323493958, 0.21994201838970184, 0.22005443274974823, 0.2207009196281433, 0.22150810062885284, 0.22236667573451996, 0.22305291891098022, 0.22335341572761536, 0.22438906133174896, 0.22547701001167297, 0.22690075635910034, 0.22823023796081543, 0.22931471467018127, 0.2304672747850418, 0.23083552718162537, 0.23143382370471954, 0.23220659792423248, 0.23257988691329956, 0.23361970484256744, 0.23437240719795227, 0.23508252203464508, 0.23578079044818878, 0.23589199781417847, 0.2360209822654724, 0.23663799464702606, 0.23749876022338867, 0.2379845231771469, 0.2389937788248062, 0.23974815011024475, 0.24030835926532745, 0.24097692966461182, 0.24143248796463013, 0.24135465919971466, 0.24079938232898712, 0.24047406017780304, 0.23995524644851685, 0.23952287435531616, 0.2394808828830719, 0.2393650859594345, 0.2392933964729309, 0.23902198672294617, 0.23857873678207397, 0.23814702033996582, 0.23804621398448944, 0.23824194073677063, 0.238600954413414, 0.23915407061576843, 0.23922540247440338, 0.23938308656215668, 0.23973360657691956, 0.2396056205034256, 0.2402850240468979, 0.24061813950538635, 0.2406792938709259, 0.24096201360225677, 0.24043606221675873, 0.2402152717113495, 0.23972514271736145, 0.23871998488903046, 0.2374413162469864, 0.23619508743286133, 0.23337280750274658, 0.22680233418941498, 0.22577503323554993, 0.22503846883773804, 0.22631137073040009, 0.2289949357509613]}

infer_onnx.py

@@ -0,0 +1,441 @@
+#!/usr/bin/env python3
+"""
+FireRedVAD ONNX inference  —  no PyTorch / kaldi_native_fbank required.
+
+Dependencies: numpy, scipy, soundfile, onnxruntime
+
+Usage:
+
+    python infer_onnx.py assets/hello_zh.wav --model_dir /path/to/FireRedVAD_onnx
+    python infer_onnx.py assets/hello_zh.wav
+    python infer_onnx.py assets/hello_zh.wav --model_dir /path/to/FireRedVAD_onnx
+    python infer_onnx.py assets/hello_en.wav --speech_threshold 0.4 --min_speech_frame 20
+"""
+
+import argparse
+import json
+import math
+import os
+from collections import deque
+
+import numpy as np
+import soundfile as sf
+import onnxruntime as ort
+
+
+# ---------------------------------------------------------------------------
+# Constants (matches fireredvad/core/constants.py)
+# ---------------------------------------------------------------------------
+SAMPLE_RATE = 16000
+FRAME_LENGTH_MS = 25
+FRAME_SHIFT_MS = 10
+FRAME_LENGTH_S = 0.025
+FRAME_SHIFT_S = 0.010
+
+
+# ---------------------------------------------------------------------------
+# Kaldi-compatible Fbank (replaces kaldi_native_fbank dependency)
+# ---------------------------------------------------------------------------
+
+def _mel_to_hz(mel):
+    return 700.0 * (np.exp(mel / 1127.0) - 1.0)
+
+def _hz_to_mel(hz):
+    return 1127.0 * np.log(1.0 + hz / 700.0)
+
+def _build_mel_filterbank(n_fft: int, n_mels: int = 80,
+                          f_min: float = 0.0, f_max: float = 8000.0,
+                          sample_rate: int = 16000) -> np.ndarray:
+    """
+    Build Kaldi-style mel filterbank matrix.
+    Returns shape (n_mels, n_fft // 2 + 1) — real spectrum bins.
+    Kaldi uses triangular filters defined on the mel scale, NOT on the FFT bin scale.
+    """
+    n_freqs = n_fft // 2 + 1
+    freq_bins = np.linspace(0, sample_rate / 2, n_freqs)  # Hz for each FFT bin
+
+    mel_min = _hz_to_mel(f_min)
+    mel_max = _hz_to_mel(f_max)
+    # n_mels + 2 points: left edge, n_mels centers, right edge
+    mel_points = np.linspace(mel_min, mel_max, n_mels + 2)
+    hz_points = _mel_to_hz(mel_points)
+
+    # For each mel band, compute the triangular weight for each FFT bin
+    filters = np.zeros((n_mels, n_freqs), dtype=np.float32)
+    for m in range(1, n_mels + 1):
+        left   = hz_points[m - 1]
+        center = hz_points[m]
+        right  = hz_points[m + 1]
+        for k, f in enumerate(freq_bins):
+            if left <= f <= center:
+                filters[m - 1, k] = (f - left) / (center - left)
+            elif center < f <= right:
+                filters[m - 1, k] = (right - f) / (right - center)
+    return filters
+
+
+def _extract_kaldi_fbank(wav_int16: np.ndarray,
+                         sample_rate: int = 16000,
+                         num_mel_bins: int = 80,
+                         frame_length_ms: float = 25.0,
+                         frame_shift_ms: float = 10.0) -> np.ndarray:
+    """
+    Compute log Mel filterbank features identical to kaldi_native_fbank with:
+        samp_freq=16000, frame_length_ms=25, frame_shift_ms=10,
+        dither=0, snip_edges=True, num_mel_bins=80
+
+    Input : int16 PCM array (1-D)
+    Output: (T, 80) float32 array  — log fbank energies
+    """
+    assert wav_int16.dtype == np.int16 and wav_int16.ndim == 1
+
+    frame_len  = int(round(sample_rate * frame_length_ms / 1000))  # 400
+    frame_shift = int(round(sample_rate * frame_shift_ms  / 1000)) # 160
+    n_fft = 1 << (frame_len - 1).bit_length()  # next power of 2 >= 400  → 512
+
+    # Hann window  (Kaldi uses the same)
+    window = np.hanning(frame_len).astype(np.float32)
+
+    # Convert to float, Kaldi: raw int16 → float32 (NOT divided by 32768 at fbank stage)
+    # Actually Kaldi processes the waveform as-is (int16 values as float).
+    wav_f = wav_int16.astype(np.float32)
+
+    # Pre-emphasis  (Kaldi default: 0.97)
+    wav_preemph = np.append(wav_f[0], wav_f[1:] - 0.97 * wav_f[:-1])
+
+    # Framing (snip_edges=True: only frames that fit completely)
+    n_frames = 1 + (len(wav_preemph) - frame_len) // frame_shift
+    if n_frames <= 0:
+        return np.zeros((0, num_mel_bins), dtype=np.float32)
+
+    frames = np.stack([
+        wav_preemph[i * frame_shift: i * frame_shift + frame_len] * window
+        for i in range(n_frames)
+    ])  # (T, frame_len)
+
+    # FFT → power spectrum
+    spec = np.fft.rfft(frames, n=n_fft)           # (T, n_fft//2+1) complex
+    power = (spec.real ** 2 + spec.imag ** 2).astype(np.float32)  # (T, n_fft//2+1)
+
+    # Mel filterbank
+    mel_fb = _build_mel_filterbank(n_fft, num_mel_bins,
+                                   f_min=0.0, f_max=sample_rate / 2,
+                                   sample_rate=sample_rate)  # (80, n_fft//2+1)
+    mel_energy = power @ mel_fb.T  # (T, 80)
+
+    # Log compression — floor at 1.0 (Kaldi: log_energy_floor = FLT_MIN, effectively 0)
+    mel_energy = np.maximum(mel_energy, 1.0)
+    log_mel = np.log(mel_energy).astype(np.float32)
+
+    return log_mel  # (T, 80)
+
+
+# ---------------------------------------------------------------------------
+# CMVN (matches fireredvad/core/audio_feat.py  CMVN.__call__)
+# ---------------------------------------------------------------------------
+
+class CMVN:
+    def __init__(self, cmvn_json_path: str):
+        with open(cmvn_json_path) as f:
+            d = json.load(f)
+        self.means = np.array(d["means"], dtype=np.float32)
+        self.inv_std = np.array(d["inverse_std_variances"], dtype=np.float32)
+
+    def __call__(self, fbank: np.ndarray) -> np.ndarray:
+        return (fbank - self.means) * self.inv_std
+
+
+# ---------------------------------------------------------------------------
+# VAD postprocessor (pure Python, mirrors vad_postprocessor.py)
+# ---------------------------------------------------------------------------
+
+class VadPostprocessor:
+    def __init__(self, smooth_window_size=5, prob_threshold=0.4,
+                 min_speech_frame=20, max_speech_frame=2000,
+                 min_silence_frame=20, merge_silence_frame=0,
+                 extend_speech_frame=0):
+        self.smooth_window_size = max(1, smooth_window_size)
+        self.prob_threshold = prob_threshold
+        self.min_speech_frame = min_speech_frame
+        self.max_speech_frame = max_speech_frame
+        self.min_silence_frame = min_silence_frame
+        self.merge_silence_frame = merge_silence_frame
+        self.extend_speech_frame = extend_speech_frame
+
+    def process(self, raw_probs):
+        if not raw_probs:
+            return []
+        smoothed = self._smooth(raw_probs)
+        binary = (np.asarray(smoothed) >= self.prob_threshold).astype(int).tolist()
+        decisions = self._state_machine(binary)
+        decisions = self._fix_start(decisions)
+        decisions = self._merge_silence(decisions)
+        decisions = self._extend_speech(decisions)
+        decisions = self._split_long(decisions, raw_probs)
+        return decisions
+
+    def decisions_to_segments(self, decisions, wav_dur=None):
+        segments = []
+        speech_start = None
+        for t, d in enumerate(decisions):
+            if d == 1 and speech_start is None:
+                speech_start = t
+            elif d == 0 and speech_start is not None:
+                segments.append((speech_start * FRAME_SHIFT_S, t * FRAME_SHIFT_S))
+                speech_start = None
+        if speech_start is not None:
+            end = len(decisions) * FRAME_SHIFT_S + FRAME_LENGTH_S
+            if wav_dur is not None:
+                end = min(end, wav_dur)
+            segments.append((speech_start * FRAME_SHIFT_S, end))
+        return [(round(s, 3), round(e, 3)) for s, e in segments]
+
+    def _smooth(self, probs):
+        if self.smooth_window_size <= 1:
+            return probs
+        probs_np = np.array(probs)
+        kernel = np.ones(self.smooth_window_size) / self.smooth_window_size
+        smoothed = np.convolve(probs_np, kernel, mode='full')[:len(probs)]
+        for i in range(min(self.smooth_window_size - 1, len(probs))):
+            smoothed[i] = np.mean(probs_np[:i + 1])
+        return smoothed
+
+    def _state_machine(self, binary):
+        SILENCE, POSSIBLE_SPEECH, SPEECH, POSSIBLE_SILENCE = 0, 1, 2, 3
+        decisions = [0] * len(binary)
+        state = SILENCE
+        speech_start = silence_start = -1
+        for t, is_speech in enumerate(binary):
+            if state == SILENCE:
+                if is_speech:
+                    state = POSSIBLE_SPEECH; speech_start = t
+            elif state == POSSIBLE_SPEECH:
+                if is_speech:
+                    if t - speech_start >= self.min_speech_frame:
+                        state = SPEECH
+                        decisions[speech_start:t] = [1] * (t - speech_start)
+                else:
+                    state = SILENCE; speech_start = -1
+            elif state == SPEECH:
+                if not is_speech:
+                    state = POSSIBLE_SILENCE; silence_start = t
+            elif state == POSSIBLE_SILENCE:
+                if not is_speech:
+                    if t - silence_start >= self.min_silence_frame:
+                        state = SILENCE; speech_start = -1
+                else:
+                    state = SPEECH; silence_start = -1
+            decisions[t] = 1 if state in (SPEECH, POSSIBLE_SILENCE) else 0
+        return decisions
+
+    def _fix_start(self, decisions):
+        new = decisions.copy()
+        for t, d in enumerate(decisions):
+            if t > 0 and decisions[t - 1] == 0 and d == 1:
+                start = max(0, t - self.smooth_window_size)
+                new[start:t] = [1] * (t - start)
+        return new
+
+    def _merge_silence(self, decisions):
+        if self.merge_silence_frame <= 0:
+            return decisions
+        new = decisions.copy()
+        silence_start = None
+        for t, d in enumerate(decisions):
+            if t > 0 and decisions[t - 1] == 1 and d == 0 and silence_start is None:
+                silence_start = t
+            elif t > 0 and decisions[t - 1] == 0 and d == 1 and silence_start is not None:
+                if t - silence_start < self.merge_silence_frame:
+                    new[silence_start:t] = [1] * (t - silence_start)
+                silence_start = None
+        return new
+
+    def _extend_speech(self, decisions):
+        if self.extend_speech_frame <= 0:
+            return decisions
+        d = np.array(decisions)
+        k = np.ones(2 * self.extend_speech_frame + 1)
+        return (np.convolve(d, k, mode='same') > 0).astype(int).tolist()
+
+    def _split_long(self, decisions, probs):
+        new = decisions.copy()
+        segments = self.decisions_to_segments(decisions)
+        for s_s, e_s in segments:
+            sf_ = int(s_s / FRAME_SHIFT_S)
+            ef_ = int(e_s / FRAME_SHIFT_S)
+            if ef_ - sf_ > self.max_speech_frame:
+                seg_probs = probs[sf_:ef_]
+                for split in self._find_splits(seg_probs):
+                    new[sf_ + split] = 0
+        return new
+
+    def _find_splits(self, probs):
+        splits, L, start = [], len(probs), 0
+        while start < L:
+            if (L - start) <= self.max_speech_frame:
+                break
+            ws = int(start + self.max_speech_frame / 2)
+            we = int(start + self.max_speech_frame)
+            splits.append(ws + int(np.argmin(probs[ws:we])))
+            start = splits[-1] + 1
+        return splits
+
+
+# ---------------------------------------------------------------------------
+# Main inference class
+# ---------------------------------------------------------------------------
+
+class FireRedVadOnnx:
+    """
+    End-to-end FireRedVAD inference using ONNX Runtime.
+    No PyTorch or kaldi_native_fbank required.
+    """
+
+    def __init__(self, model_dir: str,
+                 smooth_window_size: int = 5,
+                 speech_threshold: float = 0.4,
+                 min_speech_frame: int = 20,
+                 max_speech_frame: int = 2000,
+                 min_silence_frame: int = 20,
+                 merge_silence_frame: int = 0,
+                 extend_speech_frame: int = 0,
+                 chunk_max_frame: int = 30000,
+                 use_coreml: bool = False):
+
+        # CMVN params
+        # Support both naming conventions: cmvn.json and cmvn_params.json
+        for name in ("cmvn.json", "cmvn_params.json"):
+            p = os.path.join(model_dir, name)
+            if os.path.exists(p):
+                self.cmvn = CMVN(p)
+                break
+        else:
+            raise FileNotFoundError(f"No cmvn JSON found in {model_dir}")
+
+        # ONNX session
+        model_path = os.path.join(model_dir, "model.onnx")
+        providers = ["CPUExecutionProvider"]
+        if use_coreml:
+            providers = ["CoreMLExecutionProvider"] + providers
+        opts = ort.SessionOptions()
+        opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+        self.session = ort.InferenceSession(model_path, sess_options=opts,
+                                            providers=providers)
+
+        # Fbank config (fixed to match training)
+        self.num_mel_bins = 80
+        self.chunk_max_frame = chunk_max_frame
+
+        # Postprocessor
+        self.postprocessor = VadPostprocessor(
+            smooth_window_size=smooth_window_size,
+            prob_threshold=speech_threshold,
+            min_speech_frame=min_speech_frame,
+            max_speech_frame=max_speech_frame,
+            min_silence_frame=min_silence_frame,
+            merge_silence_frame=merge_silence_frame,
+            extend_speech_frame=extend_speech_frame)
+
+    def _load_audio(self, audio_path: str):
+        wav, sr = sf.read(audio_path, dtype="int16")
+        assert sr == SAMPLE_RATE, f"Expected 16kHz, got {sr}Hz. Convert with ffmpeg first."
+        assert wav.ndim == 1, "Expected mono audio."
+        return wav
+
+    def _extract_features(self, wav_int16: np.ndarray) -> np.ndarray:
+        """Returns (T, 80) CMVN-normalized log-fbank."""
+        fbank = _extract_kaldi_fbank(wav_int16, SAMPLE_RATE, self.num_mel_bins)
+        return self.cmvn(fbank)
+
+    def _run_model(self, feat: np.ndarray) -> np.ndarray:
+        """
+        feat: (T, 80)
+        Returns probs: (T,) float32
+        """
+        T = feat.shape[0]
+        all_probs = []
+
+        for chunk_start in range(0, T, self.chunk_max_frame):
+            chunk = feat[chunk_start: chunk_start + self.chunk_max_frame]
+            inp = chunk[np.newaxis, :, :].astype(np.float32)  # (1, t, 80)
+
+            outputs = self.session.run(["output"], {"input": inp})
+            probs_chunk = outputs[0][0, :, 0]  # (t,)
+            all_probs.append(probs_chunk)
+
+        return np.concatenate(all_probs)  # (T,)
+
+    def detect(self, audio_path: str):
+        """
+        Run VAD on a 16kHz mono WAV file.
+
+        Returns:
+            result : dict with keys 'dur', 'timestamps', 'wav_path'
+            probs  : (T,) float32 array of raw per-frame speech probabilities
+        """
+        wav = self._load_audio(audio_path)
+        dur = len(wav) / SAMPLE_RATE
+
+        feat = self._extract_features(wav)          # (T, 80)
+        probs = self._run_model(feat)               # (T,)
+
+        decisions = self.postprocessor.process(probs.tolist())
+        segments  = self.postprocessor.decisions_to_segments(decisions, dur)
+
+        result = {
+            "dur": round(dur, 3),
+            "timestamps": segments,
+            "wav_path": audio_path,
+        }
+        return result, probs
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+def main():
+    parser = argparse.ArgumentParser(description="FireRedVAD ONNX inference")
+    parser.add_argument("wav_path", help="Path to 16kHz mono WAV file")
+    parser.add_argument("--model_dir",
+                        default="/path/to/FireRedVAD_onnx",
+                        help="Directory containing model.onnx and cmvn.json")
+    parser.add_argument("--smooth_window_size", type=int, default=5)
+    parser.add_argument("--speech_threshold", type=float, default=0.4)
+    parser.add_argument("--min_speech_frame", type=int, default=20)
+    parser.add_argument("--max_speech_frame", type=int, default=2000)
+    parser.add_argument("--min_silence_frame", type=int, default=20)
+    parser.add_argument("--merge_silence_frame", type=int, default=0)
+    parser.add_argument("--extend_speech_frame", type=int, default=0)
+    parser.add_argument("--use_coreml", action="store_true",
+                        help="Use CoreML execution provider (Apple Silicon)")
+    args = parser.parse_args()
+
+    vad = FireRedVadOnnx(
+        model_dir=args.model_dir,
+        smooth_window_size=args.smooth_window_size,
+        speech_threshold=args.speech_threshold,
+        min_speech_frame=args.min_speech_frame,
+        max_speech_frame=args.max_speech_frame,
+        min_silence_frame=args.min_silence_frame,
+        merge_silence_frame=args.merge_silence_frame,
+        extend_speech_frame=args.extend_speech_frame,
+        use_coreml=args.use_coreml,
+    )
+
+    import time
+    start = time.time()
+    result, probs = vad.detect(args.wav_path)
+
+    du = time.time() - start
+    print(f"vad : {du}s")
+    print(f"Duration  : {result['dur']:.3f}s")
+    print(f"Segments  : {len(result['timestamps'])}")
+    for i, (s, e) in enumerate(result["timestamps"]):
+        print(f"  [{i+1:3d}] {s:.3f}s -- {e:.3f}s  ({e-s:.3f}s)")
+
+
+if __name__ == "__main__":
+    main()
+
+# python infer_onnx.py assets/hello_zh.wav --model_dir /path/to/FireRedVAD_onnx

model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:517e9c6207618407da41fc274b1e3f09e8cde531db42f039a52be93b29a49151
+size 2461278