Segment long audio with Silero VAD and transcribe batches of four

README.md

@@ -9,16 +9,20 @@ app_file: app.py
 pinned: false
 models:
   - KeisukeMiyamoto/lambda-audio-v6
-short_description: Japanese speech recognition for audio clips up to 30 seconds
+short_description: Japanese speech recognition with Silero VAD
 ---
 
 # LambdaAudio v6
 
 A simple Japanese speech recognition demo powered by
 [LambdaAudio v6](https://huggingface.co/KeisukeMiyamoto/lambda-audio-v6).
-Upload or record up to 30 seconds of audio, then select **文字起こし**.
+Upload or record Japanese audio, then select **文字起こし** (maximum file size: 200 MB).
+Audio up to 30 seconds is transcribed directly. Longer recordings are segmented
+with Silero VAD into speech clips of at most 20 seconds and transcribed in batches
+of four. Results appear in chronological order as each batch finishes.
 
-This Space uses **ZeroGPU**, with one transcription at a time. GPU availability
+This Space uses **ZeroGPU**, with one recording at a time and a separate GPU
+allocation for each batch. GPU availability
 and usage limits depend on the visitor's Hugging Face account.
 
 The FP16 Safetensors checkpoint is loaded into FP32 model parameters for inference.

app.py

@@ -2,8 +2,12 @@ import spaces
 
 # isort: split
 
+from collections.abc import Iterator
+from itertools import batched
+
 import gradio as gr
 import torch
+from silero_vad import load_silero_vad
 
 from src.environment import load_hf_token
 from src.shared.transcription import transcribe
@@ -18,13 +22,22 @@ def create_app() -> gr.Blocks:
     components = load_space_components(load_hf_token())
     device = torch.device("cuda")
     components.model.to(device)
+    vad_model = load_silero_vad(onnx=True)
 
     @spaces.GPU(duration=30)
-    def generate(audio: bytes) -> str:
-        return transcribe(components.model, components.feature_extractor, components.tokenizer, config, device, [audio])[0].text
-
-    def transcribe_recording(audio_path: str | None) -> str:
-        return generate(read_recording(audio_path))
+    def generate(audio: list[bytes]) -> list[str]:
+        results = transcribe(components.model, components.feature_extractor, components.tokenizer, config, device, audio)
+        return [result.text for result in results]
+
+    def transcribe_recording(audio_path: str | None) -> Iterator[str]:
+        segments = read_recording(audio_path, vad_model)
+        if not segments:
+            yield "音声内に発話が見つかりませんでした。"
+            return
+        texts: list[str] = []
+        for batch in batched(segments, 4):
+            texts.extend(generate(list(batch)))
+            yield "\n".join(texts)
 
     # ---------------------------------------------------------
     # Provide one audio input and one transcription action.
@@ -32,7 +45,8 @@ def create_app() -> gr.Blocks:
     with gr.Blocks(title="LambdaAudio v6", delete_cache=(3600, 3600)) as app:
         gr.Markdown(
             "# LambdaAudio v6\n"
-            "日本語の音声を文字起こしします。30秒以内の音声をアップロードするか、マイクで録音してください。\n\n"
+            "日本語の音声をアップロードするか、マイクで録音してください。\n"
+            "30秒を超える音声は発話区間に分割し、順番に文字起こしします（最大200 MB）。\n\n"
             "[モデルの詳細](https://huggingface.co/KeisukeMiyamoto/lambda-audio-v6)"
         )
         audio = gr.Audio(sources=["upload", "microphone"], type="filepath", format="wav", label="音声", editable=False)
@@ -43,4 +57,4 @@ def create_app() -> gr.Blocks:
 
 
 if __name__ == "__main__":
-    create_app().launch(max_file_size="20mb")
+    create_app().launch(max_file_size="200mb")

requirements.txt

@@ -4,10 +4,13 @@ einops>=0.8.0,<1.0.0
 huggingface-hub>=0.36.0,<2.0.0
 kernels==0.16.1
 numpy>=1.26.0,<3.0.0
+onnxruntime==1.31.0
 pyarrow>=15.0.0,<26.0.0
 pydantic>=2.0.0,<3.0.0
 python-dotenv>=1.1.0,<2.0.0
 safetensors==0.8.0
+silero-vad==6.2.3
+soundfile>=0.12.0,<1.0.0
 spaces==0.51.3
 torch==2.13.0
 transformers==5.16.1

src/space/runtime.py

@@ -1,10 +1,15 @@
+from io import BytesIO
 from pathlib import Path
 from typing import cast
 
 import av
 import gradio as gr
+import soundfile as sf
+import torch
 from huggingface_hub import hf_hub_download
 from safetensors.torch import load_file
+from silero_vad import get_speech_timestamps
+from silero_vad.utils_vad import OnnxWrapper
 from transformers import AutoFeatureExtractor, AutoTokenizer, SequenceFeatureExtractor
 from transformers.models.lfm2.configuration_lfm2 import Lfm2Config
 
@@ -13,7 +18,7 @@ from src.modeling.lambda_audio import LambdaAudioModel
 from src.modeling.lfm2_decoder import AudioLfm2ForCausalLM
 from src.modeling.loading import ModelComponents
 from src.modeling.zipformer import ZipformerConfig, ZipformerModel
-from src.shared.transcription import decode_audio_batch
+from src.shared.transcription import decode_audio
 
 
 class SpaceConfig(ModelConfig):
@@ -52,18 +57,42 @@ def load_space_components(token: str) -> ModelComponents:
     )
 
 
-def read_recording(audio_path: str | None) -> bytes:
+def read_recording(audio_path: str | None, vad_model: OnnxWrapper) -> list[bytes]:
     # ---------------------------------------------------------
-    # Reject missing, invalid, or oversized audio before requesting a GPU.
+    # Decode and validate audio before requesting a GPU.
     # ---------------------------------------------------------
     if audio_path is None:
         raise gr.Error("音声をアップロードするか、録音してください。")
     path = Path(audio_path)
-    if path.stat().st_size > 20 * 1024 * 1024:
-        raise gr.Error("20 MB以下の音声ファイルを使用してください。")
+    if path.stat().st_size > 200 * 1024 * 1024:
+        raise gr.Error("200 MB以下の音声ファイルを使用してください。")
     audio = path.read_bytes()
+    config = SpaceConfig()
     try:
-        decode_audio_batch([audio], SpaceConfig())
+        waveform = decode_audio(audio, config.sampling_rate)
     except (av.FFmpegError, IndexError, ValueError) as error:
-        raise gr.Error("30秒以内の、再生できる音声ファイルを使用してください。") from error
-    return audio
+        raise gr.Error("再生できる音声ファイルを使用してください。") from error
+    if waveform.size <= config.sampling_rate * config.max_audio_seconds:
+        return [audio]
+
+    # ---------------------------------------------------------
+    # Split longer recordings at speech boundaries, at most 20 seconds.
+    # Encode lossless clips in chronological order for batched ASR.
+    # ---------------------------------------------------------
+    timestamps = get_speech_timestamps(
+        torch.from_numpy(waveform),
+        vad_model,
+        sampling_rate=config.sampling_rate,
+        max_speech_duration_s=20,
+        min_silence_duration_ms=300,
+        speech_pad_ms=100,
+    )
+    segments: list[bytes] = []
+    segment_samples = 20 * config.sampling_rate
+    for timestamp in timestamps:
+        for start in range(timestamp["start"], timestamp["end"], segment_samples):
+            end = min(start + segment_samples, timestamp["end"])
+            buffer = BytesIO()
+            sf.write(buffer, waveform[start:end], config.sampling_rate, format="WAV", subtype="FLOAT")
+            segments.append(buffer.getvalue())
+    return segments