#!/usr/bin/env python3 """Portable FP32 offline SpeedASR inference, including batched beam search. model = SpeedASR.from_pretrained(revision="") texts = model.transcribe_batch([mono_float32_audio], sample_rate=16000) This frontend returns decoded text with VoiceA casing/punctuation restored. WER normalization belongs to the benchmark scorer, not the recognizer. """ from __future__ import annotations import argparse import hashlib import json import logging import math import re from itertools import pairwise from pathlib import Path from typing import Iterable, Sequence import kaldifeat import numpy as np import soundfile as sf import torch from torch.nn.utils.rnn import pad_sequence from speedasr_core.beam_search import modified_beam_search from speedasr_core.decoder import Decoder from speedasr_core.joiner import Joiner from speedasr_core.model import AsrModel from speedasr_core.scaling import ScheduledFloat from speedasr_core.subsampling import Conv2dSubsampling from speedasr_core.zipformer import Zipformer2 def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: for block in iter(lambda: stream.read(1 << 20), b""): digest.update(block) return digest.hexdigest() def _read_tokens(path: Path) -> dict[int, str]: table = {} for line in path.read_text(encoding="utf-8").splitlines(): parts = line.split() if len(parts) >= 2: token_id = int(parts[-1]) if token_id in table: raise ValueError(f"Duplicate token ID: {token_id}") table[token_id] = " ".join(parts[:-1]) if not table or "" not in table.values() or "" not in table.values(): raise ValueError("tokens.txt must contain and ") return table def restore_voicea_text(text: str) -> str: """The exact VoiceA postprocessing used by the local benchmark.""" if not text: return text processed = ( text.replace("<,>", ",").replace("<.>", ".") .replace("", "?").replace("", "!").replace("", "") ) words = processed.split() casing_set = {"", "", ""} ans = [] for w1, w2 in pairwise(words): if w1 in casing_set: continue if w2 in casing_set: if w2 == "": w1 = w1.capitalize() elif w2 == "": w1 = w1.upper() ans.append(w1) if words and words[-1] not in casing_set: ans.append(words[-1]) processed = " ".join(ans) processed = re.sub(r"<[^>]+>", "", processed) return re.sub(r"\s+", " ", processed).strip() def _ints(value) -> tuple[int, ...]: if isinstance(value, str): return tuple(map(int, value.split(","))) if isinstance(value, int): return (value,) return tuple(map(int, value)) def _build_model(config: dict, vocab_size: int, blank_id: int) -> AsrModel: """Same construction as the frozen Icefall train.get_model, without training imports.""" if config.get("causal") is not False or int(config["context_size"]) != 3: raise ValueError("This checkpoint requires offline causal=False and context_size=3") if not config.get("use_transducer", True) or any( config.get(name, False) for name in ("use_ctc", "use_attention_decoder", "use_cr_ctc") ): raise ValueError("Expected the transducer-only checkpoint") encoder_dims = _ints(config["encoder_dim"]) encoder_embed = Conv2dSubsampling( in_channels=int(config.get("feature_dim", 80)), out_channels=encoder_dims[0], dropout=ScheduledFloat((0.0, 0.3), (20000.0, 0.1)), ) encoder = Zipformer2( output_downsampling_factor=2, downsampling_factor=_ints(config["downsampling_factor"]), num_encoder_layers=_ints(config["num_encoder_layers"]), encoder_dim=encoder_dims, encoder_unmasked_dim=_ints(config["encoder_unmasked_dim"]), query_head_dim=_ints(config["query_head_dim"]), pos_head_dim=_ints(config["pos_head_dim"]), value_head_dim=_ints(config["value_head_dim"]), pos_dim=int(config["pos_dim"]), num_heads=_ints(config["num_heads"]), feedforward_dim=_ints(config["feedforward_dim"]), cnn_module_kernel=_ints(config["cnn_module_kernel"]), dropout=ScheduledFloat((0.0, 0.3), (20000.0, 0.1)), warmup_batches=4000.0, causal=False, chunk_size=_ints(config["chunk_size"]), left_context_frames=_ints(config["left_context_frames"]), ) decoder_dim = int(config["decoder_dim"]) decoder = Decoder(vocab_size, decoder_dim, blank_id, int(config["context_size"])) joiner = Joiner(max(encoder_dims), decoder_dim, int(config["joiner_dim"]), vocab_size) return AsrModel( encoder_embed=encoder_embed, encoder=encoder, decoder=decoder, joiner=joiner, attention_decoder=None, encoder_dim=max(encoder_dims), decoder_dim=decoder_dim, vocab_size=vocab_size, use_transducer=True, use_ctc=False, use_attention_decoder=False, ) class SpeedASR: """65.55M-parameter offline English recognizer. Inputs are finite, mono, one-dimensional float waveform arrays at 16 kHz. ``transcribe_batch`` preserves input order and adapts GPU batches to memory. No audio, transcripts or credentials are sent to a service during inference. """ def __init__( self, checkpoint: str | Path | None = None, tokens: str | Path | None = None, config: str | Path | None = None, device: str | torch.device = "auto", batch_size: int = 2048, batch_audio_seconds: float = 5000.0, torch_threads: int = 16, verify_hashes: bool = True, ): root = Path(__file__).resolve().parent checkpoint_path = Path(checkpoint).expanduser().resolve() if checkpoint else root / "pretrained.pt" if checkpoint_path.is_dir(): checkpoint_path = checkpoint_path / "pretrained.pt" tokens_path = Path(tokens).expanduser().resolve() if tokens else checkpoint_path.parent / "tokens.txt" config_path = Path(config).expanduser().resolve() if config else root / "model_config.json" for path in (checkpoint_path, tokens_path, config_path): if not path.is_file(): raise FileNotFoundError(path) self.config = json.loads(config_path.read_text(encoding="utf-8")) architecture = self.config.get("model_args", self.config) inference = self.config.get("inference", {}) expected = self.config.get("expected", {}) if verify_hashes: for path, key in ((checkpoint_path, "checkpoint_sha256"), (tokens_path, "tokens_sha256")): if expected.get(key) and _sha256(path) != expected[key]: raise ValueError(f"Artifact hash mismatch: {path.name}") self.token_table = _read_tokens(tokens_path) inverse_tokens = {text: token_id for token_id, text in self.token_table.items()} # Equivalent to icefall.utils.num_tokens(table) + 1: exclude disambiguation symbols. token_ids = [token_id for token_id, token in self.token_table.items() if not re.match(r"^#\d+$", token)] vocab_size = len(token_ids) - int(0 in token_ids) + 1 for key, value in (("vocab_size", vocab_size), ("blank_id", inverse_tokens[""]), ("unk_id", inverse_tokens[""])): if key in architecture and int(architecture[key]) != value: raise ValueError(f"Token table and model configuration disagree on {key}") torch.set_num_threads(int(torch_threads)) if str(device) == "auto": device = "cuda:0" if torch.cuda.is_available() else "cpu" self.device = torch.device(device) if self.device.type not in {"cpu", "cuda"}: raise ValueError("Supported devices are cpu and cuda") if self.device.type == "cuda": if not torch.cuda.is_available(): raise RuntimeError("CUDA was requested but is unavailable") torch.cuda.set_device(self.device) self.model = _build_model(architecture, vocab_size, inverse_tokens[""]) checkpoint_data = torch.load(checkpoint_path, map_location="cpu", weights_only=True) if not isinstance(checkpoint_data, dict) or not isinstance(checkpoint_data.get("model"), dict): raise ValueError("Expected an Icefall checkpoint with a model state dictionary") self.model.load_state_dict(checkpoint_data["model"], strict=True) del checkpoint_data self.num_parameters = sum(parameter.numel() for parameter in self.model.parameters()) if expected.get("parameters") and self.num_parameters != int(expected["parameters"]): raise ValueError(f"Parameter count mismatch: {self.num_parameters}") # Deliberately FP32. Quantized/FP16 accuracy was not measured by this benchmark. self.model.float().to(self.device).eval() self.max_gpu_allocated_bytes = 0 self.sample_rate = int(inference.get("sample_rate", 16000)) self.min_input_seconds = float(inference.get("min_input_seconds", 1.0)) self.long_chunk_seconds = float(inference.get("long_chunk_seconds", 30.0)) self.long_chunk_batch_size = int(inference.get("long_chunk_batch_size", 512)) self.beam_size = int(inference.get("beam_size", 4)) self.batch_size = int(batch_size) self.batch_audio_seconds = float(batch_audio_seconds) if self.batch_size < 1 or self.batch_audio_seconds <= 0: raise ValueError("Batch limits must be positive") opts = kaldifeat.FbankOptions() opts.device = self.device opts.frame_opts.dither = 0 opts.frame_opts.snip_edges = False opts.frame_opts.samp_freq = self.sample_rate opts.mel_opts.num_bins = int(architecture.get("feature_dim", 80)) opts.mel_opts.high_freq = -400 self.fbank = kaldifeat.Fbank(opts) @classmethod def from_pretrained( cls, repo_id: str = "SpeedAudio261119/SpeedASR-v1-65M", revision: str = "main", token: str | bool | None = None, cache_dir: str | Path | None = None, **kwargs, ) -> "SpeedASR": """Download only weights/tokens/config. Pin a commit SHA for reproducibility.""" from huggingface_hub import hf_hub_download paths = { name: hf_hub_download(repo_id=repo_id, filename=name, revision=revision, token=token, cache_dir=cache_dir) for name in ("pretrained.pt", "tokens.txt", "model_config.json") } return cls(checkpoint=paths["pretrained.pt"], tokens=paths["tokens.txt"], config=paths["model_config.json"], **kwargs) def _wave(self, audio) -> torch.Tensor: if isinstance(audio, torch.Tensor): if not audio.is_floating_point(): raise ValueError("Waveforms must contain floating-point samples, not PCM integers") wave = audio.detach().to(device="cpu", dtype=torch.float32).contiguous() else: array = np.asarray(audio) if not np.issubdtype(array.dtype, np.floating): raise ValueError("Waveforms must contain floating-point samples, not PCM integers") wave = torch.from_numpy(np.ascontiguousarray(array, dtype=np.float32)) if wave.ndim != 1 or wave.numel() == 0: raise ValueError("Expected a nonempty one-dimensional mono waveform") if not bool(torch.isfinite(wave).all()): raise ValueError("Waveform contains non-finite samples") return wave def _pad(self, wave: torch.Tensor) -> torch.Tensor: minimum = int(round(self.min_input_seconds * self.sample_rate)) if wave.numel() < minimum: wave = torch.nn.functional.pad(wave, (0, minimum - wave.numel())) return wave @torch.no_grad() def _decode(self, waves: list[torch.Tensor]) -> list[str]: waves = [self._pad(wave).to(self.device) for wave in waves] features = self.fbank(waves) lengths = torch.tensor([f.size(0) for f in features], device=self.device, dtype=torch.int64) features = pad_sequence(features, batch_first=True, padding_value=math.log(1e-10)) with torch.autocast(device_type=self.device.type, enabled=False): encoded, encoded_lengths = self.model.forward_encoder(features, lengths) hypotheses = modified_beam_search( model=self.model, encoder_out=encoded, encoder_out_lens=encoded_lengths, beam=self.beam_size, ) if len(hypotheses) != len(waves): raise RuntimeError("Decoder returned the wrong number of hypotheses") result = [] for hypothesis in hypotheses: text = "".join(self.token_table.get(int(token), "") for token in hypothesis) result.append(restore_voicea_text(text.replace("\u2581", " ").strip())) if self.device.type == "cuda": self.max_gpu_allocated_bytes = max( self.max_gpu_allocated_bytes, torch.cuda.max_memory_allocated(self.device) ) return result @staticmethod def _is_oom(exc: RuntimeError) -> bool: return isinstance(exc, torch.cuda.OutOfMemoryError) or "out of memory" in str(exc).lower() def _decode_long(self, wave: torch.Tensor) -> str: samples = max(1, int(round(self.long_chunk_seconds * self.sample_rate))) chunks = [wave[start:start + samples] for start in range(0, wave.numel(), samples)] batch_size = max(1, min(self.long_chunk_batch_size, len(chunks))) output = [] offset = 0 while offset < len(chunks): count = min(batch_size, len(chunks) - offset) try: output.extend(self._decode(chunks[offset:offset + count])) except RuntimeError as exc: if self.device.type == "cuda" and self._is_oom(exc) and count > 1: torch.cuda.empty_cache() batch_size = max(1, count // 2) continue raise offset += count if batch_size < self.long_chunk_batch_size and count == batch_size: batch_size = min(self.long_chunk_batch_size, batch_size * 2) if self.device.type == "cuda": torch.cuda.empty_cache() return re.sub(r"\s+", " ", " ".join(part for part in output if part)).strip() @torch.no_grad() def transcribe_batch( self, waveforms: Sequence, sample_rate: int = 16000, batch_size: int | None = None, ) -> list[str]: """Decode mono float waveforms and return one string per input, in input order.""" if int(sample_rate) != self.sample_rate: raise ValueError(f"Expected {self.sample_rate} Hz; resample before calling inference") waves = [self._wave(wave) for wave in waveforms] if not waves: return [] target_batch_size = self.batch_size if batch_size is None else int(batch_size) if target_batch_size < 1: raise ValueError("batch_size must be positive") order = sorted(range(len(waves)), key=lambda i: waves[i].numel(), reverse=True) outputs = [""] * len(waves) offset = 0 safe_seconds = self.batch_audio_seconds while offset < len(order): first = waves[order[offset]] seconds = first.numel() / self.sample_rate if seconds > self.long_chunk_seconds: outputs[order[offset]] = self._decode_long(first) offset += 1 continue count = min(target_batch_size, max(1, int(safe_seconds / max(1.0, seconds)))) count = min(count, len(order) - offset) indices = order[offset:offset + count] try: if self.device.type == "cuda": torch.cuda.reset_peak_memory_stats(self.device) texts = self._decode([waves[i] for i in indices]) except RuntimeError as exc: if self.device.type != "cuda" or not self._is_oom(exc): raise torch.cuda.empty_cache() if count == 1: outputs[indices[0]] = self._decode_long(first) offset += 1 else: safe_seconds = max(1, count // 2) * max(1.0, seconds) logging.warning("CUDA memory limit; retrying with a smaller batch") continue for index, text in zip(indices, texts): outputs[index] = text if self.device.type == "cuda": peak = torch.cuda.max_memory_allocated(self.device) target_memory = torch.cuda.get_device_properties(self.device).total_memory * 0.80 if peak > 0 and peak < target_memory * 0.85: safe_seconds *= min(1.15, target_memory / peak) offset += count if self.device.type == "cuda": torch.cuda.synchronize(self.device) return outputs def transcribe(self, waveform, sample_rate: int = 16000) -> str: """Decode one mono float waveform at 16 kHz.""" return self.transcribe_batch([waveform], sample_rate=sample_rate)[0] def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("audio", nargs="+", type=Path, help="16 kHz WAV/FLAC files") parser.add_argument("--checkpoint", type=Path) parser.add_argument("--tokens", type=Path) parser.add_argument("--config", type=Path) parser.add_argument("--repo-id", help="Download model artifacts from this Hub repository") parser.add_argument("--revision", default="main", help="Prefer an immutable Hub commit SHA") parser.add_argument("--device", default="auto") parser.add_argument("--batch-size", type=int, default=2048) parser.add_argument("--batch-audio-seconds", type=float, default=5000.0) parser.add_argument("--torch-threads", type=int, default=16) args = parser.parse_args() options = dict(device=args.device, batch_size=args.batch_size, batch_audio_seconds=args.batch_audio_seconds, torch_threads=args.torch_threads) if args.repo_id: model = SpeedASR.from_pretrained(args.repo_id, revision=args.revision, **options) else: model = SpeedASR(args.checkpoint, args.tokens, args.config, **options) waves = [] for path in args.audio: audio, sample_rate = sf.read(path, dtype="float32", always_2d=True) if sample_rate != model.sample_rate: parser.error(f"{path}: expected {model.sample_rate} Hz, got {sample_rate}") waves.append(np.ascontiguousarray(audio[:, 0])) for path, text in zip(args.audio, model.transcribe_batch(waves)): print(json.dumps({"audio_filepath": str(path), "pred_text": text}, ensure_ascii=False)) if __name__ == "__main__": main()