SpeedASR-v1-65M / inference.py
SpeedAudio's picture
Publish SpeedASR-v1-65M weights, portable inference and Open ASR evaluation package
c830bfd verified
Raw History Blame Contribute Delete
19.2 kB
#!/usr/bin/env python3
"""Portable FP32 offline SpeedASR inference, including batched beam search.
model = SpeedASR.from_pretrained(revision="<Hub commit SHA>")
texts = model.transcribe_batch([mono_float32_audio], sample_rate=16000)
This frontend returns decoded text with VoiceA casing/punctuation restored.
WER normalization belongs to the benchmark scorer, not the recognizer.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import logging
import math
import re
from itertools import pairwise
from pathlib import Path
from typing import Iterable, Sequence
import kaldifeat
import numpy as np
import soundfile as sf
import torch
from torch.nn.utils.rnn import pad_sequence
from speedasr_core.beam_search import modified_beam_search
from speedasr_core.decoder import Decoder
from speedasr_core.joiner import Joiner
from speedasr_core.model import AsrModel
from speedasr_core.scaling import ScheduledFloat
from speedasr_core.subsampling import Conv2dSubsampling
from speedasr_core.zipformer import Zipformer2
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1 << 20), b""):
digest.update(block)
return digest.hexdigest()
def _read_tokens(path: Path) -> dict[int, str]:
table = {}
for line in path.read_text(encoding="utf-8").splitlines():
parts = line.split()
if len(parts) >= 2:
token_id = int(parts[-1])
if token_id in table:
raise ValueError(f"Duplicate token ID: {token_id}")
table[token_id] = " ".join(parts[:-1])
if not table or "<blk>" not in table.values() or "<unk>" not in table.values():
raise ValueError("tokens.txt must contain <blk> and <unk>")
return table
def restore_voicea_text(text: str) -> str:
"""The exact VoiceA postprocessing used by the local benchmark."""
if not text:
return text
processed = (
text.replace("<,>", ",").replace("<.>", ".")
.replace("<?>", "?").replace("<!>", "!").replace("<unk>", "")
)
words = processed.split()
casing_set = {"<cap>", "<upp>", "<mix>"}
ans = []
for w1, w2 in pairwise(words):
if w1 in casing_set:
continue
if w2 in casing_set:
if w2 == "<cap>":
w1 = w1.capitalize()
elif w2 == "<upp>":
w1 = w1.upper()
ans.append(w1)
if words and words[-1] not in casing_set:
ans.append(words[-1])
processed = " ".join(ans)
processed = re.sub(r"<[^>]+>", "", processed)
return re.sub(r"\s+", " ", processed).strip()
def _ints(value) -> tuple[int, ...]:
if isinstance(value, str):
return tuple(map(int, value.split(",")))
if isinstance(value, int):
return (value,)
return tuple(map(int, value))
def _build_model(config: dict, vocab_size: int, blank_id: int) -> AsrModel:
"""Same construction as the frozen Icefall train.get_model, without training imports."""
if config.get("causal") is not False or int(config["context_size"]) != 3:
raise ValueError("This checkpoint requires offline causal=False and context_size=3")
if not config.get("use_transducer", True) or any(
config.get(name, False) for name in ("use_ctc", "use_attention_decoder", "use_cr_ctc")
):
raise ValueError("Expected the transducer-only checkpoint")
encoder_dims = _ints(config["encoder_dim"])
encoder_embed = Conv2dSubsampling(
in_channels=int(config.get("feature_dim", 80)),
out_channels=encoder_dims[0],
dropout=ScheduledFloat((0.0, 0.3), (20000.0, 0.1)),
)
encoder = Zipformer2(
output_downsampling_factor=2,
downsampling_factor=_ints(config["downsampling_factor"]),
num_encoder_layers=_ints(config["num_encoder_layers"]),
encoder_dim=encoder_dims,
encoder_unmasked_dim=_ints(config["encoder_unmasked_dim"]),
query_head_dim=_ints(config["query_head_dim"]),
pos_head_dim=_ints(config["pos_head_dim"]),
value_head_dim=_ints(config["value_head_dim"]),
pos_dim=int(config["pos_dim"]),
num_heads=_ints(config["num_heads"]),
feedforward_dim=_ints(config["feedforward_dim"]),
cnn_module_kernel=_ints(config["cnn_module_kernel"]),
dropout=ScheduledFloat((0.0, 0.3), (20000.0, 0.1)),
warmup_batches=4000.0,
causal=False,
chunk_size=_ints(config["chunk_size"]),
left_context_frames=_ints(config["left_context_frames"]),
)
decoder_dim = int(config["decoder_dim"])
decoder = Decoder(vocab_size, decoder_dim, blank_id, int(config["context_size"]))
joiner = Joiner(max(encoder_dims), decoder_dim, int(config["joiner_dim"]), vocab_size)
return AsrModel(
encoder_embed=encoder_embed, encoder=encoder, decoder=decoder, joiner=joiner,
attention_decoder=None, encoder_dim=max(encoder_dims), decoder_dim=decoder_dim,
vocab_size=vocab_size, use_transducer=True, use_ctc=False, use_attention_decoder=False,
)
class SpeedASR:
"""65.55M-parameter offline English recognizer.
Inputs are finite, mono, one-dimensional float waveform arrays at 16 kHz.
``transcribe_batch`` preserves input order and adapts GPU batches to memory.
No audio, transcripts or credentials are sent to a service during inference.
"""
def __init__(
self,
checkpoint: str | Path | None = None,
tokens: str | Path | None = None,
config: str | Path | None = None,
device: str | torch.device = "auto",
batch_size: int = 2048,
batch_audio_seconds: float = 5000.0,
torch_threads: int = 16,
verify_hashes: bool = True,
):
root = Path(__file__).resolve().parent
checkpoint_path = Path(checkpoint).expanduser().resolve() if checkpoint else root / "pretrained.pt"
if checkpoint_path.is_dir():
checkpoint_path = checkpoint_path / "pretrained.pt"
tokens_path = Path(tokens).expanduser().resolve() if tokens else checkpoint_path.parent / "tokens.txt"
config_path = Path(config).expanduser().resolve() if config else root / "model_config.json"
for path in (checkpoint_path, tokens_path, config_path):
if not path.is_file():
raise FileNotFoundError(path)
self.config = json.loads(config_path.read_text(encoding="utf-8"))
architecture = self.config.get("model_args", self.config)
inference = self.config.get("inference", {})
expected = self.config.get("expected", {})
if verify_hashes:
for path, key in ((checkpoint_path, "checkpoint_sha256"), (tokens_path, "tokens_sha256")):
if expected.get(key) and _sha256(path) != expected[key]:
raise ValueError(f"Artifact hash mismatch: {path.name}")
self.token_table = _read_tokens(tokens_path)
inverse_tokens = {text: token_id for token_id, text in self.token_table.items()}
# Equivalent to icefall.utils.num_tokens(table) + 1: exclude disambiguation symbols.
token_ids = [token_id for token_id, token in self.token_table.items()
if not re.match(r"^#\d+$", token)]
vocab_size = len(token_ids) - int(0 in token_ids) + 1
for key, value in (("vocab_size", vocab_size), ("blank_id", inverse_tokens["<blk>"]),
("unk_id", inverse_tokens["<unk>"])):
if key in architecture and int(architecture[key]) != value:
raise ValueError(f"Token table and model configuration disagree on {key}")
torch.set_num_threads(int(torch_threads))
if str(device) == "auto":
device = "cuda:0" if torch.cuda.is_available() else "cpu"
self.device = torch.device(device)
if self.device.type not in {"cpu", "cuda"}:
raise ValueError("Supported devices are cpu and cuda")
if self.device.type == "cuda":
if not torch.cuda.is_available():
raise RuntimeError("CUDA was requested but is unavailable")
torch.cuda.set_device(self.device)
self.model = _build_model(architecture, vocab_size, inverse_tokens["<blk>"])
checkpoint_data = torch.load(checkpoint_path, map_location="cpu", weights_only=True)
if not isinstance(checkpoint_data, dict) or not isinstance(checkpoint_data.get("model"), dict):
raise ValueError("Expected an Icefall checkpoint with a model state dictionary")
self.model.load_state_dict(checkpoint_data["model"], strict=True)
del checkpoint_data
self.num_parameters = sum(parameter.numel() for parameter in self.model.parameters())
if expected.get("parameters") and self.num_parameters != int(expected["parameters"]):
raise ValueError(f"Parameter count mismatch: {self.num_parameters}")
# Deliberately FP32. Quantized/FP16 accuracy was not measured by this benchmark.
self.model.float().to(self.device).eval()
self.max_gpu_allocated_bytes = 0
self.sample_rate = int(inference.get("sample_rate", 16000))
self.min_input_seconds = float(inference.get("min_input_seconds", 1.0))
self.long_chunk_seconds = float(inference.get("long_chunk_seconds", 30.0))
self.long_chunk_batch_size = int(inference.get("long_chunk_batch_size", 512))
self.beam_size = int(inference.get("beam_size", 4))
self.batch_size = int(batch_size)
self.batch_audio_seconds = float(batch_audio_seconds)
if self.batch_size < 1 or self.batch_audio_seconds <= 0:
raise ValueError("Batch limits must be positive")
opts = kaldifeat.FbankOptions()
opts.device = self.device
opts.frame_opts.dither = 0
opts.frame_opts.snip_edges = False
opts.frame_opts.samp_freq = self.sample_rate
opts.mel_opts.num_bins = int(architecture.get("feature_dim", 80))
opts.mel_opts.high_freq = -400
self.fbank = kaldifeat.Fbank(opts)
@classmethod
def from_pretrained(
cls,
repo_id: str = "SpeedAudio261119/SpeedASR-v1-65M",
revision: str = "main",
token: str | bool | None = None,
cache_dir: str | Path | None = None,
**kwargs,
) -> "SpeedASR":
"""Download only weights/tokens/config. Pin a commit SHA for reproducibility."""
from huggingface_hub import hf_hub_download
paths = {
name: hf_hub_download(repo_id=repo_id, filename=name, revision=revision,
token=token, cache_dir=cache_dir)
for name in ("pretrained.pt", "tokens.txt", "model_config.json")
}
return cls(checkpoint=paths["pretrained.pt"], tokens=paths["tokens.txt"],
config=paths["model_config.json"], **kwargs)
def _wave(self, audio) -> torch.Tensor:
if isinstance(audio, torch.Tensor):
if not audio.is_floating_point():
raise ValueError("Waveforms must contain floating-point samples, not PCM integers")
wave = audio.detach().to(device="cpu", dtype=torch.float32).contiguous()
else:
array = np.asarray(audio)
if not np.issubdtype(array.dtype, np.floating):
raise ValueError("Waveforms must contain floating-point samples, not PCM integers")
wave = torch.from_numpy(np.ascontiguousarray(array, dtype=np.float32))
if wave.ndim != 1 or wave.numel() == 0:
raise ValueError("Expected a nonempty one-dimensional mono waveform")
if not bool(torch.isfinite(wave).all()):
raise ValueError("Waveform contains non-finite samples")
return wave
def _pad(self, wave: torch.Tensor) -> torch.Tensor:
minimum = int(round(self.min_input_seconds * self.sample_rate))
if wave.numel() < minimum:
wave = torch.nn.functional.pad(wave, (0, minimum - wave.numel()))
return wave
@torch.no_grad()
def _decode(self, waves: list[torch.Tensor]) -> list[str]:
waves = [self._pad(wave).to(self.device) for wave in waves]
features = self.fbank(waves)
lengths = torch.tensor([f.size(0) for f in features], device=self.device, dtype=torch.int64)
features = pad_sequence(features, batch_first=True, padding_value=math.log(1e-10))
with torch.autocast(device_type=self.device.type, enabled=False):
encoded, encoded_lengths = self.model.forward_encoder(features, lengths)
hypotheses = modified_beam_search(
model=self.model, encoder_out=encoded, encoder_out_lens=encoded_lengths,
beam=self.beam_size,
)
if len(hypotheses) != len(waves):
raise RuntimeError("Decoder returned the wrong number of hypotheses")
result = []
for hypothesis in hypotheses:
text = "".join(self.token_table.get(int(token), "") for token in hypothesis)
result.append(restore_voicea_text(text.replace("\u2581", " ").strip()))
if self.device.type == "cuda":
self.max_gpu_allocated_bytes = max(
self.max_gpu_allocated_bytes, torch.cuda.max_memory_allocated(self.device)
)
return result
@staticmethod
def _is_oom(exc: RuntimeError) -> bool:
return isinstance(exc, torch.cuda.OutOfMemoryError) or "out of memory" in str(exc).lower()
def _decode_long(self, wave: torch.Tensor) -> str:
samples = max(1, int(round(self.long_chunk_seconds * self.sample_rate)))
chunks = [wave[start:start + samples] for start in range(0, wave.numel(), samples)]
batch_size = max(1, min(self.long_chunk_batch_size, len(chunks)))
output = []
offset = 0
while offset < len(chunks):
count = min(batch_size, len(chunks) - offset)
try:
output.extend(self._decode(chunks[offset:offset + count]))
except RuntimeError as exc:
if self.device.type == "cuda" and self._is_oom(exc) and count > 1:
torch.cuda.empty_cache()
batch_size = max(1, count // 2)
continue
raise
offset += count
if batch_size < self.long_chunk_batch_size and count == batch_size:
batch_size = min(self.long_chunk_batch_size, batch_size * 2)
if self.device.type == "cuda":
torch.cuda.empty_cache()
return re.sub(r"\s+", " ", " ".join(part for part in output if part)).strip()
@torch.no_grad()
def transcribe_batch(
self,
waveforms: Sequence,
sample_rate: int = 16000,
batch_size: int | None = None,
) -> list[str]:
"""Decode mono float waveforms and return one string per input, in input order."""
if int(sample_rate) != self.sample_rate:
raise ValueError(f"Expected {self.sample_rate} Hz; resample before calling inference")
waves = [self._wave(wave) for wave in waveforms]
if not waves:
return []
target_batch_size = self.batch_size if batch_size is None else int(batch_size)
if target_batch_size < 1:
raise ValueError("batch_size must be positive")
order = sorted(range(len(waves)), key=lambda i: waves[i].numel(), reverse=True)
outputs = [""] * len(waves)
offset = 0
safe_seconds = self.batch_audio_seconds
while offset < len(order):
first = waves[order[offset]]
seconds = first.numel() / self.sample_rate
if seconds > self.long_chunk_seconds:
outputs[order[offset]] = self._decode_long(first)
offset += 1
continue
count = min(target_batch_size, max(1, int(safe_seconds / max(1.0, seconds))))
count = min(count, len(order) - offset)
indices = order[offset:offset + count]
try:
if self.device.type == "cuda":
torch.cuda.reset_peak_memory_stats(self.device)
texts = self._decode([waves[i] for i in indices])
except RuntimeError as exc:
if self.device.type != "cuda" or not self._is_oom(exc):
raise
torch.cuda.empty_cache()
if count == 1:
outputs[indices[0]] = self._decode_long(first)
offset += 1
else:
safe_seconds = max(1, count // 2) * max(1.0, seconds)
logging.warning("CUDA memory limit; retrying with a smaller batch")
continue
for index, text in zip(indices, texts):
outputs[index] = text
if self.device.type == "cuda":
peak = torch.cuda.max_memory_allocated(self.device)
target_memory = torch.cuda.get_device_properties(self.device).total_memory * 0.80
if peak > 0 and peak < target_memory * 0.85:
safe_seconds *= min(1.15, target_memory / peak)
offset += count
if self.device.type == "cuda":
torch.cuda.synchronize(self.device)
return outputs
def transcribe(self, waveform, sample_rate: int = 16000) -> str:
"""Decode one mono float waveform at 16 kHz."""
return self.transcribe_batch([waveform], sample_rate=sample_rate)[0]
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("audio", nargs="+", type=Path, help="16 kHz WAV/FLAC files")
parser.add_argument("--checkpoint", type=Path)
parser.add_argument("--tokens", type=Path)
parser.add_argument("--config", type=Path)
parser.add_argument("--repo-id", help="Download model artifacts from this Hub repository")
parser.add_argument("--revision", default="main", help="Prefer an immutable Hub commit SHA")
parser.add_argument("--device", default="auto")
parser.add_argument("--batch-size", type=int, default=2048)
parser.add_argument("--batch-audio-seconds", type=float, default=5000.0)
parser.add_argument("--torch-threads", type=int, default=16)
args = parser.parse_args()
options = dict(device=args.device, batch_size=args.batch_size,
batch_audio_seconds=args.batch_audio_seconds, torch_threads=args.torch_threads)
if args.repo_id:
model = SpeedASR.from_pretrained(args.repo_id, revision=args.revision, **options)
else:
model = SpeedASR(args.checkpoint, args.tokens, args.config, **options)
waves = []
for path in args.audio:
audio, sample_rate = sf.read(path, dtype="float32", always_2d=True)
if sample_rate != model.sample_rate:
parser.error(f"{path}: expected {model.sample_rate} Hz, got {sample_rate}")
waves.append(np.ascontiguousarray(audio[:, 0]))
for path, text in zip(args.audio, model.transcribe_batch(waves)):
print(json.dumps({"audio_filepath": str(path), "pred_text": text}, ensure_ascii=False))
if __name__ == "__main__":
main()