EdgeASR / evaluator.py
frankyoujian's picture
Add offline EdgeASR decoder and FFASR runtime materials
496a112 verified
Raw History Blame Contribute Delete
9.48 kB
"""Native icefall Zipformer evaluator for FFASR's offline ASR track.
FFASR calls ``evaluate(Path) -> str`` once per recording. The model is loaded
once at import, using the supplied pretrained.py frontend and greedy decoder.
The returned string restores postfix case/punctuation tags; it is not WER
normalized. Hub use requires an immutable FFASR_MODEL_REVISION commit SHA.
"""
from __future__ import annotations
import argparse
import hashlib
import importlib.util
import json
import math
import os
import re
import sys
from pathlib import Path
import k2
import kaldifeat
import numpy as np
import soundfile as sf
import torch
from scipy.signal import resample_poly
from torch.nn.utils.rnn import pad_sequence
SAMPLE_RATE = 16000
def _artifact_root() -> Path:
local = os.environ.get("FFASR_MODEL_DIR", "").strip()
if local:
root = Path(local).expanduser().resolve()
if not root.is_dir():
raise FileNotFoundError(f"FFASR_MODEL_DIR is not a directory: {root}")
return root
model_id = os.environ.get("FFASR_MODEL_ID", "").strip()
revision = os.environ.get("FFASR_MODEL_REVISION", "").strip()
if not model_id:
raise ValueError("Set FFASR_MODEL_ID to the submitted Hugging Face model repo.")
if not re.fullmatch(r"[0-9a-fA-F]{40}", revision):
raise ValueError("FFASR_MODEL_REVISION must be an immutable 40-character commit SHA.")
from huggingface_hub import snapshot_download
return Path(
snapshot_download(
repo_id=model_id,
revision=revision,
token=os.environ.get("HF_TOKEN") or None,
allow_patterns=[
"artifact_manifest.json", "pretrained.pt", "tokens.txt",
"model_config.json", "text_format.py", "source/**",
],
)
).resolve()
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1 << 20), b""):
digest.update(block)
return digest.hexdigest()
def _verify_artifacts(root: Path) -> None:
manifest_path = root / "artifact_manifest.json"
with manifest_path.open(encoding="utf-8") as stream:
manifest = json.load(stream)
if manifest.get("schema_version") != 1 or not isinstance(manifest.get("files"), dict):
raise ValueError("Expected artifact_manifest.json schema_version=1 and files mapping.")
files = manifest["files"]
required = {
"pretrained.pt", "tokens.txt", "model_config.json", "text_format.py",
"source/egs/librispeech/ASR/zipformer/pretrained.py",
}
missing = required.difference(files)
if missing:
raise ValueError(f"Artifact manifest lacks required files: {sorted(missing)}")
for relative, metadata in files.items():
# Download only runtime inputs, but verify every downloaded artifact in
# the manifest, including the exact recipe and formatter source.
if relative not in required and not relative.startswith("source/"):
continue
if not isinstance(metadata, dict):
raise ValueError(f"Invalid artifact metadata: {relative}")
path = (root / relative).resolve()
if not path.is_relative_to(root) or not path.is_file():
raise FileNotFoundError(f"Missing or invalid artifact path: {relative}")
expected = metadata.get("sha256", "")
if not isinstance(expected, str) or not re.fullmatch(r"[0-9a-fA-F]{64}", expected):
raise ValueError(f"Invalid artifact SHA-256: {relative}")
if path.stat().st_size != metadata.get("bytes"):
raise ValueError(f"Artifact size mismatch: {relative}")
if _sha256(path) != expected.lower():
raise ValueError(f"Artifact SHA-256 mismatch: {relative}")
def _load_module(name: str, path: Path):
spec = importlib.util.spec_from_file_location(name, path)
if spec is None or spec.loader is None:
raise ImportError(f"Cannot load module: {path}")
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module
spec.loader.exec_module(module)
return module
def _native_params(native, root: Path):
with (root / "model_config.json").open(encoding="utf-8") as stream:
config = json.load(stream)
if not isinstance(config, dict):
raise ValueError("model_config.json must contain a model-argument object.")
config = config.get("model_args", config)
if not isinstance(config, dict):
raise ValueError("model_args must be an object.")
architecture_parser = argparse.ArgumentParser(add_help=False)
native.add_model_arguments(architecture_parser)
architecture_parser.add_argument("--context-size", type=int, default=2)
allowed = {action.dest for action in architecture_parser._actions}
argv = [
"--checkpoint", str(root / "pretrained.pt"),
"--tokens", str(root / "tokens.txt"),
"--method", "greedy_search", "--max-sym-per-frame", "1",
]
for key, value in config.items():
key = key.replace("-", "_")
if key not in allowed:
raise ValueError(f"Unknown model architecture argument: {key}")
if isinstance(value, (list, tuple)):
value = ",".join(str(item) for item in value)
elif isinstance(value, bool):
value = str(value).lower()
argv.append(f"--{key.replace('_', '-')}={value}")
argv.append("unused.wav")
parsed = native.get_parser().parse_args(argv)
params = native.get_params()
params.update(vars(parsed))
if not params.use_transducer or params.use_attention_decoder:
raise ValueError("Expected the offline transducer checkpoint without attention decoding.")
if params.causal:
raise ValueError("This submission is the non-causal/offline Zipformer model.")
return params
def _device() -> torch.device:
requested = os.environ.get("FFASR_DEVICE", "auto").strip().lower()
if requested == "auto":
requested = "cuda:0" if torch.cuda.is_available() else "cpu"
device = torch.device(requested)
if device.type not in {"cpu", "cuda"}:
raise ValueError(f"Unsupported FFASR_DEVICE: {requested}")
if device.type == "cuda":
if not torch.cuda.is_available():
raise RuntimeError("CUDA was requested but is unavailable.")
torch.cuda.set_device(device)
return device
_root = _artifact_root()
_verify_artifacts(_root)
_source = _root / "source"
_recipe = _source / "egs" / "librispeech" / "ASR" / "zipformer"
sys.path.insert(0, str(_source))
sys.path.insert(0, str(_recipe))
_native = _load_module("ffasr_icefall_pretrained", _recipe / "pretrained.py")
_format_transcript = _load_module("ffasr_icefall_text_format", _root / "text_format.py").format_transcript
_params = _native_params(_native, _root)
_tokens = k2.SymbolTable.from_file(str(_root / "tokens.txt"))
_params.blank_id = _tokens["<blk>"]
_params.unk_id = _tokens["<unk>"]
_params.vocab_size = _native.num_tokens(_tokens) + 1
_inference_device = _device()
model = _native.get_model(_params)
_checkpoint = torch.load(_root / "pretrained.pt", map_location="cpu", weights_only=True)
if not isinstance(_checkpoint, dict) or not isinstance(_checkpoint.get("model"), dict):
raise ValueError("Expected an icefall checkpoint containing the model state dictionary.")
model.load_state_dict(_checkpoint["model"], strict=True)
del _checkpoint
model.to(_inference_device).eval()
NUM_PARAMS = sum(parameter.numel() for parameter in model.parameters())
_opts = kaldifeat.FbankOptions()
_opts.device = _inference_device
_opts.frame_opts.dither = 0
_opts.frame_opts.snip_edges = False
_opts.frame_opts.samp_freq = SAMPLE_RATE
_opts.mel_opts.num_bins = _params.feature_dim
_opts.mel_opts.high_freq = -400
_fbank = kaldifeat.Fbank(_opts)
@torch.inference_mode()
def evaluate(file: Path) -> str:
"""Transcribe one WAV and restore the model's casing/punctuation tokens."""
audio, sample_rate = sf.read(str(file), dtype="float32", always_2d=True)
if sample_rate <= 0 or audio.shape[0] == 0 or audio.shape[1] == 0:
raise ValueError("Input audio must be nonempty and have a valid sample rate.")
# Match pretrained.py's first-channel policy, including stereo recordings.
audio = audio[:, 0]
if not np.isfinite(audio).all():
raise ValueError("Input audio contains non-finite samples.")
if sample_rate != SAMPLE_RATE:
divisor = math.gcd(int(sample_rate), SAMPLE_RATE)
audio = resample_poly(audio, SAMPLE_RATE // divisor, int(sample_rate) // divisor)
wave = torch.from_numpy(np.ascontiguousarray(audio, dtype=np.float32)).to(_inference_device)
features = _fbank([wave])
lengths = torch.tensor([feature.size(0) for feature in features], device=_inference_device)
padded = pad_sequence(features, batch_first=True, padding_value=math.log(1e-10))
encoded, encoded_lengths = model.forward_encoder(padded, lengths)
hypotheses = _native.greedy_search_batch(
model=model, encoder_out=encoded, encoder_out_lens=encoded_lengths,
)
if len(hypotheses) != 1:
raise RuntimeError("The decoder must return exactly one hypothesis per WAV.")
raw = "".join(_tokens[token] for token in hypotheses[0]).replace("\u2581", " ").strip()
transcript = _format_transcript(raw)
if _inference_device.type == "cuda":
torch.cuda.synchronize(_inference_device)
return transcript