#!/usr/bin/env python3 # SPDX-License-Identifier: AGPL-3.0-or-later """Word timings for an audio file and its text with CavaAlign. pip install onnxruntime numpy soundfile regex num2words python align.py speech.wav text.txt --lang en > words.json """ import argparse import json import os import numpy as np import onnxruntime as ort import soundfile as sf import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from cavaalign.align import AlignConfig, expansion_to_tokens, forced_align, linear_band_center, spans_to_words from cavaalign.romanize import prepare_text from cavaalign.vocab import vocab_for_tokens HERE = os.path.dirname(os.path.abspath(__file__)) def load(model=os.path.join(HERE, "cavaalign.onnx")): so = ort.SessionOptions() so.enable_cpu_mem_arena = False # keeps memory flat on long recordings so.enable_mem_pattern = False sess = ort.InferenceSession(model, so, providers=["CPUExecutionProvider"]) with open(os.path.join(HERE, "align_config.json"), encoding="utf-8") as f: cfg = json.load(f) return sess, cfg CHUNK = 32 * 320 # the model's attention chunk: 32 frames of 20 ms = 10,240 samples def log_probs(sess, audio, window_chunks=10, context_chunks=5): """Model output for any length of audio in bounded memory. Attention memory grows with the square of the input length, so long audio is run in windows of `window_chunks` chunks (6.4 s) with `context_chunks` (3.2 s) of preceding audio as context, cut on chunk boundaries. Same precision as one pass, about 0.3 GB peak. """ a = np.asarray(audio, dtype=np.float32) out, pos = [], 0 while pos < len(a): st = max(0, pos - context_chunks * CHUNK) x = a[st: pos + window_chunks * CHUNK + 80] # +80: the last frame spans 400 samples lp, n = sess.run(None, {"waveform": x[None], "lengths": np.array([len(x)], dtype=np.int64)}) skip = (pos - st) // 320 out.append(lp[0, skip: min(int(n[0]), skip + window_chunks * 32)]) pos += window_chunks * CHUNK return np.concatenate(out) def align(sess, cfg, audio, text, lang="en"): """audio: float32 mono 16 kHz. Returns [{word, start, end}] for every word of `text`.""" lp = log_probs(sess, audio) exp = prepare_text(text, lang) toks = expansion_to_tokens(exp, vocab_for_tokens(cfg["vocab_tokens"])) res = forced_align(lp, toks, AlignConfig(**cfg["align"]), linear_band_center(len(lp), len(toks))) words = spans_to_words(text, res.spans, cfg["frame_shift_sec"], expansion=exp, emissions=lp) return [{"word": exp.words[i], "start": round(s, 3), "end": round(e, 3)} for i, s, e, _ in words] if __name__ == "__main__": ap = argparse.ArgumentParser() ap.add_argument("audio") ap.add_argument("text") ap.add_argument("--lang", default="en", help="ISO 639-1 code of the text (see README for the 35 languages)") a = ap.parse_args() audio, sr = sf.read(a.audio, dtype="float32") if audio.ndim > 1: audio = audio.mean(1) if sr != 16000: raise SystemExit(f"{a.audio}: expected 16 kHz, got {sr} Hz (resample first, e.g. ffmpeg -ar 16000 -ac 1)") sess, cfg = load() with open(a.text, encoding="utf-8") as f: text = f.read() print(json.dumps(align(sess, cfg, audio, text, a.lang), ensure_ascii=False, indent=1))