cavaalign-v1 / align.py
micr0-dev's picture
CavaAlign v1
8f8b0fa
Raw History Blame Contribute Delete
3.39 kB
#!/usr/bin/env python3
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Word timings for an audio file and its text with CavaAlign.
pip install onnxruntime numpy soundfile regex num2words
python align.py speech.wav text.txt --lang en > words.json
"""
import argparse
import json
import os
import numpy as np
import onnxruntime as ort
import soundfile as sf
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from cavaalign.align import AlignConfig, expansion_to_tokens, forced_align, linear_band_center, spans_to_words
from cavaalign.romanize import prepare_text
from cavaalign.vocab import vocab_for_tokens
HERE = os.path.dirname(os.path.abspath(__file__))
def load(model=os.path.join(HERE, "cavaalign.onnx")):
so = ort.SessionOptions()
so.enable_cpu_mem_arena = False # keeps memory flat on long recordings
so.enable_mem_pattern = False
sess = ort.InferenceSession(model, so, providers=["CPUExecutionProvider"])
with open(os.path.join(HERE, "align_config.json"), encoding="utf-8") as f:
cfg = json.load(f)
return sess, cfg
CHUNK = 32 * 320 # the model's attention chunk: 32 frames of 20 ms = 10,240 samples
def log_probs(sess, audio, window_chunks=10, context_chunks=5):
"""Model output for any length of audio in bounded memory.
Attention memory grows with the square of the input length, so long audio is run in windows
of `window_chunks` chunks (6.4 s) with `context_chunks` (3.2 s) of preceding audio as
context, cut on chunk boundaries. Same precision as one pass, about 0.3 GB peak.
"""
a = np.asarray(audio, dtype=np.float32)
out, pos = [], 0
while pos < len(a):
st = max(0, pos - context_chunks * CHUNK)
x = a[st: pos + window_chunks * CHUNK + 80] # +80: the last frame spans 400 samples
lp, n = sess.run(None, {"waveform": x[None], "lengths": np.array([len(x)], dtype=np.int64)})
skip = (pos - st) // 320
out.append(lp[0, skip: min(int(n[0]), skip + window_chunks * 32)])
pos += window_chunks * CHUNK
return np.concatenate(out)
def align(sess, cfg, audio, text, lang="en"):
"""audio: float32 mono 16 kHz. Returns [{word, start, end}] for every word of `text`."""
lp = log_probs(sess, audio)
exp = prepare_text(text, lang)
toks = expansion_to_tokens(exp, vocab_for_tokens(cfg["vocab_tokens"]))
res = forced_align(lp, toks, AlignConfig(**cfg["align"]), linear_band_center(len(lp), len(toks)))
words = spans_to_words(text, res.spans, cfg["frame_shift_sec"], expansion=exp, emissions=lp)
return [{"word": exp.words[i], "start": round(s, 3), "end": round(e, 3)} for i, s, e, _ in words]
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("audio")
ap.add_argument("text")
ap.add_argument("--lang", default="en", help="ISO 639-1 code of the text (see README for the 35 languages)")
a = ap.parse_args()
audio, sr = sf.read(a.audio, dtype="float32")
if audio.ndim > 1:
audio = audio.mean(1)
if sr != 16000:
raise SystemExit(f"{a.audio}: expected 16 kHz, got {sr} Hz (resample first, e.g. ffmpeg -ar 16000 -ac 1)")
sess, cfg = load()
with open(a.text, encoding="utf-8") as f:
text = f.read()
print(json.dumps(align(sess, cfg, audio, text, a.lang), ensure_ascii=False, indent=1))