Download align.py from cava-reader/cavaalign-v1: direct link, hf CLI and curl.
- Browser
- Download file 3.39 kB
-
https://huggingface.co/cava-reader/cavaalign-v1/resolve/main/align.py
- Command line
-
hf download hf://cava-reader/cavaalign-v1/align.py
-
curl -L -o align.py https://huggingface.co/cava-reader/cavaalign-v1/resolve/main/align.py
3.39 kB
| #!/usr/bin/env python3 | |
| # SPDX-License-Identifier: AGPL-3.0-or-later | |
| """Word timings for an audio file and its text with CavaAlign. | |
| pip install onnxruntime numpy soundfile regex num2words | |
| python align.py speech.wav text.txt --lang en > words.json | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import numpy as np | |
| import onnxruntime as ort | |
| import soundfile as sf | |
| import sys | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| from cavaalign.align import AlignConfig, expansion_to_tokens, forced_align, linear_band_center, spans_to_words | |
| from cavaalign.romanize import prepare_text | |
| from cavaalign.vocab import vocab_for_tokens | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| def load(model=os.path.join(HERE, "cavaalign.onnx")): | |
| so = ort.SessionOptions() | |
| so.enable_cpu_mem_arena = False # keeps memory flat on long recordings | |
| so.enable_mem_pattern = False | |
| sess = ort.InferenceSession(model, so, providers=["CPUExecutionProvider"]) | |
| with open(os.path.join(HERE, "align_config.json"), encoding="utf-8") as f: | |
| cfg = json.load(f) | |
| return sess, cfg | |
| CHUNK = 32 * 320 # the model's attention chunk: 32 frames of 20 ms = 10,240 samples | |
| def log_probs(sess, audio, window_chunks=10, context_chunks=5): | |
| """Model output for any length of audio in bounded memory. | |
| Attention memory grows with the square of the input length, so long audio is run in windows | |
| of `window_chunks` chunks (6.4 s) with `context_chunks` (3.2 s) of preceding audio as | |
| context, cut on chunk boundaries. Same precision as one pass, about 0.3 GB peak. | |
| """ | |
| a = np.asarray(audio, dtype=np.float32) | |
| out, pos = [], 0 | |
| while pos < len(a): | |
| st = max(0, pos - context_chunks * CHUNK) | |
| x = a[st: pos + window_chunks * CHUNK + 80] # +80: the last frame spans 400 samples | |
| lp, n = sess.run(None, {"waveform": x[None], "lengths": np.array([len(x)], dtype=np.int64)}) | |
| skip = (pos - st) // 320 | |
| out.append(lp[0, skip: min(int(n[0]), skip + window_chunks * 32)]) | |
| pos += window_chunks * CHUNK | |
| return np.concatenate(out) | |
| def align(sess, cfg, audio, text, lang="en"): | |
| """audio: float32 mono 16 kHz. Returns [{word, start, end}] for every word of `text`.""" | |
| lp = log_probs(sess, audio) | |
| exp = prepare_text(text, lang) | |
| toks = expansion_to_tokens(exp, vocab_for_tokens(cfg["vocab_tokens"])) | |
| res = forced_align(lp, toks, AlignConfig(**cfg["align"]), linear_band_center(len(lp), len(toks))) | |
| words = spans_to_words(text, res.spans, cfg["frame_shift_sec"], expansion=exp, emissions=lp) | |
| return [{"word": exp.words[i], "start": round(s, 3), "end": round(e, 3)} for i, s, e, _ in words] | |
| if __name__ == "__main__": | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("audio") | |
| ap.add_argument("text") | |
| ap.add_argument("--lang", default="en", help="ISO 639-1 code of the text (see README for the 35 languages)") | |
| a = ap.parse_args() | |
| audio, sr = sf.read(a.audio, dtype="float32") | |
| if audio.ndim > 1: | |
| audio = audio.mean(1) | |
| if sr != 16000: | |
| raise SystemExit(f"{a.audio}: expected 16 kHz, got {sr} Hz (resample first, e.g. ffmpeg -ar 16000 -ac 1)") | |
| sess, cfg = load() | |
| with open(a.text, encoding="utf-8") as f: | |
| text = f.read() | |
| print(json.dumps(align(sess, cfg, audio, text, a.lang), ensure_ascii=False, indent=1)) | |