File size: 3,387 Bytes
8f8b0fa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
#!/usr/bin/env python3
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Word timings for an audio file and its text with CavaAlign.

    pip install onnxruntime numpy soundfile regex num2words
    python align.py speech.wav text.txt --lang en > words.json
"""
import argparse
import json
import os

import numpy as np
import onnxruntime as ort
import soundfile as sf

import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from cavaalign.align import AlignConfig, expansion_to_tokens, forced_align, linear_band_center, spans_to_words
from cavaalign.romanize import prepare_text
from cavaalign.vocab import vocab_for_tokens

HERE = os.path.dirname(os.path.abspath(__file__))


def load(model=os.path.join(HERE, "cavaalign.onnx")):
    so = ort.SessionOptions()
    so.enable_cpu_mem_arena = False          # keeps memory flat on long recordings
    so.enable_mem_pattern = False
    sess = ort.InferenceSession(model, so, providers=["CPUExecutionProvider"])
    with open(os.path.join(HERE, "align_config.json"), encoding="utf-8") as f:
        cfg = json.load(f)
    return sess, cfg


CHUNK = 32 * 320          # the model's attention chunk: 32 frames of 20 ms = 10,240 samples


def log_probs(sess, audio, window_chunks=10, context_chunks=5):
    """Model output for any length of audio in bounded memory.

    Attention memory grows with the square of the input length, so long audio is run in windows
    of `window_chunks` chunks (6.4 s) with `context_chunks` (3.2 s) of preceding audio as
    context, cut on chunk boundaries. Same precision as one pass, about 0.3 GB peak.
    """
    a = np.asarray(audio, dtype=np.float32)
    out, pos = [], 0
    while pos < len(a):
        st = max(0, pos - context_chunks * CHUNK)
        x = a[st: pos + window_chunks * CHUNK + 80]          # +80: the last frame spans 400 samples
        lp, n = sess.run(None, {"waveform": x[None], "lengths": np.array([len(x)], dtype=np.int64)})
        skip = (pos - st) // 320
        out.append(lp[0, skip: min(int(n[0]), skip + window_chunks * 32)])
        pos += window_chunks * CHUNK
    return np.concatenate(out)


def align(sess, cfg, audio, text, lang="en"):
    """audio: float32 mono 16 kHz. Returns [{word, start, end}] for every word of `text`."""
    lp = log_probs(sess, audio)
    exp = prepare_text(text, lang)
    toks = expansion_to_tokens(exp, vocab_for_tokens(cfg["vocab_tokens"]))
    res = forced_align(lp, toks, AlignConfig(**cfg["align"]), linear_band_center(len(lp), len(toks)))
    words = spans_to_words(text, res.spans, cfg["frame_shift_sec"], expansion=exp, emissions=lp)
    return [{"word": exp.words[i], "start": round(s, 3), "end": round(e, 3)} for i, s, e, _ in words]


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("audio")
    ap.add_argument("text")
    ap.add_argument("--lang", default="en", help="ISO 639-1 code of the text (see README for the 35 languages)")
    a = ap.parse_args()
    audio, sr = sf.read(a.audio, dtype="float32")
    if audio.ndim > 1:
        audio = audio.mean(1)
    if sr != 16000:
        raise SystemExit(f"{a.audio}: expected 16 kHz, got {sr} Hz (resample first, e.g. ffmpeg -ar 16000 -ac 1)")
    sess, cfg = load()
    with open(a.text, encoding="utf-8") as f:
        text = f.read()
    print(json.dumps(align(sess, cfg, audio, text, a.lang), ensure_ascii=False, indent=1))