File size: 15,297 Bytes
a3adeb1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a582904
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a3adeb1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a582904
 
a3adeb1
a582904
 
 
 
 
 
a3adeb1
 
 
 
 
a582904
 
 
 
 
 
 
 
 
 
 
a3adeb1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
"""EG2-KO Reader: Korean speech-to-text on the EmbeddingGemma 2 (EG2) audio encoder.

One file, no package to install. The folder that holds this file also holds the weights:

    encoder/   unmodified copy of google/embeddinggemma-2 (Apache-2.0), frozen
    reader/    the Korean syllable CTC reader (about 19.7M parameters) and its vocabulary

Command line:

    python eg2_ko_reader.py audio.wav [more.wav ...]

Python:

    from eg2_ko_reader import load
    stt = load()                      # or load("/path/to/this/folder", device="cuda")
    print(stt.transcribe("audio.wav"))

Audio of any length is cut into 30 s windows that overlap by 96 tokens (3.84 s). Each window
goes through the encoder (25 tokens per second, 512 numbers each) and the reader; the reader
outputs are joined and decoded once with greedy CTC. No language model is used.

The reader, this file and the Core AI bundle are under the EG2-KO Reader License 1.0 (see
LICENSE). The files under encoder/ are under the Apache License 2.0 (see encoder/LICENSE).
"""

from __future__ import annotations

import hashlib
import json
import math
import re
import sys
from dataclasses import dataclass
from pathlib import Path

import numpy as np
import torch
import torch.nn.functional as F
from torch import Tensor, nn

SAMPLE_RATE = 16_000
SAMPLES_PER_TOKEN = SAMPLE_RATE * 40 // 1000  # 640: one token per 40 ms
WINDOW_TOKENS = 750  # 30 s
WINDOW_SAMPLES = WINDOW_TOKENS * SAMPLES_PER_TOKEN
OVERLAP_TOKENS = 96
ENCODER_REVISION = "914f7f89142e33e77833254d9c9b90c3cef7303b"  # google/embeddinggemma-2

BLANK, SPACE, UNK = "<blank>", "▁", "<unk>"
_SPACES = re.compile(r"\s+")


# ---------------------------------------------------------------------------- windows


@dataclass(frozen=True)
class Window:
    start_tok: int  # absolute token index where this window starts
    start_sample: int
    end_sample: int
    emit_from: int  # window-relative token range this window contributes
    emit_to: int


def n_tokens(n_samples: int) -> int:
    """Tokens the encoder returns for this many samples: mel frames = floor((n - 1) / 160),
    then 4 frames per token."""
    frames = max(0, (n_samples - 1) // 160)
    return math.ceil(frames / 4)


def plan_windows(n_samples: int) -> list[Window]:
    """30 s windows; each next one starts OVERLAP_TOKENS before the previous end; a cut
    window drops its last token; a window emits only the tokens after what earlier windows
    already emitted."""
    if n_samples <= 0:
        return []
    step = WINDOW_TOKENS - OVERLAP_TOKENS
    windows: list[Window] = []
    start_tok = kept_until = 0
    while True:
        a = start_tok * SAMPLES_PER_TOKEN
        b = min(a + WINDOW_SAMPLES, n_samples)
        is_last = a + WINDOW_SAMPLES >= n_samples
        usable = n_tokens(b - a) - (0 if is_last else 1)
        windows.append(Window(start_tok, a, b, kept_until - start_tok, usable))
        kept_until = start_tok + usable
        if is_last:
            return windows
        start_tok += step


def split_on_pauses(
    wav: np.ndarray, max_s: float = 30.0, search_s: float = 8.0
) -> list[tuple[int, int]]:
    """Cut long audio into chunks of at most `max_s` seconds, each cut at the quietest
    100 ms in the last `search_s` seconds before the limit, so a cut falls in a pause rather
    than inside a word. Returns (start, end) sample ranges that cover the audio end to end."""
    n = len(wav)
    limit = int(max_s * SAMPLE_RATE)
    if n <= limit:
        return [(0, n)] if n else []
    hop = SAMPLE_RATE // 50  # 20 ms frames
    frames = len(wav) // hop
    energy = np.sqrt(
        np.mean(wav[: frames * hop].reshape(frames, hop).astype(np.float64) ** 2, axis=1)
    )
    smooth = np.convolve(energy, np.ones(5) / 5, mode="same")  # 100 ms
    out, start = [], 0
    while n - start > limit:
        lo = (start + limit - int(search_s * SAMPLE_RATE)) // hop
        hi = (start + limit) // hop
        lo = max(lo, start // hop + 1)
        cut = (lo + int(np.argmin(smooth[lo:hi]))) * hop + hop // 2 if hi > lo else start + limit
        out.append((start, cut))
        start = cut
    out.append((start, n))
    return out


# ----------------------------------------------------------------------------- reader


@dataclass(frozen=True)
class ReaderConfig:
    vocab_size: int
    d_in: int = 512
    d_model: int = 512
    layers: int = 6
    heads: int = 8
    ffn: int = 2048
    dropout: float = 0.1
    rope_base: float = 10_000.0


def rope_tables(seq_len: int, head_dim: int, base: float, device, dtype) -> tuple[Tensor, Tensor]:
    inv = 1.0 / (
        base ** (torch.arange(0, head_dim, 2, device=device, dtype=torch.float32) / head_dim)
    )
    t = torch.arange(seq_len, device=device, dtype=torch.float32)
    freqs = torch.outer(t, inv)
    return freqs.cos().to(dtype), freqs.sin().to(dtype)


def apply_rope(x: Tensor, cos: Tensor, sin: Tensor) -> Tensor:
    x1, x2 = x[..., 0::2], x[..., 1::2]
    rotated = torch.stack((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1)
    return rotated.flatten(-2)


class Block(nn.Module):
    def __init__(self, cfg: ReaderConfig) -> None:
        super().__init__()
        self.heads = cfg.heads
        self.norm1 = nn.LayerNorm(cfg.d_model)
        self.qkv = nn.Linear(cfg.d_model, 3 * cfg.d_model)
        self.proj = nn.Linear(cfg.d_model, cfg.d_model)
        self.norm2 = nn.LayerNorm(cfg.d_model)
        self.ff = nn.Sequential(
            nn.Linear(cfg.d_model, cfg.ffn),
            nn.GELU(),
            nn.Dropout(cfg.dropout),
            nn.Linear(cfg.ffn, cfg.d_model),
        )
        self.drop = nn.Dropout(cfg.dropout)

    def forward(self, x: Tensor, key_mask: Tensor, cos: Tensor, sin: Tensor) -> Tensor:
        b, t, d = x.shape
        h = self.norm1(x)
        q, k, v = self.qkv(h).view(b, t, 3, self.heads, d // self.heads).permute(2, 0, 3, 1, 4)
        q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin)
        out = F.scaled_dot_product_attention(q, k, v, attn_mask=key_mask[:, None, None, :])
        x = x + self.drop(self.proj(out.transpose(1, 2).reshape(b, t, d)))
        return x + self.drop(self.ff(self.norm2(x)))


class Reader(nn.Module):
    """forward(x [B, T, 512], mask [B, T] bool) -> logits [B, T, V]; index 0 is the CTC blank.

    LayerNorm -> Linear -> 6 pre-LN Transformer blocks (RoPE, bidirectional) -> LayerNorm ->
    Linear. Inference only: dropout is never active here."""

    def __init__(self, cfg: ReaderConfig) -> None:
        super().__init__()
        self.cfg = cfg
        self.in_norm = nn.LayerNorm(cfg.d_in)
        self.in_proj = nn.Linear(cfg.d_in, cfg.d_model)
        self.blocks = nn.ModuleList(Block(cfg) for _ in range(cfg.layers))
        self.out_norm = nn.LayerNorm(cfg.d_model)
        self.head = nn.Linear(cfg.d_model, cfg.vocab_size)

    def forward(self, x: Tensor, mask: Tensor) -> Tensor:
        mask = mask.bool()
        h = self.in_proj(self.in_norm(x))
        cos, sin = rope_tables(
            h.shape[1], self.cfg.d_model // self.cfg.heads, self.cfg.rope_base, h.device, h.dtype
        )
        for block in self.blocks:
            h = block(h, mask, cos, sin)
        logits = self.head(self.out_norm(h))
        return logits.masked_fill(~mask[..., None], 0.0)


def greedy_ctc(logits: np.ndarray, blank_id: int = 0) -> list[int]:
    """[T, V] -> label ids: frame-wise argmax, merge repeats, drop blanks."""
    ids = logits.argmax(-1).tolist()
    merged = [i for j, i in enumerate(ids) if j == 0 or i != ids[j - 1]]
    return [i for i in merged if i != blank_id]


# ------------------------------------------------------------------------- vocabulary


class Vocab:
    """Index 0 = CTC blank, 1 = word space, 2 = unknown, 3.. = Hangul syllables."""

    blank_id, space_id, unk_id = 0, 1, 2

    def __init__(self, tokens: list[str]) -> None:
        if tuple(tokens[:3]) != (BLANK, SPACE, UNK):
            raise ValueError(f"first tokens must be {(BLANK, SPACE, UNK)}, got {tokens[:3]}")
        if len(set(tokens)) != len(tokens):
            raise ValueError("duplicate tokens")
        self.tokens = list(tokens)

    def __len__(self) -> int:
        return len(self.tokens)

    def decode(self, ids: list[int]) -> str:
        out = [" " if i == self.space_id else self.tokens[i] for i in ids if i != self.blank_id]
        return _SPACES.sub(" ", "".join(out)).strip()

    @classmethod
    def load(cls, path: str | Path) -> Vocab:
        return cls(Path(path).read_text(encoding="utf-8").splitlines())

    def sha256(self) -> str:
        return hashlib.sha256(("\n".join(self.tokens) + "\n").encode("utf-8")).hexdigest()


# ------------------------------------------------------------------------------ audio


def to_mono_16k(wav: np.ndarray, sample_rate: int) -> np.ndarray:
    """Any [n] or [n, channels] array -> float32 mono at 16 kHz in [-1, 1]."""
    wav = np.asarray(wav)
    if np.issubdtype(wav.dtype, np.unsignedinteger):
        raise ValueError(f"unsigned integer audio ({wav.dtype}) is not supported")
    if np.issubdtype(wav.dtype, np.integer):
        wav = wav.astype(np.float32) / float(np.iinfo(wav.dtype).max + 1)
    wav = wav.astype(np.float32)
    if wav.ndim == 2:
        wav = wav.mean(axis=1)
    if wav.ndim != 1:
        raise ValueError(f"audio must be [n] or [n, channels], got shape {wav.shape}")
    if sample_rate != SAMPLE_RATE:
        from scipy.signal import resample_poly

        g = math.gcd(SAMPLE_RATE, int(sample_rate))
        wav = resample_poly(wav, SAMPLE_RATE // g, int(sample_rate) // g).astype(np.float32)
    return np.ascontiguousarray(wav)


def load_audio(path: str | Path) -> np.ndarray:
    """An audio file that libsndfile reads (wav, flac, ogg, mp3) -> float32 mono 16 kHz."""
    import soundfile as sf

    wav, sr = sf.read(str(path), dtype="float32", always_2d=True)
    return to_mono_16k(wav, sr)


# ------------------------------------------------------------------------ transcriber


def load_reader(path: str | Path, vocab: Vocab) -> Reader:
    """Reader weights; the vocabulary must be the one the weights were trained with."""
    from safetensors import safe_open
    from safetensors.torch import load_file

    with safe_open(str(path), framework="pt") as f:
        meta = f.metadata() or {}
    if meta.get("vocab_sha256") != vocab.sha256():
        raise ValueError(
            f"vocabulary does not match the weights: weights expect "
            f"{meta.get('vocab_sha256')}, vocabulary is {vocab.sha256()}"
        )
    cfg = ReaderConfig(**json.loads(meta["reader_config"]))
    if cfg.vocab_size != len(vocab):
        raise ValueError(f"reader has {cfg.vocab_size} outputs, vocabulary has {len(vocab)}")
    reader = Reader(cfg)
    reader.load_state_dict(load_file(str(path)))
    return reader.eval()


class EG2Encoder:
    """The frozen EG2 audio encoder: at most 30 s of audio -> [T, 512] float32."""

    def __init__(self, folder: str | Path, device: str = "cpu") -> None:
        from transformers import AutoFeatureExtractor, AutoModel

        self.extractor = AutoFeatureExtractor.from_pretrained(str(folder), local_files_only=True)
        model = AutoModel.from_pretrained(str(folder), local_files_only=True, dtype=torch.float32)
        self.model = model.to(device).eval()
        self.device = device

    def __call__(self, wav: np.ndarray) -> np.ndarray:
        feats = self.extractor([wav], sampling_rate=SAMPLE_RATE, return_tensors="pt")
        with torch.no_grad():
            out = self.model.get_audio_features(
                input_features=feats["input_features"].to(self.device),
                input_features_mask=feats["input_features_mask"].to(self.device),
            )
        mask = out.attention_mask
        valid = int(mask.sum()) if mask is not None else out.pooler_output.shape[1]
        return out.pooler_output[0, :valid].float().cpu().numpy()


class Transcriber:
    """`encoder` is any callable: float32 mono 16 kHz audio (at most 30 s) -> [T, 512]."""

    def __init__(self, encoder, reader: Reader, vocab: Vocab, device: str = "cpu") -> None:
        self.encoder, self.vocab, self.device = encoder, vocab, device
        self.reader = reader.to(device).eval()

    def logits(self, wav: np.ndarray) -> np.ndarray:
        """float32 mono 16 kHz audio of any length -> stitched reader logits [T, V]."""
        pieces = []
        for w in plan_windows(len(wav)):
            tokens = self.encoder(wav[w.start_sample : w.end_sample])
            x = torch.from_numpy(np.ascontiguousarray(tokens, dtype=np.float32))[None]
            x = x.to(self.device)
            mask = torch.ones(1, x.shape[1], dtype=torch.bool, device=self.device)
            with torch.no_grad():
                lg = self.reader(x, mask)[0].float().cpu().numpy()
            pieces.append(lg[w.emit_from : w.emit_to])
        if not pieces:
            return np.zeros((0, len(self.vocab)), np.float32)
        return np.concatenate(pieces)

    @staticmethod
    def _wav(audio, sample_rate: int | None) -> np.ndarray:
        if isinstance(audio, str | Path):
            return load_audio(audio)
        if sample_rate is None:
            raise ValueError("pass sample_rate with an audio array")
        return to_mono_16k(audio, sample_rate)

    def _text(self, wav: np.ndarray) -> str:
        logits = self.logits(wav)
        if logits.shape[0] == 0:
            return ""
        return self.vocab.decode(greedy_ctc(logits, self.vocab.blank_id))

    def transcribe(self, audio, sample_rate: int | None = None) -> str:
        """`audio` is a file path, or an array with its `sample_rate`."""
        return self._text(self._wav(audio, sample_rate))

    def chunks(self, audio, sample_rate: int | None = None, max_s: float = 30.0):
        """Transcribe long audio piece by piece, cut at pauses (`split_on_pauses`). Yields
        {"start": s, "end": s, "text": ...} per chunk, so the text can be shown as it comes."""
        wav = self._wav(audio, sample_rate)
        for a, b in split_on_pauses(wav, max_s):
            yield {"start": a / SAMPLE_RATE, "end": b / SAMPLE_RATE, "text": self._text(wav[a:b])}


def load(folder: str | Path | None = None, device: str | None = None) -> Transcriber:
    """Load the encoder and the reader from `folder` (default: the folder of this file).
    `device` defaults to "cuda" when available, else "cpu"; pass "mps" on Apple silicon."""
    root = Path(folder) if folder is not None else Path(__file__).resolve().parent
    if device is None:
        device = "cuda" if torch.cuda.is_available() else "cpu"
    vocab = Vocab.load(root / "reader" / "vocab_ko.txt")
    reader = load_reader(root / "reader" / "model.safetensors", vocab)
    return Transcriber(EG2Encoder(root / "encoder", device), reader, vocab, device)


def main(argv: list[str] | None = None) -> int:
    args = sys.argv[1:] if argv is None else argv
    if not args or args[0] in ("-h", "--help"):
        print("usage: python eg2_ko_reader.py audio.wav [more.wav ...]", file=sys.stderr)
        return 2
    stt = load()
    for path in args:
        text = stt.transcribe(path)
        print(text if len(args) == 1 else f"{path}\t{text}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())