eg2-ko-reader / eg2_ko_reader.py
topabaem's picture
chunks(): cut long audio at pauses (source 2e0cb98449a3)
a582904 verified
Raw History Blame Contribute Delete
15.3 kB
"""EG2-KO Reader: Korean speech-to-text on the EmbeddingGemma 2 (EG2) audio encoder.
One file, no package to install. The folder that holds this file also holds the weights:
encoder/ unmodified copy of google/embeddinggemma-2 (Apache-2.0), frozen
reader/ the Korean syllable CTC reader (about 19.7M parameters) and its vocabulary
Command line:
python eg2_ko_reader.py audio.wav [more.wav ...]
Python:
from eg2_ko_reader import load
stt = load() # or load("/path/to/this/folder", device="cuda")
print(stt.transcribe("audio.wav"))
Audio of any length is cut into 30 s windows that overlap by 96 tokens (3.84 s). Each window
goes through the encoder (25 tokens per second, 512 numbers each) and the reader; the reader
outputs are joined and decoded once with greedy CTC. No language model is used.
The reader, this file and the Core AI bundle are under the EG2-KO Reader License 1.0 (see
LICENSE). The files under encoder/ are under the Apache License 2.0 (see encoder/LICENSE).
"""
from __future__ import annotations
import hashlib
import json
import math
import re
import sys
from dataclasses import dataclass
from pathlib import Path
import numpy as np
import torch
import torch.nn.functional as F
from torch import Tensor, nn
SAMPLE_RATE = 16_000
SAMPLES_PER_TOKEN = SAMPLE_RATE * 40 // 1000 # 640: one token per 40 ms
WINDOW_TOKENS = 750 # 30 s
WINDOW_SAMPLES = WINDOW_TOKENS * SAMPLES_PER_TOKEN
OVERLAP_TOKENS = 96
ENCODER_REVISION = "914f7f89142e33e77833254d9c9b90c3cef7303b" # google/embeddinggemma-2
BLANK, SPACE, UNK = "<blank>", "▁", "<unk>"
_SPACES = re.compile(r"\s+")
# ---------------------------------------------------------------------------- windows
@dataclass(frozen=True)
class Window:
start_tok: int # absolute token index where this window starts
start_sample: int
end_sample: int
emit_from: int # window-relative token range this window contributes
emit_to: int
def n_tokens(n_samples: int) -> int:
"""Tokens the encoder returns for this many samples: mel frames = floor((n - 1) / 160),
then 4 frames per token."""
frames = max(0, (n_samples - 1) // 160)
return math.ceil(frames / 4)
def plan_windows(n_samples: int) -> list[Window]:
"""30 s windows; each next one starts OVERLAP_TOKENS before the previous end; a cut
window drops its last token; a window emits only the tokens after what earlier windows
already emitted."""
if n_samples <= 0:
return []
step = WINDOW_TOKENS - OVERLAP_TOKENS
windows: list[Window] = []
start_tok = kept_until = 0
while True:
a = start_tok * SAMPLES_PER_TOKEN
b = min(a + WINDOW_SAMPLES, n_samples)
is_last = a + WINDOW_SAMPLES >= n_samples
usable = n_tokens(b - a) - (0 if is_last else 1)
windows.append(Window(start_tok, a, b, kept_until - start_tok, usable))
kept_until = start_tok + usable
if is_last:
return windows
start_tok += step
def split_on_pauses(
wav: np.ndarray, max_s: float = 30.0, search_s: float = 8.0
) -> list[tuple[int, int]]:
"""Cut long audio into chunks of at most `max_s` seconds, each cut at the quietest
100 ms in the last `search_s` seconds before the limit, so a cut falls in a pause rather
than inside a word. Returns (start, end) sample ranges that cover the audio end to end."""
n = len(wav)
limit = int(max_s * SAMPLE_RATE)
if n <= limit:
return [(0, n)] if n else []
hop = SAMPLE_RATE // 50 # 20 ms frames
frames = len(wav) // hop
energy = np.sqrt(
np.mean(wav[: frames * hop].reshape(frames, hop).astype(np.float64) ** 2, axis=1)
)
smooth = np.convolve(energy, np.ones(5) / 5, mode="same") # 100 ms
out, start = [], 0
while n - start > limit:
lo = (start + limit - int(search_s * SAMPLE_RATE)) // hop
hi = (start + limit) // hop
lo = max(lo, start // hop + 1)
cut = (lo + int(np.argmin(smooth[lo:hi]))) * hop + hop // 2 if hi > lo else start + limit
out.append((start, cut))
start = cut
out.append((start, n))
return out
# ----------------------------------------------------------------------------- reader
@dataclass(frozen=True)
class ReaderConfig:
vocab_size: int
d_in: int = 512
d_model: int = 512
layers: int = 6
heads: int = 8
ffn: int = 2048
dropout: float = 0.1
rope_base: float = 10_000.0
def rope_tables(seq_len: int, head_dim: int, base: float, device, dtype) -> tuple[Tensor, Tensor]:
inv = 1.0 / (
base ** (torch.arange(0, head_dim, 2, device=device, dtype=torch.float32) / head_dim)
)
t = torch.arange(seq_len, device=device, dtype=torch.float32)
freqs = torch.outer(t, inv)
return freqs.cos().to(dtype), freqs.sin().to(dtype)
def apply_rope(x: Tensor, cos: Tensor, sin: Tensor) -> Tensor:
x1, x2 = x[..., 0::2], x[..., 1::2]
rotated = torch.stack((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1)
return rotated.flatten(-2)
class Block(nn.Module):
def __init__(self, cfg: ReaderConfig) -> None:
super().__init__()
self.heads = cfg.heads
self.norm1 = nn.LayerNorm(cfg.d_model)
self.qkv = nn.Linear(cfg.d_model, 3 * cfg.d_model)
self.proj = nn.Linear(cfg.d_model, cfg.d_model)
self.norm2 = nn.LayerNorm(cfg.d_model)
self.ff = nn.Sequential(
nn.Linear(cfg.d_model, cfg.ffn),
nn.GELU(),
nn.Dropout(cfg.dropout),
nn.Linear(cfg.ffn, cfg.d_model),
)
self.drop = nn.Dropout(cfg.dropout)
def forward(self, x: Tensor, key_mask: Tensor, cos: Tensor, sin: Tensor) -> Tensor:
b, t, d = x.shape
h = self.norm1(x)
q, k, v = self.qkv(h).view(b, t, 3, self.heads, d // self.heads).permute(2, 0, 3, 1, 4)
q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin)
out = F.scaled_dot_product_attention(q, k, v, attn_mask=key_mask[:, None, None, :])
x = x + self.drop(self.proj(out.transpose(1, 2).reshape(b, t, d)))
return x + self.drop(self.ff(self.norm2(x)))
class Reader(nn.Module):
"""forward(x [B, T, 512], mask [B, T] bool) -> logits [B, T, V]; index 0 is the CTC blank.
LayerNorm -> Linear -> 6 pre-LN Transformer blocks (RoPE, bidirectional) -> LayerNorm ->
Linear. Inference only: dropout is never active here."""
def __init__(self, cfg: ReaderConfig) -> None:
super().__init__()
self.cfg = cfg
self.in_norm = nn.LayerNorm(cfg.d_in)
self.in_proj = nn.Linear(cfg.d_in, cfg.d_model)
self.blocks = nn.ModuleList(Block(cfg) for _ in range(cfg.layers))
self.out_norm = nn.LayerNorm(cfg.d_model)
self.head = nn.Linear(cfg.d_model, cfg.vocab_size)
def forward(self, x: Tensor, mask: Tensor) -> Tensor:
mask = mask.bool()
h = self.in_proj(self.in_norm(x))
cos, sin = rope_tables(
h.shape[1], self.cfg.d_model // self.cfg.heads, self.cfg.rope_base, h.device, h.dtype
)
for block in self.blocks:
h = block(h, mask, cos, sin)
logits = self.head(self.out_norm(h))
return logits.masked_fill(~mask[..., None], 0.0)
def greedy_ctc(logits: np.ndarray, blank_id: int = 0) -> list[int]:
"""[T, V] -> label ids: frame-wise argmax, merge repeats, drop blanks."""
ids = logits.argmax(-1).tolist()
merged = [i for j, i in enumerate(ids) if j == 0 or i != ids[j - 1]]
return [i for i in merged if i != blank_id]
# ------------------------------------------------------------------------- vocabulary
class Vocab:
"""Index 0 = CTC blank, 1 = word space, 2 = unknown, 3.. = Hangul syllables."""
blank_id, space_id, unk_id = 0, 1, 2
def __init__(self, tokens: list[str]) -> None:
if tuple(tokens[:3]) != (BLANK, SPACE, UNK):
raise ValueError(f"first tokens must be {(BLANK, SPACE, UNK)}, got {tokens[:3]}")
if len(set(tokens)) != len(tokens):
raise ValueError("duplicate tokens")
self.tokens = list(tokens)
def __len__(self) -> int:
return len(self.tokens)
def decode(self, ids: list[int]) -> str:
out = [" " if i == self.space_id else self.tokens[i] for i in ids if i != self.blank_id]
return _SPACES.sub(" ", "".join(out)).strip()
@classmethod
def load(cls, path: str | Path) -> Vocab:
return cls(Path(path).read_text(encoding="utf-8").splitlines())
def sha256(self) -> str:
return hashlib.sha256(("\n".join(self.tokens) + "\n").encode("utf-8")).hexdigest()
# ------------------------------------------------------------------------------ audio
def to_mono_16k(wav: np.ndarray, sample_rate: int) -> np.ndarray:
"""Any [n] or [n, channels] array -> float32 mono at 16 kHz in [-1, 1]."""
wav = np.asarray(wav)
if np.issubdtype(wav.dtype, np.unsignedinteger):
raise ValueError(f"unsigned integer audio ({wav.dtype}) is not supported")
if np.issubdtype(wav.dtype, np.integer):
wav = wav.astype(np.float32) / float(np.iinfo(wav.dtype).max + 1)
wav = wav.astype(np.float32)
if wav.ndim == 2:
wav = wav.mean(axis=1)
if wav.ndim != 1:
raise ValueError(f"audio must be [n] or [n, channels], got shape {wav.shape}")
if sample_rate != SAMPLE_RATE:
from scipy.signal import resample_poly
g = math.gcd(SAMPLE_RATE, int(sample_rate))
wav = resample_poly(wav, SAMPLE_RATE // g, int(sample_rate) // g).astype(np.float32)
return np.ascontiguousarray(wav)
def load_audio(path: str | Path) -> np.ndarray:
"""An audio file that libsndfile reads (wav, flac, ogg, mp3) -> float32 mono 16 kHz."""
import soundfile as sf
wav, sr = sf.read(str(path), dtype="float32", always_2d=True)
return to_mono_16k(wav, sr)
# ------------------------------------------------------------------------ transcriber
def load_reader(path: str | Path, vocab: Vocab) -> Reader:
"""Reader weights; the vocabulary must be the one the weights were trained with."""
from safetensors import safe_open
from safetensors.torch import load_file
with safe_open(str(path), framework="pt") as f:
meta = f.metadata() or {}
if meta.get("vocab_sha256") != vocab.sha256():
raise ValueError(
f"vocabulary does not match the weights: weights expect "
f"{meta.get('vocab_sha256')}, vocabulary is {vocab.sha256()}"
)
cfg = ReaderConfig(**json.loads(meta["reader_config"]))
if cfg.vocab_size != len(vocab):
raise ValueError(f"reader has {cfg.vocab_size} outputs, vocabulary has {len(vocab)}")
reader = Reader(cfg)
reader.load_state_dict(load_file(str(path)))
return reader.eval()
class EG2Encoder:
"""The frozen EG2 audio encoder: at most 30 s of audio -> [T, 512] float32."""
def __init__(self, folder: str | Path, device: str = "cpu") -> None:
from transformers import AutoFeatureExtractor, AutoModel
self.extractor = AutoFeatureExtractor.from_pretrained(str(folder), local_files_only=True)
model = AutoModel.from_pretrained(str(folder), local_files_only=True, dtype=torch.float32)
self.model = model.to(device).eval()
self.device = device
def __call__(self, wav: np.ndarray) -> np.ndarray:
feats = self.extractor([wav], sampling_rate=SAMPLE_RATE, return_tensors="pt")
with torch.no_grad():
out = self.model.get_audio_features(
input_features=feats["input_features"].to(self.device),
input_features_mask=feats["input_features_mask"].to(self.device),
)
mask = out.attention_mask
valid = int(mask.sum()) if mask is not None else out.pooler_output.shape[1]
return out.pooler_output[0, :valid].float().cpu().numpy()
class Transcriber:
"""`encoder` is any callable: float32 mono 16 kHz audio (at most 30 s) -> [T, 512]."""
def __init__(self, encoder, reader: Reader, vocab: Vocab, device: str = "cpu") -> None:
self.encoder, self.vocab, self.device = encoder, vocab, device
self.reader = reader.to(device).eval()
def logits(self, wav: np.ndarray) -> np.ndarray:
"""float32 mono 16 kHz audio of any length -> stitched reader logits [T, V]."""
pieces = []
for w in plan_windows(len(wav)):
tokens = self.encoder(wav[w.start_sample : w.end_sample])
x = torch.from_numpy(np.ascontiguousarray(tokens, dtype=np.float32))[None]
x = x.to(self.device)
mask = torch.ones(1, x.shape[1], dtype=torch.bool, device=self.device)
with torch.no_grad():
lg = self.reader(x, mask)[0].float().cpu().numpy()
pieces.append(lg[w.emit_from : w.emit_to])
if not pieces:
return np.zeros((0, len(self.vocab)), np.float32)
return np.concatenate(pieces)
@staticmethod
def _wav(audio, sample_rate: int | None) -> np.ndarray:
if isinstance(audio, str | Path):
return load_audio(audio)
if sample_rate is None:
raise ValueError("pass sample_rate with an audio array")
return to_mono_16k(audio, sample_rate)
def _text(self, wav: np.ndarray) -> str:
logits = self.logits(wav)
if logits.shape[0] == 0:
return ""
return self.vocab.decode(greedy_ctc(logits, self.vocab.blank_id))
def transcribe(self, audio, sample_rate: int | None = None) -> str:
"""`audio` is a file path, or an array with its `sample_rate`."""
return self._text(self._wav(audio, sample_rate))
def chunks(self, audio, sample_rate: int | None = None, max_s: float = 30.0):
"""Transcribe long audio piece by piece, cut at pauses (`split_on_pauses`). Yields
{"start": s, "end": s, "text": ...} per chunk, so the text can be shown as it comes."""
wav = self._wav(audio, sample_rate)
for a, b in split_on_pauses(wav, max_s):
yield {"start": a / SAMPLE_RATE, "end": b / SAMPLE_RATE, "text": self._text(wav[a:b])}
def load(folder: str | Path | None = None, device: str | None = None) -> Transcriber:
"""Load the encoder and the reader from `folder` (default: the folder of this file).
`device` defaults to "cuda" when available, else "cpu"; pass "mps" on Apple silicon."""
root = Path(folder) if folder is not None else Path(__file__).resolve().parent
if device is None:
device = "cuda" if torch.cuda.is_available() else "cpu"
vocab = Vocab.load(root / "reader" / "vocab_ko.txt")
reader = load_reader(root / "reader" / "model.safetensors", vocab)
return Transcriber(EG2Encoder(root / "encoder", device), reader, vocab, device)
def main(argv: list[str] | None = None) -> int:
args = sys.argv[1:] if argv is None else argv
if not args or args[0] in ("-h", "--help"):
print("usage: python eg2_ko_reader.py audio.wav [more.wav ...]", file=sys.stderr)
return 2
stt = load()
for path in args:
text = stt.transcribe(path)
print(text if len(args) == 1 else f"{path}\t{text}")
return 0
if __name__ == "__main__":
raise SystemExit(main())