"""EG2-KO Reader: Korean speech-to-text on the EmbeddingGemma 2 (EG2) audio encoder. One file, no package to install. The folder that holds this file also holds the weights: encoder/ unmodified copy of google/embeddinggemma-2 (Apache-2.0), frozen reader/ the Korean syllable CTC reader (about 19.7M parameters) and its vocabulary Command line: python eg2_ko_reader.py audio.wav [more.wav ...] Python: from eg2_ko_reader import load stt = load() # or load("/path/to/this/folder", device="cuda") print(stt.transcribe("audio.wav")) Audio of any length is cut into 30 s windows that overlap by 96 tokens (3.84 s). Each window goes through the encoder (25 tokens per second, 512 numbers each) and the reader; the reader outputs are joined and decoded once with greedy CTC. No language model is used. The reader, this file and the Core AI bundle are under the EG2-KO Reader License 1.0 (see LICENSE). The files under encoder/ are under the Apache License 2.0 (see encoder/LICENSE). """ from __future__ import annotations import hashlib import json import math import re import sys from dataclasses import dataclass from pathlib import Path import numpy as np import torch import torch.nn.functional as F from torch import Tensor, nn SAMPLE_RATE = 16_000 SAMPLES_PER_TOKEN = SAMPLE_RATE * 40 // 1000 # 640: one token per 40 ms WINDOW_TOKENS = 750 # 30 s WINDOW_SAMPLES = WINDOW_TOKENS * SAMPLES_PER_TOKEN OVERLAP_TOKENS = 96 ENCODER_REVISION = "914f7f89142e33e77833254d9c9b90c3cef7303b" # google/embeddinggemma-2 BLANK, SPACE, UNK = "", "▁", "" _SPACES = re.compile(r"\s+") # ---------------------------------------------------------------------------- windows @dataclass(frozen=True) class Window: start_tok: int # absolute token index where this window starts start_sample: int end_sample: int emit_from: int # window-relative token range this window contributes emit_to: int def n_tokens(n_samples: int) -> int: """Tokens the encoder returns for this many samples: mel frames = floor((n - 1) / 160), then 4 frames per token.""" frames = max(0, (n_samples - 1) // 160) return math.ceil(frames / 4) def plan_windows(n_samples: int) -> list[Window]: """30 s windows; each next one starts OVERLAP_TOKENS before the previous end; a cut window drops its last token; a window emits only the tokens after what earlier windows already emitted.""" if n_samples <= 0: return [] step = WINDOW_TOKENS - OVERLAP_TOKENS windows: list[Window] = [] start_tok = kept_until = 0 while True: a = start_tok * SAMPLES_PER_TOKEN b = min(a + WINDOW_SAMPLES, n_samples) is_last = a + WINDOW_SAMPLES >= n_samples usable = n_tokens(b - a) - (0 if is_last else 1) windows.append(Window(start_tok, a, b, kept_until - start_tok, usable)) kept_until = start_tok + usable if is_last: return windows start_tok += step def split_on_pauses( wav: np.ndarray, max_s: float = 30.0, search_s: float = 8.0 ) -> list[tuple[int, int]]: """Cut long audio into chunks of at most `max_s` seconds, each cut at the quietest 100 ms in the last `search_s` seconds before the limit, so a cut falls in a pause rather than inside a word. Returns (start, end) sample ranges that cover the audio end to end.""" n = len(wav) limit = int(max_s * SAMPLE_RATE) if n <= limit: return [(0, n)] if n else [] hop = SAMPLE_RATE // 50 # 20 ms frames frames = len(wav) // hop energy = np.sqrt( np.mean(wav[: frames * hop].reshape(frames, hop).astype(np.float64) ** 2, axis=1) ) smooth = np.convolve(energy, np.ones(5) / 5, mode="same") # 100 ms out, start = [], 0 while n - start > limit: lo = (start + limit - int(search_s * SAMPLE_RATE)) // hop hi = (start + limit) // hop lo = max(lo, start // hop + 1) cut = (lo + int(np.argmin(smooth[lo:hi]))) * hop + hop // 2 if hi > lo else start + limit out.append((start, cut)) start = cut out.append((start, n)) return out # ----------------------------------------------------------------------------- reader @dataclass(frozen=True) class ReaderConfig: vocab_size: int d_in: int = 512 d_model: int = 512 layers: int = 6 heads: int = 8 ffn: int = 2048 dropout: float = 0.1 rope_base: float = 10_000.0 def rope_tables(seq_len: int, head_dim: int, base: float, device, dtype) -> tuple[Tensor, Tensor]: inv = 1.0 / ( base ** (torch.arange(0, head_dim, 2, device=device, dtype=torch.float32) / head_dim) ) t = torch.arange(seq_len, device=device, dtype=torch.float32) freqs = torch.outer(t, inv) return freqs.cos().to(dtype), freqs.sin().to(dtype) def apply_rope(x: Tensor, cos: Tensor, sin: Tensor) -> Tensor: x1, x2 = x[..., 0::2], x[..., 1::2] rotated = torch.stack((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1) return rotated.flatten(-2) class Block(nn.Module): def __init__(self, cfg: ReaderConfig) -> None: super().__init__() self.heads = cfg.heads self.norm1 = nn.LayerNorm(cfg.d_model) self.qkv = nn.Linear(cfg.d_model, 3 * cfg.d_model) self.proj = nn.Linear(cfg.d_model, cfg.d_model) self.norm2 = nn.LayerNorm(cfg.d_model) self.ff = nn.Sequential( nn.Linear(cfg.d_model, cfg.ffn), nn.GELU(), nn.Dropout(cfg.dropout), nn.Linear(cfg.ffn, cfg.d_model), ) self.drop = nn.Dropout(cfg.dropout) def forward(self, x: Tensor, key_mask: Tensor, cos: Tensor, sin: Tensor) -> Tensor: b, t, d = x.shape h = self.norm1(x) q, k, v = self.qkv(h).view(b, t, 3, self.heads, d // self.heads).permute(2, 0, 3, 1, 4) q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin) out = F.scaled_dot_product_attention(q, k, v, attn_mask=key_mask[:, None, None, :]) x = x + self.drop(self.proj(out.transpose(1, 2).reshape(b, t, d))) return x + self.drop(self.ff(self.norm2(x))) class Reader(nn.Module): """forward(x [B, T, 512], mask [B, T] bool) -> logits [B, T, V]; index 0 is the CTC blank. LayerNorm -> Linear -> 6 pre-LN Transformer blocks (RoPE, bidirectional) -> LayerNorm -> Linear. Inference only: dropout is never active here.""" def __init__(self, cfg: ReaderConfig) -> None: super().__init__() self.cfg = cfg self.in_norm = nn.LayerNorm(cfg.d_in) self.in_proj = nn.Linear(cfg.d_in, cfg.d_model) self.blocks = nn.ModuleList(Block(cfg) for _ in range(cfg.layers)) self.out_norm = nn.LayerNorm(cfg.d_model) self.head = nn.Linear(cfg.d_model, cfg.vocab_size) def forward(self, x: Tensor, mask: Tensor) -> Tensor: mask = mask.bool() h = self.in_proj(self.in_norm(x)) cos, sin = rope_tables( h.shape[1], self.cfg.d_model // self.cfg.heads, self.cfg.rope_base, h.device, h.dtype ) for block in self.blocks: h = block(h, mask, cos, sin) logits = self.head(self.out_norm(h)) return logits.masked_fill(~mask[..., None], 0.0) def greedy_ctc(logits: np.ndarray, blank_id: int = 0) -> list[int]: """[T, V] -> label ids: frame-wise argmax, merge repeats, drop blanks.""" ids = logits.argmax(-1).tolist() merged = [i for j, i in enumerate(ids) if j == 0 or i != ids[j - 1]] return [i for i in merged if i != blank_id] # ------------------------------------------------------------------------- vocabulary class Vocab: """Index 0 = CTC blank, 1 = word space, 2 = unknown, 3.. = Hangul syllables.""" blank_id, space_id, unk_id = 0, 1, 2 def __init__(self, tokens: list[str]) -> None: if tuple(tokens[:3]) != (BLANK, SPACE, UNK): raise ValueError(f"first tokens must be {(BLANK, SPACE, UNK)}, got {tokens[:3]}") if len(set(tokens)) != len(tokens): raise ValueError("duplicate tokens") self.tokens = list(tokens) def __len__(self) -> int: return len(self.tokens) def decode(self, ids: list[int]) -> str: out = [" " if i == self.space_id else self.tokens[i] for i in ids if i != self.blank_id] return _SPACES.sub(" ", "".join(out)).strip() @classmethod def load(cls, path: str | Path) -> Vocab: return cls(Path(path).read_text(encoding="utf-8").splitlines()) def sha256(self) -> str: return hashlib.sha256(("\n".join(self.tokens) + "\n").encode("utf-8")).hexdigest() # ------------------------------------------------------------------------------ audio def to_mono_16k(wav: np.ndarray, sample_rate: int) -> np.ndarray: """Any [n] or [n, channels] array -> float32 mono at 16 kHz in [-1, 1].""" wav = np.asarray(wav) if np.issubdtype(wav.dtype, np.unsignedinteger): raise ValueError(f"unsigned integer audio ({wav.dtype}) is not supported") if np.issubdtype(wav.dtype, np.integer): wav = wav.astype(np.float32) / float(np.iinfo(wav.dtype).max + 1) wav = wav.astype(np.float32) if wav.ndim == 2: wav = wav.mean(axis=1) if wav.ndim != 1: raise ValueError(f"audio must be [n] or [n, channels], got shape {wav.shape}") if sample_rate != SAMPLE_RATE: from scipy.signal import resample_poly g = math.gcd(SAMPLE_RATE, int(sample_rate)) wav = resample_poly(wav, SAMPLE_RATE // g, int(sample_rate) // g).astype(np.float32) return np.ascontiguousarray(wav) def load_audio(path: str | Path) -> np.ndarray: """An audio file that libsndfile reads (wav, flac, ogg, mp3) -> float32 mono 16 kHz.""" import soundfile as sf wav, sr = sf.read(str(path), dtype="float32", always_2d=True) return to_mono_16k(wav, sr) # ------------------------------------------------------------------------ transcriber def load_reader(path: str | Path, vocab: Vocab) -> Reader: """Reader weights; the vocabulary must be the one the weights were trained with.""" from safetensors import safe_open from safetensors.torch import load_file with safe_open(str(path), framework="pt") as f: meta = f.metadata() or {} if meta.get("vocab_sha256") != vocab.sha256(): raise ValueError( f"vocabulary does not match the weights: weights expect " f"{meta.get('vocab_sha256')}, vocabulary is {vocab.sha256()}" ) cfg = ReaderConfig(**json.loads(meta["reader_config"])) if cfg.vocab_size != len(vocab): raise ValueError(f"reader has {cfg.vocab_size} outputs, vocabulary has {len(vocab)}") reader = Reader(cfg) reader.load_state_dict(load_file(str(path))) return reader.eval() class EG2Encoder: """The frozen EG2 audio encoder: at most 30 s of audio -> [T, 512] float32.""" def __init__(self, folder: str | Path, device: str = "cpu") -> None: from transformers import AutoFeatureExtractor, AutoModel self.extractor = AutoFeatureExtractor.from_pretrained(str(folder), local_files_only=True) model = AutoModel.from_pretrained(str(folder), local_files_only=True, dtype=torch.float32) self.model = model.to(device).eval() self.device = device def __call__(self, wav: np.ndarray) -> np.ndarray: feats = self.extractor([wav], sampling_rate=SAMPLE_RATE, return_tensors="pt") with torch.no_grad(): out = self.model.get_audio_features( input_features=feats["input_features"].to(self.device), input_features_mask=feats["input_features_mask"].to(self.device), ) mask = out.attention_mask valid = int(mask.sum()) if mask is not None else out.pooler_output.shape[1] return out.pooler_output[0, :valid].float().cpu().numpy() class Transcriber: """`encoder` is any callable: float32 mono 16 kHz audio (at most 30 s) -> [T, 512].""" def __init__(self, encoder, reader: Reader, vocab: Vocab, device: str = "cpu") -> None: self.encoder, self.vocab, self.device = encoder, vocab, device self.reader = reader.to(device).eval() def logits(self, wav: np.ndarray) -> np.ndarray: """float32 mono 16 kHz audio of any length -> stitched reader logits [T, V].""" pieces = [] for w in plan_windows(len(wav)): tokens = self.encoder(wav[w.start_sample : w.end_sample]) x = torch.from_numpy(np.ascontiguousarray(tokens, dtype=np.float32))[None] x = x.to(self.device) mask = torch.ones(1, x.shape[1], dtype=torch.bool, device=self.device) with torch.no_grad(): lg = self.reader(x, mask)[0].float().cpu().numpy() pieces.append(lg[w.emit_from : w.emit_to]) if not pieces: return np.zeros((0, len(self.vocab)), np.float32) return np.concatenate(pieces) @staticmethod def _wav(audio, sample_rate: int | None) -> np.ndarray: if isinstance(audio, str | Path): return load_audio(audio) if sample_rate is None: raise ValueError("pass sample_rate with an audio array") return to_mono_16k(audio, sample_rate) def _text(self, wav: np.ndarray) -> str: logits = self.logits(wav) if logits.shape[0] == 0: return "" return self.vocab.decode(greedy_ctc(logits, self.vocab.blank_id)) def transcribe(self, audio, sample_rate: int | None = None) -> str: """`audio` is a file path, or an array with its `sample_rate`.""" return self._text(self._wav(audio, sample_rate)) def chunks(self, audio, sample_rate: int | None = None, max_s: float = 30.0): """Transcribe long audio piece by piece, cut at pauses (`split_on_pauses`). Yields {"start": s, "end": s, "text": ...} per chunk, so the text can be shown as it comes.""" wav = self._wav(audio, sample_rate) for a, b in split_on_pauses(wav, max_s): yield {"start": a / SAMPLE_RATE, "end": b / SAMPLE_RATE, "text": self._text(wav[a:b])} def load(folder: str | Path | None = None, device: str | None = None) -> Transcriber: """Load the encoder and the reader from `folder` (default: the folder of this file). `device` defaults to "cuda" when available, else "cpu"; pass "mps" on Apple silicon.""" root = Path(folder) if folder is not None else Path(__file__).resolve().parent if device is None: device = "cuda" if torch.cuda.is_available() else "cpu" vocab = Vocab.load(root / "reader" / "vocab_ko.txt") reader = load_reader(root / "reader" / "model.safetensors", vocab) return Transcriber(EG2Encoder(root / "encoder", device), reader, vocab, device) def main(argv: list[str] | None = None) -> int: args = sys.argv[1:] if argv is None else argv if not args or args[0] in ("-h", "--help"): print("usage: python eg2_ko_reader.py audio.wav [more.wav ...]", file=sys.stderr) return 2 stt = load() for path in args: text = stt.transcribe(path) print(text if len(args) == 1 else f"{path}\t{text}") return 0 if __name__ == "__main__": raise SystemExit(main())