Download eg2_ko_reader.py from Haverbex/eg2-ko-reader: direct link, hf CLI and curl.
- Browser
- Download file 15.3 kB
-
https://huggingface.co/Haverbex/eg2-ko-reader/resolve/main/eg2_ko_reader.py
- Command line
-
hf download hf://Haverbex/eg2-ko-reader/eg2_ko_reader.py
-
curl -L -o eg2_ko_reader.py https://huggingface.co/Haverbex/eg2-ko-reader/resolve/main/eg2_ko_reader.py
15.3 kB
| """EG2-KO Reader: Korean speech-to-text on the EmbeddingGemma 2 (EG2) audio encoder. | |
| One file, no package to install. The folder that holds this file also holds the weights: | |
| encoder/ unmodified copy of google/embeddinggemma-2 (Apache-2.0), frozen | |
| reader/ the Korean syllable CTC reader (about 19.7M parameters) and its vocabulary | |
| Command line: | |
| python eg2_ko_reader.py audio.wav [more.wav ...] | |
| Python: | |
| from eg2_ko_reader import load | |
| stt = load() # or load("/path/to/this/folder", device="cuda") | |
| print(stt.transcribe("audio.wav")) | |
| Audio of any length is cut into 30 s windows that overlap by 96 tokens (3.84 s). Each window | |
| goes through the encoder (25 tokens per second, 512 numbers each) and the reader; the reader | |
| outputs are joined and decoded once with greedy CTC. No language model is used. | |
| The reader, this file and the Core AI bundle are under the EG2-KO Reader License 1.0 (see | |
| LICENSE). The files under encoder/ are under the Apache License 2.0 (see encoder/LICENSE). | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import math | |
| import re | |
| import sys | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from torch import Tensor, nn | |
| SAMPLE_RATE = 16_000 | |
| SAMPLES_PER_TOKEN = SAMPLE_RATE * 40 // 1000 # 640: one token per 40 ms | |
| WINDOW_TOKENS = 750 # 30 s | |
| WINDOW_SAMPLES = WINDOW_TOKENS * SAMPLES_PER_TOKEN | |
| OVERLAP_TOKENS = 96 | |
| ENCODER_REVISION = "914f7f89142e33e77833254d9c9b90c3cef7303b" # google/embeddinggemma-2 | |
| BLANK, SPACE, UNK = "<blank>", "▁", "<unk>" | |
| _SPACES = re.compile(r"\s+") | |
| # ---------------------------------------------------------------------------- windows | |
| class Window: | |
| start_tok: int # absolute token index where this window starts | |
| start_sample: int | |
| end_sample: int | |
| emit_from: int # window-relative token range this window contributes | |
| emit_to: int | |
| def n_tokens(n_samples: int) -> int: | |
| """Tokens the encoder returns for this many samples: mel frames = floor((n - 1) / 160), | |
| then 4 frames per token.""" | |
| frames = max(0, (n_samples - 1) // 160) | |
| return math.ceil(frames / 4) | |
| def plan_windows(n_samples: int) -> list[Window]: | |
| """30 s windows; each next one starts OVERLAP_TOKENS before the previous end; a cut | |
| window drops its last token; a window emits only the tokens after what earlier windows | |
| already emitted.""" | |
| if n_samples <= 0: | |
| return [] | |
| step = WINDOW_TOKENS - OVERLAP_TOKENS | |
| windows: list[Window] = [] | |
| start_tok = kept_until = 0 | |
| while True: | |
| a = start_tok * SAMPLES_PER_TOKEN | |
| b = min(a + WINDOW_SAMPLES, n_samples) | |
| is_last = a + WINDOW_SAMPLES >= n_samples | |
| usable = n_tokens(b - a) - (0 if is_last else 1) | |
| windows.append(Window(start_tok, a, b, kept_until - start_tok, usable)) | |
| kept_until = start_tok + usable | |
| if is_last: | |
| return windows | |
| start_tok += step | |
| def split_on_pauses( | |
| wav: np.ndarray, max_s: float = 30.0, search_s: float = 8.0 | |
| ) -> list[tuple[int, int]]: | |
| """Cut long audio into chunks of at most `max_s` seconds, each cut at the quietest | |
| 100 ms in the last `search_s` seconds before the limit, so a cut falls in a pause rather | |
| than inside a word. Returns (start, end) sample ranges that cover the audio end to end.""" | |
| n = len(wav) | |
| limit = int(max_s * SAMPLE_RATE) | |
| if n <= limit: | |
| return [(0, n)] if n else [] | |
| hop = SAMPLE_RATE // 50 # 20 ms frames | |
| frames = len(wav) // hop | |
| energy = np.sqrt( | |
| np.mean(wav[: frames * hop].reshape(frames, hop).astype(np.float64) ** 2, axis=1) | |
| ) | |
| smooth = np.convolve(energy, np.ones(5) / 5, mode="same") # 100 ms | |
| out, start = [], 0 | |
| while n - start > limit: | |
| lo = (start + limit - int(search_s * SAMPLE_RATE)) // hop | |
| hi = (start + limit) // hop | |
| lo = max(lo, start // hop + 1) | |
| cut = (lo + int(np.argmin(smooth[lo:hi]))) * hop + hop // 2 if hi > lo else start + limit | |
| out.append((start, cut)) | |
| start = cut | |
| out.append((start, n)) | |
| return out | |
| # ----------------------------------------------------------------------------- reader | |
| class ReaderConfig: | |
| vocab_size: int | |
| d_in: int = 512 | |
| d_model: int = 512 | |
| layers: int = 6 | |
| heads: int = 8 | |
| ffn: int = 2048 | |
| dropout: float = 0.1 | |
| rope_base: float = 10_000.0 | |
| def rope_tables(seq_len: int, head_dim: int, base: float, device, dtype) -> tuple[Tensor, Tensor]: | |
| inv = 1.0 / ( | |
| base ** (torch.arange(0, head_dim, 2, device=device, dtype=torch.float32) / head_dim) | |
| ) | |
| t = torch.arange(seq_len, device=device, dtype=torch.float32) | |
| freqs = torch.outer(t, inv) | |
| return freqs.cos().to(dtype), freqs.sin().to(dtype) | |
| def apply_rope(x: Tensor, cos: Tensor, sin: Tensor) -> Tensor: | |
| x1, x2 = x[..., 0::2], x[..., 1::2] | |
| rotated = torch.stack((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1) | |
| return rotated.flatten(-2) | |
| class Block(nn.Module): | |
| def __init__(self, cfg: ReaderConfig) -> None: | |
| super().__init__() | |
| self.heads = cfg.heads | |
| self.norm1 = nn.LayerNorm(cfg.d_model) | |
| self.qkv = nn.Linear(cfg.d_model, 3 * cfg.d_model) | |
| self.proj = nn.Linear(cfg.d_model, cfg.d_model) | |
| self.norm2 = nn.LayerNorm(cfg.d_model) | |
| self.ff = nn.Sequential( | |
| nn.Linear(cfg.d_model, cfg.ffn), | |
| nn.GELU(), | |
| nn.Dropout(cfg.dropout), | |
| nn.Linear(cfg.ffn, cfg.d_model), | |
| ) | |
| self.drop = nn.Dropout(cfg.dropout) | |
| def forward(self, x: Tensor, key_mask: Tensor, cos: Tensor, sin: Tensor) -> Tensor: | |
| b, t, d = x.shape | |
| h = self.norm1(x) | |
| q, k, v = self.qkv(h).view(b, t, 3, self.heads, d // self.heads).permute(2, 0, 3, 1, 4) | |
| q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin) | |
| out = F.scaled_dot_product_attention(q, k, v, attn_mask=key_mask[:, None, None, :]) | |
| x = x + self.drop(self.proj(out.transpose(1, 2).reshape(b, t, d))) | |
| return x + self.drop(self.ff(self.norm2(x))) | |
| class Reader(nn.Module): | |
| """forward(x [B, T, 512], mask [B, T] bool) -> logits [B, T, V]; index 0 is the CTC blank. | |
| LayerNorm -> Linear -> 6 pre-LN Transformer blocks (RoPE, bidirectional) -> LayerNorm -> | |
| Linear. Inference only: dropout is never active here.""" | |
| def __init__(self, cfg: ReaderConfig) -> None: | |
| super().__init__() | |
| self.cfg = cfg | |
| self.in_norm = nn.LayerNorm(cfg.d_in) | |
| self.in_proj = nn.Linear(cfg.d_in, cfg.d_model) | |
| self.blocks = nn.ModuleList(Block(cfg) for _ in range(cfg.layers)) | |
| self.out_norm = nn.LayerNorm(cfg.d_model) | |
| self.head = nn.Linear(cfg.d_model, cfg.vocab_size) | |
| def forward(self, x: Tensor, mask: Tensor) -> Tensor: | |
| mask = mask.bool() | |
| h = self.in_proj(self.in_norm(x)) | |
| cos, sin = rope_tables( | |
| h.shape[1], self.cfg.d_model // self.cfg.heads, self.cfg.rope_base, h.device, h.dtype | |
| ) | |
| for block in self.blocks: | |
| h = block(h, mask, cos, sin) | |
| logits = self.head(self.out_norm(h)) | |
| return logits.masked_fill(~mask[..., None], 0.0) | |
| def greedy_ctc(logits: np.ndarray, blank_id: int = 0) -> list[int]: | |
| """[T, V] -> label ids: frame-wise argmax, merge repeats, drop blanks.""" | |
| ids = logits.argmax(-1).tolist() | |
| merged = [i for j, i in enumerate(ids) if j == 0 or i != ids[j - 1]] | |
| return [i for i in merged if i != blank_id] | |
| # ------------------------------------------------------------------------- vocabulary | |
| class Vocab: | |
| """Index 0 = CTC blank, 1 = word space, 2 = unknown, 3.. = Hangul syllables.""" | |
| blank_id, space_id, unk_id = 0, 1, 2 | |
| def __init__(self, tokens: list[str]) -> None: | |
| if tuple(tokens[:3]) != (BLANK, SPACE, UNK): | |
| raise ValueError(f"first tokens must be {(BLANK, SPACE, UNK)}, got {tokens[:3]}") | |
| if len(set(tokens)) != len(tokens): | |
| raise ValueError("duplicate tokens") | |
| self.tokens = list(tokens) | |
| def __len__(self) -> int: | |
| return len(self.tokens) | |
| def decode(self, ids: list[int]) -> str: | |
| out = [" " if i == self.space_id else self.tokens[i] for i in ids if i != self.blank_id] | |
| return _SPACES.sub(" ", "".join(out)).strip() | |
| def load(cls, path: str | Path) -> Vocab: | |
| return cls(Path(path).read_text(encoding="utf-8").splitlines()) | |
| def sha256(self) -> str: | |
| return hashlib.sha256(("\n".join(self.tokens) + "\n").encode("utf-8")).hexdigest() | |
| # ------------------------------------------------------------------------------ audio | |
| def to_mono_16k(wav: np.ndarray, sample_rate: int) -> np.ndarray: | |
| """Any [n] or [n, channels] array -> float32 mono at 16 kHz in [-1, 1].""" | |
| wav = np.asarray(wav) | |
| if np.issubdtype(wav.dtype, np.unsignedinteger): | |
| raise ValueError(f"unsigned integer audio ({wav.dtype}) is not supported") | |
| if np.issubdtype(wav.dtype, np.integer): | |
| wav = wav.astype(np.float32) / float(np.iinfo(wav.dtype).max + 1) | |
| wav = wav.astype(np.float32) | |
| if wav.ndim == 2: | |
| wav = wav.mean(axis=1) | |
| if wav.ndim != 1: | |
| raise ValueError(f"audio must be [n] or [n, channels], got shape {wav.shape}") | |
| if sample_rate != SAMPLE_RATE: | |
| from scipy.signal import resample_poly | |
| g = math.gcd(SAMPLE_RATE, int(sample_rate)) | |
| wav = resample_poly(wav, SAMPLE_RATE // g, int(sample_rate) // g).astype(np.float32) | |
| return np.ascontiguousarray(wav) | |
| def load_audio(path: str | Path) -> np.ndarray: | |
| """An audio file that libsndfile reads (wav, flac, ogg, mp3) -> float32 mono 16 kHz.""" | |
| import soundfile as sf | |
| wav, sr = sf.read(str(path), dtype="float32", always_2d=True) | |
| return to_mono_16k(wav, sr) | |
| # ------------------------------------------------------------------------ transcriber | |
| def load_reader(path: str | Path, vocab: Vocab) -> Reader: | |
| """Reader weights; the vocabulary must be the one the weights were trained with.""" | |
| from safetensors import safe_open | |
| from safetensors.torch import load_file | |
| with safe_open(str(path), framework="pt") as f: | |
| meta = f.metadata() or {} | |
| if meta.get("vocab_sha256") != vocab.sha256(): | |
| raise ValueError( | |
| f"vocabulary does not match the weights: weights expect " | |
| f"{meta.get('vocab_sha256')}, vocabulary is {vocab.sha256()}" | |
| ) | |
| cfg = ReaderConfig(**json.loads(meta["reader_config"])) | |
| if cfg.vocab_size != len(vocab): | |
| raise ValueError(f"reader has {cfg.vocab_size} outputs, vocabulary has {len(vocab)}") | |
| reader = Reader(cfg) | |
| reader.load_state_dict(load_file(str(path))) | |
| return reader.eval() | |
| class EG2Encoder: | |
| """The frozen EG2 audio encoder: at most 30 s of audio -> [T, 512] float32.""" | |
| def __init__(self, folder: str | Path, device: str = "cpu") -> None: | |
| from transformers import AutoFeatureExtractor, AutoModel | |
| self.extractor = AutoFeatureExtractor.from_pretrained(str(folder), local_files_only=True) | |
| model = AutoModel.from_pretrained(str(folder), local_files_only=True, dtype=torch.float32) | |
| self.model = model.to(device).eval() | |
| self.device = device | |
| def __call__(self, wav: np.ndarray) -> np.ndarray: | |
| feats = self.extractor([wav], sampling_rate=SAMPLE_RATE, return_tensors="pt") | |
| with torch.no_grad(): | |
| out = self.model.get_audio_features( | |
| input_features=feats["input_features"].to(self.device), | |
| input_features_mask=feats["input_features_mask"].to(self.device), | |
| ) | |
| mask = out.attention_mask | |
| valid = int(mask.sum()) if mask is not None else out.pooler_output.shape[1] | |
| return out.pooler_output[0, :valid].float().cpu().numpy() | |
| class Transcriber: | |
| """`encoder` is any callable: float32 mono 16 kHz audio (at most 30 s) -> [T, 512].""" | |
| def __init__(self, encoder, reader: Reader, vocab: Vocab, device: str = "cpu") -> None: | |
| self.encoder, self.vocab, self.device = encoder, vocab, device | |
| self.reader = reader.to(device).eval() | |
| def logits(self, wav: np.ndarray) -> np.ndarray: | |
| """float32 mono 16 kHz audio of any length -> stitched reader logits [T, V].""" | |
| pieces = [] | |
| for w in plan_windows(len(wav)): | |
| tokens = self.encoder(wav[w.start_sample : w.end_sample]) | |
| x = torch.from_numpy(np.ascontiguousarray(tokens, dtype=np.float32))[None] | |
| x = x.to(self.device) | |
| mask = torch.ones(1, x.shape[1], dtype=torch.bool, device=self.device) | |
| with torch.no_grad(): | |
| lg = self.reader(x, mask)[0].float().cpu().numpy() | |
| pieces.append(lg[w.emit_from : w.emit_to]) | |
| if not pieces: | |
| return np.zeros((0, len(self.vocab)), np.float32) | |
| return np.concatenate(pieces) | |
| def _wav(audio, sample_rate: int | None) -> np.ndarray: | |
| if isinstance(audio, str | Path): | |
| return load_audio(audio) | |
| if sample_rate is None: | |
| raise ValueError("pass sample_rate with an audio array") | |
| return to_mono_16k(audio, sample_rate) | |
| def _text(self, wav: np.ndarray) -> str: | |
| logits = self.logits(wav) | |
| if logits.shape[0] == 0: | |
| return "" | |
| return self.vocab.decode(greedy_ctc(logits, self.vocab.blank_id)) | |
| def transcribe(self, audio, sample_rate: int | None = None) -> str: | |
| """`audio` is a file path, or an array with its `sample_rate`.""" | |
| return self._text(self._wav(audio, sample_rate)) | |
| def chunks(self, audio, sample_rate: int | None = None, max_s: float = 30.0): | |
| """Transcribe long audio piece by piece, cut at pauses (`split_on_pauses`). Yields | |
| {"start": s, "end": s, "text": ...} per chunk, so the text can be shown as it comes.""" | |
| wav = self._wav(audio, sample_rate) | |
| for a, b in split_on_pauses(wav, max_s): | |
| yield {"start": a / SAMPLE_RATE, "end": b / SAMPLE_RATE, "text": self._text(wav[a:b])} | |
| def load(folder: str | Path | None = None, device: str | None = None) -> Transcriber: | |
| """Load the encoder and the reader from `folder` (default: the folder of this file). | |
| `device` defaults to "cuda" when available, else "cpu"; pass "mps" on Apple silicon.""" | |
| root = Path(folder) if folder is not None else Path(__file__).resolve().parent | |
| if device is None: | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| vocab = Vocab.load(root / "reader" / "vocab_ko.txt") | |
| reader = load_reader(root / "reader" / "model.safetensors", vocab) | |
| return Transcriber(EG2Encoder(root / "encoder", device), reader, vocab, device) | |
| def main(argv: list[str] | None = None) -> int: | |
| args = sys.argv[1:] if argv is None else argv | |
| if not args or args[0] in ("-h", "--help"): | |
| print("usage: python eg2_ko_reader.py audio.wav [more.wav ...]", file=sys.stderr) | |
| return 2 | |
| stt = load() | |
| for path in args: | |
| text = stt.transcribe(path) | |
| print(text if len(args) == 1 else f"{path}\t{text}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |