Instructions to use batuhan28/tr-fastconformer-stream with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- NeMo
How to use batuhan28/tr-fastconformer-stream with NeMo:
# tag did not correspond to a valid NeMo domain.
- Notebooks
- Google Colab
- Kaggle
Download v4/code/lib.py from batuhan28/tr-fastconformer-stream: direct link, hf CLI and curl.
- Browser
- Download file 17.4 kB
-
https://huggingface.co/batuhan28/tr-fastconformer-stream/resolve/main/v4/code/lib.py
- Command line
-
hf download hf://batuhan28/tr-fastconformer-stream/v4/code/lib.py
-
curl -L -o lib.py https://huggingface.co/batuhan28/tr-fastconformer-stream/resolve/main/v4/code/lib.py
17.4 kB
| """Torch'suz yardımcılar: metin temizleme, Türkçe ekli sayı okunuşu, soxr augmentasyon, 85GB RAM önbellekli akış, WER, KenLM.""" | |
| import io, math, re, subprocess, shutil, os, sys, time, collections | |
| from pathlib import Path | |
| import numpy as np | |
| from scipy.signal import resample_poly, butter, sosfilt | |
| try: | |
| import soxr | |
| except ImportError: | |
| soxr = None | |
| SR = 16000 | |
| TOK0 = 0x4E00 | |
| _TR = str.maketrans({"â": "a", "î": "i", "û": "u", "ê": "e", "ô": "o", "ć": "c", | |
| "é": "e", "è": "e", "á": "a", "à": "a", "ó": "o", "ú": "u", "í": "i", | |
| "ñ": "n", "ß": "ss", "'": "", "’": "", "`": "", "‘": ""}) | |
| def lower_tr(t): | |
| return t.replace("İ", "i").replace("I", "ı").lower() | |
| def lower_en(t): | |
| return t.replace("İ", "i").lower() | |
| B = "sıfır bir iki üç dört beş altı yedi sekiz dokuz".split() | |
| O = ["", "on", "yirmi", "otuz", "kırk", "elli", "altmış", "yetmiş", "seksen", "doksan"] | |
| SCALE = ["", "bin", "milyon", "milyar", "trilyon"] | |
| def cardinal(n): | |
| if n == 0: | |
| return B[0] | |
| if n >= 10 ** 15: | |
| return " ".join(B[int(x)] for x in str(n)) | |
| def three(v): | |
| h, t, u = v // 100, v // 10 % 10, v % 10 | |
| return " ".join(x for x in [(("" if h == 1 else B[h] + " ") + "yüz") if h else "", O[t], B[u] if u else ""] if x) | |
| parts, i = [], 0 | |
| while n: | |
| v = n % 1000 | |
| n //= 1000 | |
| if v: | |
| body = "" if (i == 1 and v == 1) else three(v) | |
| parts.append((body + " " + SCALE[i]).strip()) | |
| i += 1 | |
| return " ".join(reversed(parts)) | |
| def digits_spoken(s): | |
| return " ".join(B[int(c)] for c in s if c.isdigit()) | |
| def attach_tr_suffix(spoken, raw_suffix): | |
| suf = lower_tr(raw_suffix.lstrip("'’`‘")).translate(_TR) | |
| if not suf: | |
| return spoken | |
| words = spoken.split() | |
| if not words: | |
| return spoken | |
| last = words[-1] | |
| if last == "dört" and suf[0] in "aeıioöuü": | |
| words[-1] = "dörd" + suf | |
| else: | |
| words[-1] = last + suf | |
| return " ".join(words) | |
| _NUM = re.compile(r"(%?)(\d[\d.,]*\d|\d)(%?)(['’`‘][a-zA-ZçğıöşüÇĞİÖŞÜ]+)?") | |
| def verbalize(text, strict=True): | |
| bad = False | |
| def one(m): | |
| nonlocal bad | |
| pre, num, post, suffix = m.group(1), m.group(2), m.group(3), m.group(4) | |
| out = None | |
| if re.fullmatch(r"\d+", num): | |
| if len(num) >= 7 or (len(num) > 1 and num.startswith("0")): | |
| bad = True | |
| out = digits_spoken(num) | |
| else: | |
| out = cardinal(int(num)) | |
| elif re.fullmatch(r"\d{1,3}(\.\d{3})+", num): | |
| out = cardinal(int(num.replace(".", ""))) | |
| elif re.fullmatch(r"\d+,\d+", num): | |
| a, b = num.split(",") | |
| out = cardinal(int(a)) + " virgül " + digits_spoken(b) | |
| else: | |
| bad = True | |
| out = digits_spoken(num) | |
| if suffix and out: | |
| out = attach_tr_suffix(out, suffix) | |
| if pre or post: | |
| out = "yüzde " + out | |
| return " " + out + " " | |
| res = _NUM.sub(one, text) | |
| if bad and strict: | |
| return None | |
| return res | |
| def clean_text(t, lang="tr", strict=True): | |
| if not isinstance(t, str): | |
| return None | |
| t = verbalize(t, strict=strict) | |
| if t is None: | |
| return None | |
| t = (lower_en(t) if lang == "en" else lower_tr(t)).translate(_TR) | |
| return re.sub(r"\s+", " ", re.sub(r"[^a-zçğıöşü ]", " ", t)).strip() | |
| def label_ok(text, dur, max_cps=25.0, min_cps=2.0): | |
| if not text or dur <= 0: | |
| return False | |
| cps = len(text) / dur | |
| if cps > max_cps or cps < min_cps: | |
| return False | |
| w = text.split() | |
| if len(w) >= 10 and len(set(w)) / len(w) < 0.25: | |
| return False | |
| if re.search(r"\b(\w+(?: \w+){0,3})(?: \1){5,}", text): | |
| return False | |
| return True | |
| def frames_needed(ids): | |
| return len(ids) + sum(a == b for a, b in zip(ids, ids[1:])) | |
| def lev(a, b): | |
| from rapidfuzz.distance import Levenshtein as L | |
| return L.distance(a, b) | |
| def wer_counts(refs, hyps): | |
| we = wn = ce = cn = 0 | |
| for r, h in zip(refs, hyps): | |
| rw, hw = r.split(), h.split() | |
| we += lev(rw, hw); wn += len(rw) | |
| ce += lev(r, h); cn += len(r) | |
| return we, wn, ce, cn | |
| def fast_resample(w, sr_in, sr_out): | |
| if sr_in == sr_out: | |
| return w.astype("float32", copy=False) | |
| if soxr is not None: | |
| return soxr.resample(w, sr_in, sr_out, quality="HQ").astype("float32") | |
| g = math.gcd(int(sr_in), int(sr_out)) | |
| return resample_poly(w, int(sr_out) // g, int(sr_in) // g).astype("float32") | |
| def to_16k(w, sr): | |
| return fast_resample(w, sr, SR) | |
| def decode_audio(raw): | |
| import soundfile as sf | |
| w, sr = sf.read(io.BytesIO(raw), dtype="float32", always_2d=False) | |
| if w.ndim > 1: | |
| w = w.mean(1) | |
| return to_16k(w, sr) | |
| _SOS_TEL = butter(4, [300, 3400], btype="bandpass", fs=8000, output="sos") | |
| def mulaw(w, mu=255): | |
| x = np.clip(w, -1, 1) | |
| y = np.sign(x) * np.log1p(mu * np.abs(x)) / np.log1p(mu) | |
| y = np.round((y + 1) / 2 * 255) / 255 * 2 - 1 | |
| return (np.sign(y) * ((1 + mu) ** np.abs(y) - 1) / mu).astype("float32") | |
| def telephone(w): | |
| n = fast_resample(w, 16000, 8000) | |
| n = mulaw(sosfilt(_SOS_TEL, n).astype("float32")) | |
| out = fast_resample(n, 8000, 16000) | |
| return out[: len(w)] if len(out) >= len(w) else np.pad(out, (0, len(w) - len(out))) | |
| def edge(w, rng, thr=0.01, max_pad=0.6): | |
| a = np.abs(w) | |
| if a.max() > 0: | |
| idx = np.flatnonzero(a > thr * a.max()) | |
| if len(idx): | |
| w = w[max(0, idx[0] - 800): idx[-1] + 800] | |
| pre = np.zeros(int(rng.uniform(0, max_pad) * SR), "float32") | |
| post = np.zeros(int(rng.uniform(0, max_pad) * SR), "float32") | |
| return np.concatenate([pre, w, post]) | |
| def augment(w, rng, tel_p=0.4, edge_p=0.4, noise_p=0.12, speed_p=0.15, gain_p=0.3): | |
| if rng.random() < tel_p: | |
| w = telephone(w) | |
| if rng.random() < noise_p: | |
| noise = rng.standard_normal(len(w)).astype("float32") | |
| snr = rng.uniform(8, 25) | |
| w = w + noise * np.sqrt((np.mean(w * w) + 1e-9) / 10 ** (snr / 10)) | |
| if rng.random() < speed_p: | |
| rate = float(rng.choice([0.9, 0.95, 1.05, 1.1])) | |
| w = fast_resample(w, int(SR * rate), SR) | |
| if rng.random() < gain_p: | |
| w = w * float(10 ** (rng.uniform(-9, 6) / 20)) | |
| if rng.random() < edge_p: | |
| w = edge(w, rng) | |
| return np.clip(w, -1, 1).astype("float32") | |
| def make_batches(buffer, max_seconds, max_clips, rng): | |
| buffer = sorted(buffer, key=lambda z: len(z[0])) | |
| batches, cur, longest = [], [], 0 | |
| for item in buffer: | |
| n = len(item[0]) | |
| if cur and (max(longest, n) * (len(cur) + 1) > max_seconds * SR or len(cur) >= max_clips): | |
| batches.append(cur); cur, longest = [], 0 | |
| cur.append(item); longest = max(longest, n) | |
| if cur: | |
| batches.append(cur) | |
| order = rng.permutation(len(batches)) | |
| return [batches[i] for i in order] | |
| def pick_representative(durs, words, chars, n, k_dur=5, k_rate=3): | |
| d = np.asarray(durs, float); w = np.asarray(words, float); c = np.asarray(chars, float) | |
| N = len(d) | |
| if n >= N: | |
| return list(range(N)) | |
| rate = c / np.maximum(d, 1e-6) | |
| wl = c / np.maximum(w, 1) | |
| def qbin(x, k): | |
| return np.searchsorted(np.quantile(x, np.linspace(0, 1, k + 1)[1:-1]), x, side="right") | |
| strata = qbin(d, k_dur) * k_rate + qbin(rate, k_rate) | |
| ids = np.unique(strata) | |
| mass = {s: w[strata == s].sum() for s in ids} | |
| tot = sum(mass.values()) | |
| quota = {s: n * mass[s] / tot for s in ids} | |
| alloc = {s: int(np.floor(quota[s])) for s in ids} | |
| for s in sorted(ids, key=lambda s: -(quota[s] - alloc[s]))[: n - sum(alloc.values())]: | |
| alloc[s] += 1 | |
| chosen = [] | |
| for s in ids: | |
| idx = np.flatnonzero(strata == s) | |
| k = min(alloc[s], len(idx)) | |
| if k: | |
| order = idx[np.argsort(wl[idx], kind="stable")] | |
| chosen += [int(order[int((j + 0.5) / k * len(order))]) for j in range(k)] | |
| return sorted(chosen) | |
| def to_i16(w): | |
| return (np.clip(w, -1, 1) * 32767).astype("int16") | |
| def batch_stream(groups, share, hours, worker, nworkers, rng, read, ids_fn, blocked, seconds, max_clips, | |
| lang_of=lambda s: "tr", tel_p=0.4, edge_p=0.4, min_buffer=3200, ram_cache_shards=4, status_fn=None): | |
| """85GB RAM Sömürücü: Her worker 4 arka plan indirme iş parçacığı + RAM içi LRU Shard Havuzu + 3200 kliplik dev tampon kullanır.""" | |
| from concurrent.futures import ThreadPoolExecutor | |
| sources = list(groups) | |
| weight = np.array([share[s] / (hours[s] / max(1, len(groups[s]))) for s in sources], dtype="float64") | |
| weight /= weight.sum() | |
| pools = {s: groups[s][worker::nworkers] for s in sources} | |
| assert all(pools.values()), "Worker başına shard kalmadı." | |
| # Worker başına RAM içi LRU önbelleği (Disk kullanmaz, doğrudan 85GB sistem RAM'ini değerlendirir) | |
| ram_cache = collections.OrderedDict() | |
| ex = ThreadPoolExecutor(4) | |
| def fetch_shard(shard_path): | |
| if shard_path in ram_cache: | |
| ram_cache.move_to_end(shard_path) | |
| return ram_cache[shard_path] | |
| d = read(shard_path) | |
| # Sadece geçerli satırları hafif tuple listesi olarak RAM önbelleğine al | |
| parsed = [] | |
| for a, t, dur in zip(d["audio"], d["text"], d["duration"]): | |
| if a and a.get("bytes") and dur is not None and 1 <= dur <= 20: | |
| parsed.append((a["bytes"], t, float(dur))) | |
| del d | |
| if ram_cache_shards > 0: | |
| ram_cache[shard_path] = parsed | |
| while len(ram_cache) > ram_cache_shards: | |
| ram_cache.popitem(last=False) | |
| return parsed | |
| def nxt(): | |
| s = sources[int(rng.choice(len(sources), p=weight))] | |
| shard_path = pools[s][int(rng.integers(len(pools[s])))] | |
| return s, shard_path, ex.submit(fetch_shard, shard_path) | |
| # 4 shard'ı paralel olarak ağdan RAM'e çekmeye başla | |
| q = collections.deque([nxt() for _ in range(4)]) | |
| buf = [] | |
| first_flush = True | |
| while True: | |
| s, spath, fut = q.popleft() | |
| q.append(nxt()) | |
| if first_flush and worker == 0 and status_fn: | |
| status_fn(f"📥 [Worker 0] 4x Paralel Shard RAM'e çekiliyor: {spath} (Tampon: {len(buf)}/900 klip)...") | |
| rows = fut.result() | |
| lang = lang_of(s) | |
| target_buf = 900 if first_flush else min_buffer | |
| # Shard içindeki sırayı da karıştır ki her tekrarda farklı kombinasyonlarla augment edilsin | |
| idxs = rng.permutation(len(rows)) | |
| for i in idxs: | |
| raw_bytes, t, dur = rows[i] | |
| text = clean_text(t, lang) | |
| if not text or text in blocked or not label_ok(text, dur): | |
| continue | |
| ids = ids_fn(text) | |
| if not ids: | |
| continue | |
| need = frames_needed(ids) + 2 | |
| if need >= int(dur * 25 * 0.9): | |
| continue | |
| w = augment(decode_audio(raw_bytes), rng, tel_p, edge_p) | |
| if need >= len(w) // 640 - 1: | |
| continue | |
| buf.append((to_i16(w), ids)) | |
| if first_flush and worker == 0 and status_fn and len(buf) % 150 == 0: | |
| status_fn(f"⚡ [Worker 0] Dev A100 Batch hazırlanıyor ({s}): {len(buf)}/{target_buf} klip...") | |
| if first_flush and len(buf) >= target_buf: | |
| break | |
| if len(buf) < target_buf: | |
| continue | |
| if first_flush and worker == 0 and status_fn: | |
| status_fn(f"🔥 İlk Dev Batch ({seconds} sn) hazır! A100 40GB VRAM tam kapasite devreye giriyor...") | |
| first_flush = False | |
| for b in make_batches(buf, seconds, max_clips, rng): | |
| yield b | |
| buf = [] | |
| def featurize_np(w, fb, window, n_fft=512, hop=160, preemph=0.97, pad_mode="constant", guard=2.0 ** -24): | |
| w = np.asarray(w, dtype="float32") | |
| x = np.concatenate([w[:1], w[1:] - preemph * w[:-1]]) if preemph else w | |
| xp = np.pad(x, (n_fft // 2, n_fft // 2), mode=pad_mode) | |
| n_all = 1 + len(w) // hop | |
| idx = np.arange(n_fft)[None, :] + hop * np.arange(n_all)[:, None] | |
| win = np.zeros(n_fft, "float32") | |
| lp = (n_fft - len(window)) // 2 | |
| win[lp: lp + len(window)] = window | |
| spec = np.abs(np.fft.rfft(xp[idx] * win[None, :], axis=1)).astype("float32") ** 2 | |
| mel = np.log(fb @ spec.T + guard) | |
| n = len(w) // hop | |
| mel = mel[:, :n] | |
| mean = mel.mean(1, keepdims=True) | |
| std = np.sqrt(((mel - mean) ** 2).sum(1, keepdims=True) / max(n - 1, 1)) | |
| return ((mel - mean) / (std + 1e-5)).astype("float32") | |
| def lm_line(sp, text): | |
| return " ".join(chr(TOK0 + i) for i in sp.encode(text)) | |
| def ensure_kenlm(log=print): | |
| p = shutil.which("lmplz") | |
| if p: | |
| return str(Path(p).parent) | |
| root = Path("/tmp/kenlm") | |
| if (root / "build/bin/lmplz").exists(): | |
| return str(root / "build/bin") | |
| try: | |
| def sh(c): | |
| subprocess.run(c, shell=True, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) | |
| log("KenLM C++ araçları (lmplz/build_binary) derleniyor...") | |
| sh("apt-get update -qq || true") | |
| sh("DEBIAN_FRONTEND=noninteractive apt-get install -y -qq build-essential libboost-all-dev " | |
| "libeigen3-dev zlib1g-dev libbz2-dev liblzma-dev") | |
| if not root.exists(): | |
| sh(f"git clone -q --depth 1 https://github.com/kpu/kenlm.git {root}") | |
| cml = root / "CMakeLists.txt" | |
| if cml.exists(): | |
| cml.write_text(re.sub(r"cmake_minimum_required\s*\([^)]*\)", "cmake_minimum_required(VERSION 3.5)", cml.read_text(), flags=re.I)) | |
| sh(f"mkdir -p {root}/build && cd {root}/build && " | |
| f"cmake .. -DCMAKE_BUILD_TYPE=Release && " | |
| f"make -j{os.cpu_count() or 4} lmplz build_binary") | |
| return str(root / "build/bin") | |
| except Exception as e: | |
| log("KenLM derlenemedi:", repr(e)[:200]) | |
| return None | |
| def build_kenlm(corpus_path, out_binary, bin_dir, order=6, mem="24G", tmp="/tmp"): | |
| arpa = str(Path(out_binary).with_suffix(".arpa")) | |
| with open(corpus_path, "rb") as src: | |
| subprocess.run([f"{bin_dir}/lmplz", "-o", str(order), "-S", mem, "-T", tmp + "/", "--arpa", arpa, | |
| "--discount_fallback", "--prune", *map(str, ([0, 0, 1, 1, 2, 2] + [2] * 8)[:order])], | |
| stdin=src, check=True, stderr=subprocess.DEVNULL) | |
| subprocess.run([f"{bin_dir}/build_binary", "-q", "8", "-b", "8", "-a", "22", "trie", arpa, str(out_binary)], | |
| check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) | |
| os.remove(arpa) | |
| class LMDecoder: | |
| def __init__(self, lm_path, vocab, alpha=0.5, beta=0.0, beam=48, top_tokens=16, thresh=15.0): | |
| self.V = vocab | |
| self.lm_path = str(lm_path) | |
| self.beam = beam | |
| self.top_tokens = min(top_tokens, vocab + 2) | |
| self.thresh = thresh | |
| self.backend = None | |
| try: | |
| from flashlight.lib.text.decoder import (CriterionType, KenLM, LexiconFreeDecoder, | |
| LexiconFreeDecoderOptions) | |
| from flashlight.lib.text.dictionary import Dictionary | |
| self._fl_CriterionType = CriterionType | |
| self._fl_Opts = LexiconFreeDecoderOptions | |
| self._fl_Cls = LexiconFreeDecoder | |
| toks = [chr(TOK0 + i) for i in range(vocab)] + ["<blank>", "<sil>"] | |
| self.dict = Dictionary(toks) | |
| self.blank, self.sil = vocab, vocab + 1 | |
| self.lm = KenLM(self.lm_path, self.dict) | |
| self.backend = "flashlight" | |
| self.set(alpha=alpha, beta=beta, beam=beam) | |
| except Exception: | |
| from pyctcdecode import build_ctcdecoder | |
| labels = [chr(TOK0 + i) + " " for i in range(vocab)] | |
| self._pyctc_labels = labels | |
| self._pyctc_build = build_ctcdecoder | |
| self.backend = "pyctcdecode" | |
| self.set(alpha=alpha, beta=beta, beam=beam) | |
| def set(self, alpha=0.5, beta=0.0, beam=None): | |
| if beam is not None: | |
| self.beam = beam | |
| if self.backend == "flashlight": | |
| self.opts = self._fl_Opts( | |
| self.beam, self.top_tokens, self.thresh, float(alpha), float(beta), False, self._fl_CriterionType.CTC | |
| ) | |
| self.dec = self._fl_Cls(self.opts, self.lm, self.sil, self.blank, []) | |
| else: | |
| self.dec = self._pyctc_build(self._pyctc_labels, kenlm_model_path=self.lm_path, alpha=float(alpha), beta=float(beta)) | |
| def decode(self, logp): | |
| if self.backend == "flashlight": | |
| T = logp.shape[0] | |
| em = np.full((T, self.V + 2), -50.0, dtype="float32") | |
| em[:, : self.V + 1] = logp | |
| em[:, self.sil] = logp[:, self.V] | |
| em = np.ascontiguousarray(em) | |
| res = self.dec.decode(em.ctypes.data, T, self.V + 2) | |
| if not res: | |
| return [] | |
| best = res[0].tokens | |
| out, prev = [], None | |
| for t in best: | |
| if t != prev and 0 <= t < self.V: | |
| out.append(t) | |
| prev = t | |
| return out | |
| else: | |
| txt = self.dec.decode(logp, beam_width=self.beam) | |
| return [ord(ch) - TOK0 for ch in txt if 0 <= ord(ch) - TOK0 < self.V] | |