"""Torch'suz yardımcılar: metin temizleme, Türkçe ekli sayı okunuşu, soxr augmentasyon, 85GB RAM önbellekli akış, WER, KenLM.""" import io, math, re, subprocess, shutil, os, sys, time, collections from pathlib import Path import numpy as np from scipy.signal import resample_poly, butter, sosfilt try: import soxr except ImportError: soxr = None SR = 16000 TOK0 = 0x4E00 _TR = str.maketrans({"â": "a", "î": "i", "û": "u", "ê": "e", "ô": "o", "ć": "c", "é": "e", "è": "e", "á": "a", "à": "a", "ó": "o", "ú": "u", "í": "i", "ñ": "n", "ß": "ss", "'": "", "’": "", "`": "", "‘": ""}) def lower_tr(t): return t.replace("İ", "i").replace("I", "ı").lower() def lower_en(t): return t.replace("İ", "i").lower() B = "sıfır bir iki üç dört beş altı yedi sekiz dokuz".split() O = ["", "on", "yirmi", "otuz", "kırk", "elli", "altmış", "yetmiş", "seksen", "doksan"] SCALE = ["", "bin", "milyon", "milyar", "trilyon"] def cardinal(n): if n == 0: return B[0] if n >= 10 ** 15: return " ".join(B[int(x)] for x in str(n)) def three(v): h, t, u = v // 100, v // 10 % 10, v % 10 return " ".join(x for x in [(("" if h == 1 else B[h] + " ") + "yüz") if h else "", O[t], B[u] if u else ""] if x) parts, i = [], 0 while n: v = n % 1000 n //= 1000 if v: body = "" if (i == 1 and v == 1) else three(v) parts.append((body + " " + SCALE[i]).strip()) i += 1 return " ".join(reversed(parts)) def digits_spoken(s): return " ".join(B[int(c)] for c in s if c.isdigit()) def attach_tr_suffix(spoken, raw_suffix): suf = lower_tr(raw_suffix.lstrip("'’`‘")).translate(_TR) if not suf: return spoken words = spoken.split() if not words: return spoken last = words[-1] if last == "dört" and suf[0] in "aeıioöuü": words[-1] = "dörd" + suf else: words[-1] = last + suf return " ".join(words) _NUM = re.compile(r"(%?)(\d[\d.,]*\d|\d)(%?)(['’`‘][a-zA-ZçğıöşüÇĞİÖŞÜ]+)?") def verbalize(text, strict=True): bad = False def one(m): nonlocal bad pre, num, post, suffix = m.group(1), m.group(2), m.group(3), m.group(4) out = None if re.fullmatch(r"\d+", num): if len(num) >= 7 or (len(num) > 1 and num.startswith("0")): bad = True out = digits_spoken(num) else: out = cardinal(int(num)) elif re.fullmatch(r"\d{1,3}(\.\d{3})+", num): out = cardinal(int(num.replace(".", ""))) elif re.fullmatch(r"\d+,\d+", num): a, b = num.split(",") out = cardinal(int(a)) + " virgül " + digits_spoken(b) else: bad = True out = digits_spoken(num) if suffix and out: out = attach_tr_suffix(out, suffix) if pre or post: out = "yüzde " + out return " " + out + " " res = _NUM.sub(one, text) if bad and strict: return None return res def clean_text(t, lang="tr", strict=True): if not isinstance(t, str): return None t = verbalize(t, strict=strict) if t is None: return None t = (lower_en(t) if lang == "en" else lower_tr(t)).translate(_TR) return re.sub(r"\s+", " ", re.sub(r"[^a-zçğıöşü ]", " ", t)).strip() def label_ok(text, dur, max_cps=25.0, min_cps=2.0): if not text or dur <= 0: return False cps = len(text) / dur if cps > max_cps or cps < min_cps: return False w = text.split() if len(w) >= 10 and len(set(w)) / len(w) < 0.25: return False if re.search(r"\b(\w+(?: \w+){0,3})(?: \1){5,}", text): return False return True def frames_needed(ids): return len(ids) + sum(a == b for a, b in zip(ids, ids[1:])) def lev(a, b): from rapidfuzz.distance import Levenshtein as L return L.distance(a, b) def wer_counts(refs, hyps): we = wn = ce = cn = 0 for r, h in zip(refs, hyps): rw, hw = r.split(), h.split() we += lev(rw, hw); wn += len(rw) ce += lev(r, h); cn += len(r) return we, wn, ce, cn def fast_resample(w, sr_in, sr_out): if sr_in == sr_out: return w.astype("float32", copy=False) if soxr is not None: return soxr.resample(w, sr_in, sr_out, quality="HQ").astype("float32") g = math.gcd(int(sr_in), int(sr_out)) return resample_poly(w, int(sr_out) // g, int(sr_in) // g).astype("float32") def to_16k(w, sr): return fast_resample(w, sr, SR) def decode_audio(raw): import soundfile as sf w, sr = sf.read(io.BytesIO(raw), dtype="float32", always_2d=False) if w.ndim > 1: w = w.mean(1) return to_16k(w, sr) _SOS_TEL = butter(4, [300, 3400], btype="bandpass", fs=8000, output="sos") def mulaw(w, mu=255): x = np.clip(w, -1, 1) y = np.sign(x) * np.log1p(mu * np.abs(x)) / np.log1p(mu) y = np.round((y + 1) / 2 * 255) / 255 * 2 - 1 return (np.sign(y) * ((1 + mu) ** np.abs(y) - 1) / mu).astype("float32") def telephone(w): n = fast_resample(w, 16000, 8000) n = mulaw(sosfilt(_SOS_TEL, n).astype("float32")) out = fast_resample(n, 8000, 16000) return out[: len(w)] if len(out) >= len(w) else np.pad(out, (0, len(w) - len(out))) def edge(w, rng, thr=0.01, max_pad=0.6): a = np.abs(w) if a.max() > 0: idx = np.flatnonzero(a > thr * a.max()) if len(idx): w = w[max(0, idx[0] - 800): idx[-1] + 800] pre = np.zeros(int(rng.uniform(0, max_pad) * SR), "float32") post = np.zeros(int(rng.uniform(0, max_pad) * SR), "float32") return np.concatenate([pre, w, post]) def augment(w, rng, tel_p=0.4, edge_p=0.4, noise_p=0.12, speed_p=0.15, gain_p=0.3): if rng.random() < tel_p: w = telephone(w) if rng.random() < noise_p: noise = rng.standard_normal(len(w)).astype("float32") snr = rng.uniform(8, 25) w = w + noise * np.sqrt((np.mean(w * w) + 1e-9) / 10 ** (snr / 10)) if rng.random() < speed_p: rate = float(rng.choice([0.9, 0.95, 1.05, 1.1])) w = fast_resample(w, int(SR * rate), SR) if rng.random() < gain_p: w = w * float(10 ** (rng.uniform(-9, 6) / 20)) if rng.random() < edge_p: w = edge(w, rng) return np.clip(w, -1, 1).astype("float32") def make_batches(buffer, max_seconds, max_clips, rng): buffer = sorted(buffer, key=lambda z: len(z[0])) batches, cur, longest = [], [], 0 for item in buffer: n = len(item[0]) if cur and (max(longest, n) * (len(cur) + 1) > max_seconds * SR or len(cur) >= max_clips): batches.append(cur); cur, longest = [], 0 cur.append(item); longest = max(longest, n) if cur: batches.append(cur) order = rng.permutation(len(batches)) return [batches[i] for i in order] def pick_representative(durs, words, chars, n, k_dur=5, k_rate=3): d = np.asarray(durs, float); w = np.asarray(words, float); c = np.asarray(chars, float) N = len(d) if n >= N: return list(range(N)) rate = c / np.maximum(d, 1e-6) wl = c / np.maximum(w, 1) def qbin(x, k): return np.searchsorted(np.quantile(x, np.linspace(0, 1, k + 1)[1:-1]), x, side="right") strata = qbin(d, k_dur) * k_rate + qbin(rate, k_rate) ids = np.unique(strata) mass = {s: w[strata == s].sum() for s in ids} tot = sum(mass.values()) quota = {s: n * mass[s] / tot for s in ids} alloc = {s: int(np.floor(quota[s])) for s in ids} for s in sorted(ids, key=lambda s: -(quota[s] - alloc[s]))[: n - sum(alloc.values())]: alloc[s] += 1 chosen = [] for s in ids: idx = np.flatnonzero(strata == s) k = min(alloc[s], len(idx)) if k: order = idx[np.argsort(wl[idx], kind="stable")] chosen += [int(order[int((j + 0.5) / k * len(order))]) for j in range(k)] return sorted(chosen) def to_i16(w): return (np.clip(w, -1, 1) * 32767).astype("int16") def batch_stream(groups, share, hours, worker, nworkers, rng, read, ids_fn, blocked, seconds, max_clips, lang_of=lambda s: "tr", tel_p=0.4, edge_p=0.4, min_buffer=3200, ram_cache_shards=4, status_fn=None): """85GB RAM Sömürücü: Her worker 4 arka plan indirme iş parçacığı + RAM içi LRU Shard Havuzu + 3200 kliplik dev tampon kullanır.""" from concurrent.futures import ThreadPoolExecutor sources = list(groups) weight = np.array([share[s] / (hours[s] / max(1, len(groups[s]))) for s in sources], dtype="float64") weight /= weight.sum() pools = {s: groups[s][worker::nworkers] for s in sources} assert all(pools.values()), "Worker başına shard kalmadı." # Worker başına RAM içi LRU önbelleği (Disk kullanmaz, doğrudan 85GB sistem RAM'ini değerlendirir) ram_cache = collections.OrderedDict() ex = ThreadPoolExecutor(4) def fetch_shard(shard_path): if shard_path in ram_cache: ram_cache.move_to_end(shard_path) return ram_cache[shard_path] d = read(shard_path) # Sadece geçerli satırları hafif tuple listesi olarak RAM önbelleğine al parsed = [] for a, t, dur in zip(d["audio"], d["text"], d["duration"]): if a and a.get("bytes") and dur is not None and 1 <= dur <= 20: parsed.append((a["bytes"], t, float(dur))) del d if ram_cache_shards > 0: ram_cache[shard_path] = parsed while len(ram_cache) > ram_cache_shards: ram_cache.popitem(last=False) return parsed def nxt(): s = sources[int(rng.choice(len(sources), p=weight))] shard_path = pools[s][int(rng.integers(len(pools[s])))] return s, shard_path, ex.submit(fetch_shard, shard_path) # 4 shard'ı paralel olarak ağdan RAM'e çekmeye başla q = collections.deque([nxt() for _ in range(4)]) buf = [] first_flush = True while True: s, spath, fut = q.popleft() q.append(nxt()) if first_flush and worker == 0 and status_fn: status_fn(f"📥 [Worker 0] 4x Paralel Shard RAM'e çekiliyor: {spath} (Tampon: {len(buf)}/900 klip)...") rows = fut.result() lang = lang_of(s) target_buf = 900 if first_flush else min_buffer # Shard içindeki sırayı da karıştır ki her tekrarda farklı kombinasyonlarla augment edilsin idxs = rng.permutation(len(rows)) for i in idxs: raw_bytes, t, dur = rows[i] text = clean_text(t, lang) if not text or text in blocked or not label_ok(text, dur): continue ids = ids_fn(text) if not ids: continue need = frames_needed(ids) + 2 if need >= int(dur * 25 * 0.9): continue w = augment(decode_audio(raw_bytes), rng, tel_p, edge_p) if need >= len(w) // 640 - 1: continue buf.append((to_i16(w), ids)) if first_flush and worker == 0 and status_fn and len(buf) % 150 == 0: status_fn(f"⚡ [Worker 0] Dev A100 Batch hazırlanıyor ({s}): {len(buf)}/{target_buf} klip...") if first_flush and len(buf) >= target_buf: break if len(buf) < target_buf: continue if first_flush and worker == 0 and status_fn: status_fn(f"🔥 İlk Dev Batch ({seconds} sn) hazır! A100 40GB VRAM tam kapasite devreye giriyor...") first_flush = False for b in make_batches(buf, seconds, max_clips, rng): yield b buf = [] def featurize_np(w, fb, window, n_fft=512, hop=160, preemph=0.97, pad_mode="constant", guard=2.0 ** -24): w = np.asarray(w, dtype="float32") x = np.concatenate([w[:1], w[1:] - preemph * w[:-1]]) if preemph else w xp = np.pad(x, (n_fft // 2, n_fft // 2), mode=pad_mode) n_all = 1 + len(w) // hop idx = np.arange(n_fft)[None, :] + hop * np.arange(n_all)[:, None] win = np.zeros(n_fft, "float32") lp = (n_fft - len(window)) // 2 win[lp: lp + len(window)] = window spec = np.abs(np.fft.rfft(xp[idx] * win[None, :], axis=1)).astype("float32") ** 2 mel = np.log(fb @ spec.T + guard) n = len(w) // hop mel = mel[:, :n] mean = mel.mean(1, keepdims=True) std = np.sqrt(((mel - mean) ** 2).sum(1, keepdims=True) / max(n - 1, 1)) return ((mel - mean) / (std + 1e-5)).astype("float32") def lm_line(sp, text): return " ".join(chr(TOK0 + i) for i in sp.encode(text)) def ensure_kenlm(log=print): p = shutil.which("lmplz") if p: return str(Path(p).parent) root = Path("/tmp/kenlm") if (root / "build/bin/lmplz").exists(): return str(root / "build/bin") try: def sh(c): subprocess.run(c, shell=True, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) log("KenLM C++ araçları (lmplz/build_binary) derleniyor...") sh("apt-get update -qq || true") sh("DEBIAN_FRONTEND=noninteractive apt-get install -y -qq build-essential libboost-all-dev " "libeigen3-dev zlib1g-dev libbz2-dev liblzma-dev") if not root.exists(): sh(f"git clone -q --depth 1 https://github.com/kpu/kenlm.git {root}") cml = root / "CMakeLists.txt" if cml.exists(): cml.write_text(re.sub(r"cmake_minimum_required\s*\([^)]*\)", "cmake_minimum_required(VERSION 3.5)", cml.read_text(), flags=re.I)) sh(f"mkdir -p {root}/build && cd {root}/build && " f"cmake .. -DCMAKE_BUILD_TYPE=Release && " f"make -j{os.cpu_count() or 4} lmplz build_binary") return str(root / "build/bin") except Exception as e: log("KenLM derlenemedi:", repr(e)[:200]) return None def build_kenlm(corpus_path, out_binary, bin_dir, order=6, mem="24G", tmp="/tmp"): arpa = str(Path(out_binary).with_suffix(".arpa")) with open(corpus_path, "rb") as src: subprocess.run([f"{bin_dir}/lmplz", "-o", str(order), "-S", mem, "-T", tmp + "/", "--arpa", arpa, "--discount_fallback", "--prune", *map(str, ([0, 0, 1, 1, 2, 2] + [2] * 8)[:order])], stdin=src, check=True, stderr=subprocess.DEVNULL) subprocess.run([f"{bin_dir}/build_binary", "-q", "8", "-b", "8", "-a", "22", "trie", arpa, str(out_binary)], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) os.remove(arpa) class LMDecoder: def __init__(self, lm_path, vocab, alpha=0.5, beta=0.0, beam=48, top_tokens=16, thresh=15.0): self.V = vocab self.lm_path = str(lm_path) self.beam = beam self.top_tokens = min(top_tokens, vocab + 2) self.thresh = thresh self.backend = None try: from flashlight.lib.text.decoder import (CriterionType, KenLM, LexiconFreeDecoder, LexiconFreeDecoderOptions) from flashlight.lib.text.dictionary import Dictionary self._fl_CriterionType = CriterionType self._fl_Opts = LexiconFreeDecoderOptions self._fl_Cls = LexiconFreeDecoder toks = [chr(TOK0 + i) for i in range(vocab)] + ["", ""] self.dict = Dictionary(toks) self.blank, self.sil = vocab, vocab + 1 self.lm = KenLM(self.lm_path, self.dict) self.backend = "flashlight" self.set(alpha=alpha, beta=beta, beam=beam) except Exception: from pyctcdecode import build_ctcdecoder labels = [chr(TOK0 + i) + " " for i in range(vocab)] self._pyctc_labels = labels self._pyctc_build = build_ctcdecoder self.backend = "pyctcdecode" self.set(alpha=alpha, beta=beta, beam=beam) def set(self, alpha=0.5, beta=0.0, beam=None): if beam is not None: self.beam = beam if self.backend == "flashlight": self.opts = self._fl_Opts( self.beam, self.top_tokens, self.thresh, float(alpha), float(beta), False, self._fl_CriterionType.CTC ) self.dec = self._fl_Cls(self.opts, self.lm, self.sil, self.blank, []) else: self.dec = self._pyctc_build(self._pyctc_labels, kenlm_model_path=self.lm_path, alpha=float(alpha), beta=float(beta)) def decode(self, logp): if self.backend == "flashlight": T = logp.shape[0] em = np.full((T, self.V + 2), -50.0, dtype="float32") em[:, : self.V + 1] = logp em[:, self.sil] = logp[:, self.V] em = np.ascontiguousarray(em) res = self.dec.decode(em.ctypes.data, T, self.V + 2) if not res: return [] best = res[0].tokens out, prev = [], None for t in best: if t != prev and 0 <= t < self.V: out.append(t) prev = t return out else: txt = self.dec.decode(logp, beam_width=self.beam) return [ord(ch) - TOK0 for ch in txt if 0 <= ord(ch) - TOK0 < self.V]