import os, io, re, json, time, math, pickle, sys, subprocess, shutil, datetime, glob, gc, uuid, warnings, logging from pathlib import Path from concurrent.futures import ThreadPoolExecutor warnings.filterwarnings("ignore") os.environ["HYDRA_FULL_ERROR"] = "0" import numpy as np import torch import torch.nn.functional as F import soundfile as sf import sentencepiece as spm import pyarrow.parquet as pq from torch.utils.data import IterableDataset, DataLoader from huggingface_hub import HfApi, HfFileSystem import lightning.pytorch as pl from lightning.pytorch.strategies import DDPStrategy from lightning.pytorch.callbacks import ModelCheckpoint from nemo.collections.asr.models import EncDecCTCModelBPE from nemo.utils import logging as nemo_logging nemo_logging.setLevel(logging.ERROR) logging.getLogger("lightning.pytorch").setLevel(logging.ERROR) sys.path.insert(0, str(Path(__file__).parent)) import lib R = Path(__file__).parent STATUS = R / "status.txt" C = json.loads((R / "config.json").read_text()) rank = int(os.environ.get("RANK", 0)) world = int(os.environ.get("WORLD_SIZE", 1)) api = HfApi(token=os.environ["HF_TOKEN"]) fs = HfFileSystem(token=os.environ["HF_TOKEN"]) pl.seed_everything(42, workers=True) # ============================================================================== # ⚡ A100 EXTREME DONANIM BAYRAKLARI # ============================================================================== torch.set_num_threads(2) torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = True torch.backends.cudnn.benchmark = True torch.set_float32_matmul_precision("high") USE_BF16 = bool(C.get("bf16", True) and torch.cuda.is_bf16_supported()) AMP_DTYPE = torch.bfloat16 if USE_BF16 else torch.float16 PRECISION_STR = "bf16-mixed" if USE_BF16 else "16-mixed" SOURCES = list(C["share"]) lang_of = lambda s: "en" if s == "librispeech" else "tr" SCRATCH = Path(f"/tmp/shard_scratch_r{rank}") SCRATCH.mkdir(parents=True, exist_ok=True) def say(*a): ts = time.strftime("%H:%M:%S") msg = " ".join(str(x) for x in a) print(f"[{ts} r{rank}] {msg}", flush=True) if rank == 0: try: STATUS.write_text(f"[{ts}] {msg}", encoding="utf-8") except Exception: pass def read(p, cols): """64MB Bloklarla Ultra-Hızlı İndir -> Çok Çekirdekli Arrow ile RAM'e Al -> Diskten Anında Sil.""" err = None for k in range(4): tmp_file = SCRATCH / f"{uuid.uuid4().hex}_{Path(p).name}" try: with fs.open(f"datasets/{C['data']}/{p}", "rb", block_size=64 * 1024 * 1024) as r_src, open(tmp_file, "wb") as w_dst: shutil.copyfileobj(r_src, w_dst, length=64 * 1024 * 1024) table = pq.read_table(tmp_file, columns=cols, use_threads=True) out = table.to_pydict() del table return out except Exception as e: err = e time.sleep(2 * (k + 1)) finally: if tmp_file.exists(): try: tmp_file.unlink() except OSError: pass raise RuntimeError(f"Shard okunamadı: {p}") from err def atomic_json(obj, path): path = Path(path); tmp = path.with_suffix(".tmp") tmp.write_text(json.dumps(obj, ensure_ascii=False)); tmp.replace(path) def upload(path, name): if C["upload"]: try: api.upload_file(path_or_fileobj=str(path), path_in_repo=f"v4/{name}", repo_id=C["repo"]) except Exception as e: say("yükleme başarısız:", name, repr(e)[:150]) say(f"🚀 EXTREME A100 Süreci Başlatıldı (Batch={C['seconds']}sn | Clips={C['max_clips']} | Workers={C['workers']} | {PRECISION_STR})...") if world > 1: torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) torch.distributed.init_process_group("nccl", timeout=datetime.timedelta(hours=3)) def barrier(): if world > 1: torch.distributed.barrier() # ---------------------------------------------------------------- 1) shard listesi if rank == 0 and not (R / "groups.json").exists(): say("📂 Adım 1/5: HuggingFace veri deposundaki Parquet shard listesi çekiliyor...") files = api.list_repo_files(C["data"], repo_type="dataset") allg = {s: sorted(p for p in files if p.startswith(f"data/{s}/") and p.endswith(".parquet")) for s in SOURCES} assert all(allg.values()), "Bir eğitim kaynağı eksik: " + str([s for s in allg if not allg[s]]) atomic_json({"train": {s: v[1:] for s, v in allg.items()}, "dev": {s: v[0] for s, v in allg.items()}}, R / "groups.json") say("✅ Shard listesi kaydedildi:", {s: len(v) for s, v in allg.items()}) barrier() G = json.loads((R / "groups.json").read_text()) groups, devshard = G["train"], G["dev"] # ---------------------------------------------------------------- 2) FLEURS dev/test def load_fleurs(split): dest = R / f"fleurs_{split}.pkl" if dest.exists(): return pickle.load(dest.open("rb")) say(f"🌍 Adım 2/5: Google FLEURS Türkçe '{split}' veri seti indiriliyor ve sesleri çözülüyor...") from datasets import load_dataset, Audio ds = None for kwargs in ( dict(path="google/fleurs", name="tr_tr", split=split, trust_remote_code=True), dict(path="google/fleurs", name="tr_tr", split=split), dict(path="xtreme_s", name="fleurs.tr_tr", split=split), ): try: ds = load_dataset(**kwargs) break except Exception: continue if ds is None: from huggingface_hub import hf_hub_download import tarfile, csv tsv_name = "dev.tsv" if split == "validation" else f"{split}.tsv" tar_name = "dev.tar.gz" if split == "validation" else f"{split}.tar.gz" tsv_p = hf_hub_download("google/fleurs", f"data/tr_tr/{tsv_name}", repo_type="dataset") tar_p = hf_hub_download("google/fleurs", f"data/tr_tr/audio/{tar_name}", repo_type="dataset") trans_map = {} with open(tsv_p, "r", encoding="utf-8") as f: for row in csv.reader(f, delimiter="\t"): if len(row) >= 3: trans_map[row[1].strip()] = row[2].strip() out = [] with tarfile.open(tar_p, "r:gz") as tar: members = [m for m in tar.getmembers() if m.isfile() and m.name.endswith(".wav")] for idx, m in enumerate(members, 1): fname = Path(m.name).name if fname in trans_map: raw = tar.extractfile(m).read() out.append((lib.decode_audio(raw), trans_map[fname])) if idx % 200 == 0 or idx == len(members): say(f" -> FLEURS {split} (doğrudan tar.gz) ses çözümü: {idx}/{len(members)}") with dest.open("wb") as f: pickle.dump(out, f) return out ds = ds.cast_column("audio", Audio(decode=False)) out = [] tot = len(ds) for idx, ex in enumerate(ds, 1): a = ex["audio"] raw = a.get("bytes") or Path(a["path"]).read_bytes() out.append((lib.decode_audio(raw), ex["transcription"])) if idx % 200 == 0 or idx == tot: say(f" -> FLEURS {split} ses çözümü: {idx}/{tot}") with dest.open("wb") as f: pickle.dump(out, f) shutil.rmtree(Path.home() / ".cache/huggingface/datasets", ignore_errors=True) return out if rank == 0: load_fleurs("validation") atomic_json([t for _, t in load_fleurs("test")], R / "fleurs_test_refs.json") barrier() rawdev = load_fleurs("validation") rawtest_refs = json.loads((R / "fleurs_test_refs.json").read_text()) ref_clean = lambda t: lib.clean_text(t, "tr", strict=False) blocked = {ref_clean(t) for _, t in rawdev} | {ref_clean(t) for t in rawtest_refs} del rawtest_refs # ---------------------------------------------------------------- 3) tokenizer (BPE) + başlangıç modeli if rank == 0 and not (R / "init.nemo").exists(): tokdir = R / "tokenizer"; tokdir.mkdir(exist_ok=True) rng = np.random.default_rng(0) per_source = {s: [] for s in SOURCES} jobs = [] for s in SOURCES: paths = groups[s] pick = [paths[i] for i in np.linspace(0, len(paths) - 1, min(8, len(paths))).astype(int)] for p in pick: jobs.append((s, p)) say(f"🔤 Adım 3/5: Tokenizer eğitimi için {len(jobs)} shard'dan 12 paralel kolla metin örnekleniyor...") def fetch_tok_texts(item): s, p = item d = read(p, ["text"]) out = [t for t in (lib.clean_text(x, lang_of(s)) for x in d["text"]) if t and t not in blocked] return s, p, out done_cnt = 0 with ThreadPoolExecutor(12) as ex: for s, p, out in ex.map(fetch_tok_texts, jobs): per_source[s].extend(out) done_cnt += 1 if done_cnt % 4 == 0 or done_cnt == len(jobs): say(f" -> Tokenizer metin indirme ilerlemesi: [{done_cnt}/{len(jobs)}] shard tamamlandı ({s})") lines = [] for s in SOURCES: n = min(len(per_source[s]), int(C["share"][s] * 2_000_000)) lines += list(rng.choice(per_source[s], size=n, replace=False)) if n < len(per_source[s]) else per_source[s] assert lines, "Tokenizer metni yok." say(f"🔤 SentencePiece BPE ({C['vocab']} token) toplam {len(lines):,} cümle üzerinde eğitiliyor...") (tokdir / "text.txt").write_text("\n".join(lines), encoding="utf-8") del per_source, lines; gc.collect() spm.SentencePieceTrainer.train( input=str(tokdir / "text.txt"), model_prefix=str(tokdir / "tokenizer"), model_type="bpe", vocab_size=C["vocab"], character_coverage=1.0, normalization_rule_name="identity", bos_id=-1, eos_id=-1, pad_id=-1, hard_vocab_limit=False, num_threads=C["cpus"], required_chars="abcçdefgğhıijklmnoöprsştuüvyzqwx", input_sentence_size=2_000_000, shuffle_input_sentence=True, minloglevel=2) (tokdir / "text.txt").unlink(missing_ok=True) sp0 = spm.SentencePieceProcessor(model_file=str(tokdir / "tokenizer.model")) (tokdir / "vocab.txt").write_text("\n".join(sp0.id_to_piece(i) for i in range(sp0.vocab_size()))) say(f"📦 Taban model ({C['base']}) indiriliyor ve Türkçe BPE sözlüğü enjekte ediliyor...") m = EncDecCTCModelBPE.from_pretrained(C["base"], map_location="cpu") m.change_vocabulary(new_tokenizer_dir=str(tokdir), new_tokenizer_type="bpe") m.save_to(str(R / "init.nemo")); del m say("☁️ init.nemo yedekleniyor...") upload(R / "init.nemo", "init.nemo") barrier() # ---------------------------------------------------------------- 4) model (2400s Dev Batch + OOM Korumalı Forward + FusedAdamW) say("🧠 Adım 4/5: init.nemo belleğe yükleniyor...") class ASR(EncDecCTCModelBPE): group_names = ["fleurs"] + SOURCES + ["fleurs_rest"] def configure_optimizers(self): if USE_BF16: opt = torch.optim.AdamW(self.parameters(), lr=C["lr"], betas=(0.9, 0.98), weight_decay=C["wd"], fused=True) else: opt = torch.optim.AdamW(self.parameters(), lr=C["lr"], betas=(0.9, 0.98), weight_decay=C["wd"], foreach=True) total, warm = C["steps"], C["warmup"] decay = int(total * 0.25) def f(s): if s < warm: return (s + 1) / warm if s < total - decay: return 1.0 return max(0.05, 1.0 - 0.95 * (s - (total - decay)) / decay) return {"optimizer": opt, "lr_scheduler": {"scheduler": torch.optim.lr_scheduler.LambdaLR(opt, f), "interval": "step"}} def on_before_optimizer_step(self, optimizer): torch.nn.utils.clip_grad_norm_(self.parameters(), max_norm=1.0) def _chunk_loss(self, x, xl, y, yl): proc_x, proc_xl = self.preprocessor(input_signal=x, length=xl) if self.spec_augmentation is not None and self.training: proc_x = self.spec_augmentation(input_spec=proc_x, length=proc_xl) encoded, el = self.encoder(audio_signal=proc_x, length=proc_xl) lp = self.decoder(encoder_output=encoded) loss = F.ctc_loss(lp.float().transpose(0, 1), y, el.long(), yl, blank=lp.shape[-1] - 1, reduction="none", zero_infinity=True) return loss.sum() def training_step(self, batch, batch_idx): x, xl, y, yl = batch tot_tokens = yl.sum().clamp(min=1) try: return self._chunk_loss(x, xl, y, yl) / tot_tokens except torch.cuda.OutOfMemoryError: # Eğer 2400 sn'lik ekstrem bir batch 40GB sınırını aşarsa çökmez, ikiye bölüp işler! torch.cuda.empty_cache() mid = max(1, x.shape[0] // 2) l1 = self._chunk_loss(x[:mid], xl[:mid], y[:mid], yl[:mid]) l2 = self._chunk_loss(x[mid:], xl[mid:], y[mid:], yl[mid:]) return (l1 + l2) / tot_tokens def on_validation_epoch_start(self): self._acc = torch.zeros(len(self.group_names), 6, device=self.device) self.full_pass = self.global_step % C["full_every"] == 0 def validation_step(self, batch, batch_idx, dataloader_idx=0): if dataloader_idx == 2 and not self.full_pass: return x, xl, y, yl, g, refs = batch lp, el, _ = self.forward(input_signal=x, input_signal_length=xl) lpf = lp.float(); V = lpf.shape[-1] - 1 nll = F.ctc_loss(lpf.transpose(0, 1), y, el.long(), yl, blank=V, reduction="none", zero_infinity=True) pred = lpf.argmax(-1).cpu(); elc = el.cpu(); gl = g.tolist() for i in range(len(refs)): z = pred[i, : int(elc[i])].unique_consecutive().tolist() hyp = self.tokenizer.ids_to_text([v for v in z if v != V]) we, wn, ce, cn = lib.wer_counts([refs[i]], [hyp]) self._acc[gl[i]] += torch.tensor([we, wn, ce, cn, float(nll[i]), float(yl[i])], device=self.device) def on_validation_epoch_end(self): acc = self._acc if world > 1: torch.distributed.all_reduce(acc) rep = {} for k, name in enumerate(self.group_names): we, wn, ce, cn, nl, nt = acc[k].tolist() if wn > 0: rep[name] = dict(wer=100 * we / wn, cer=100 * ce / max(cn, 1), nll=nl / max(nt, 1)) if self.full_pass and "fleurs" in rep and "fleurs_rest" in rep: a, b = acc[0], acc[len(self.group_names) - 1] s_ = (a + b).tolist() rep["fleurs_full"] = dict(wer=100 * s_[0] / max(s_[1], 1), cer=100 * s_[2] / max(s_[3], 1), nll=s_[4] / max(s_[5], 1)) self.domain_report = rep f = rep.get("fleurs", dict(wer=100.0, cer=100.0, nll=99.0)) for key in ("wer", "cer", "nll"): self.log(f"dev_{key}", float(f[key]), sync_dist=(world > 1), rank_zero_only=False, batch_size=1) model = ASR.restore_from(str(R / "init.nemo"), map_location="cpu") tok = model.tokenizer if hasattr(model.encoder, "sync_max_audio_length"): model.encoder.sync_max_audio_length = False V = tok.vocab_size say("✅ Model yüklendi -> Vokabüler:", V, "| Parametre (M):", round(sum(p.numel() for p in model.parameters()) / 1e6, 1)) # ---------------------------------------------------------------- 5) veri def pack(items): ws, ids = zip(*items) x = torch.zeros(len(items), max(map(len, ws))) y = torch.zeros(len(items), max(map(len, ids)), dtype=torch.long) for i, (w, t) in enumerate(items): x[i, : len(w)] = torch.from_numpy(w.astype("float32") / 32767.0) if w.dtype == np.int16 else torch.from_numpy(w) y[i, : len(t)] = torch.tensor(t) return x, torch.tensor(list(map(len, ws))), y, torch.tensor(list(map(len, ids))) class Stream(IterableDataset): def __iter__(self): wi = torch.utils.data.get_worker_info() wid, nw = (wi.id, wi.num_workers) if wi else (0, 1) worker = rank * nw + wid rng = np.random.default_rng(42 + 1000 * C["launch"] + worker) cols = ["audio", "text", "duration"] for b in lib.batch_stream(groups, C["share"], C["hours"], worker, world * nw, rng, lambda p: read(p, cols), tok.text_to_ids, blocked, C["seconds"], C["max_clips"], lang_of=lang_of, tel_p=C["tel_p"], edge_p=C["edge_p"], min_buffer=C.get("min_buffer", 3200), ram_cache_shards=C.get("ram_cache_shards", 4), status_fn=say if worker == 0 else None): yield pack(b) def dev_items(raw, g): out = [] for w, t in raw: ref = ref_clean(t) ids = tok.text_to_ids(ref) if ids: out.append((w, ids, ref, g)) out.sort(key=lambda z: len(z[0])) return out def pack_dev(items): x, xl, y, yl = pack([(w, ids) for w, ids, _, _ in items]) return x, xl, y, yl, torch.tensor([g for *_, g in items]), [r for _, _, r, _ in items] if rank == 0 and not (R / "domain_dev.pkl").exists(): out = [] for idx_s, s in enumerate(SOURCES, 1): say(f"📊 Adım 5/5: Kaynak bazlı doğrulama kümesi hazırlanıyor [{idx_s}/{len(SOURCES)}]: {s}...") d = read(devshard[s], ["audio", "text", "duration"]) rows = [] for a, t, dur in zip(d["audio"], d["text"], d["duration"]): text = lib.clean_text(t, lang_of(s)) if a and a.get("bytes") and dur and 1 <= dur <= 20 and text and lib.label_ok(text, dur) and text not in blocked: rows.append((a["bytes"], text)) pick = np.random.default_rng(0).permutation(len(rows))[: C["dev_per_source"]] out += [(lib.decode_audio(rows[i][0]), rows[i][1], s) for i in pick] del d, rows; gc.collect() with (R / "domain_dev.pkl").open("wb") as f: pickle.dump(out, f) barrier() dom_raw = pickle.load((R / "domain_dev.pkl").open("rb")) dom = [] for w, text, s in dom_raw: ids = tok.text_to_ids(text) if ids: dom.append((w, ids, text, 1 + SOURCES.index(s))) dom.sort(key=lambda z: len(z[0])) del dom_raw train = DataLoader(Stream(), batch_size=None, num_workers=C["workers"], prefetch_factor=C.get("prefetch", 6), pin_memory=True, persistent_workers=(C["workers"] > 0)) fl_all = dev_items(rawdev, 0) sel = set(lib.pick_representative([len(w) / 16000 for w, *_ in fl_all], [len(r.split()) for _, _, r, _ in fl_all], [len(r) for _, _, r, _ in fl_all], C["dev_subset"])) fl_sub = [it for i, it in enumerate(fl_all) if i in sel] fl_rest = [(w, ids, r, 1 + len(SOURCES)) for i, (w, ids, r, _) in enumerate(fl_all) if i not in sel] say("🎯 Hazırlık tamamlandı! FLEURS dev:", len(fl_all), "| Alt küme:", len(fl_sub), "-> A100 Extreme Eğitimi başlıyor...") dev = [DataLoader(fl_sub, batch_size=64, collate_fn=pack_dev, pin_memory=True), DataLoader(dom, batch_size=64, collate_fn=pack_dev, pin_memory=True), DataLoader(fl_rest, batch_size=64, collate_fn=pack_dev, pin_memory=True)] # ---------------------------------------------------------------- 6) callback'ler class Report(pl.Callback): def on_train_start(self, t, m): self.f = (R / "metrics.jsonl").open("a", buffering=1) if t.is_global_zero else None if t.is_global_zero: say(f"🔥 A100 EXTREME ({PRECISION_STR} | Batch={C['seconds']}sn) eğitimi aktif!") def on_train_batch_end(self, t, m, outputs, batch, batch_idx): if self.f and (t.global_step == 1 or t.global_step % 5 == 0): loss = outputs["loss"] if isinstance(outputs, dict) else outputs lr = t.optimizers[0].param_groups[0]["lr"] self.f.write(json.dumps(dict(step=t.global_step, loss=float(loss.detach()), lr=lr, time=time.time())) + "\n") if t.global_step <= 15 or t.global_step % 25 == 0: say(f"⚡ Adım {t.global_step:,}/{C['steps']:,} | Loss: {float(loss.detach()):.3f} | LR: {lr:.2e} | Klip/Batch: {batch[0].shape[0]}") def on_validation_end(self, t, m): if t.sanity_checking or not t.is_global_zero or not self.f: return rep = getattr(m, "domain_report", {}) d = dict(step=t.global_step, time=time.time(), domains=rep) d.update({k: float(v) for k, v in t.callback_metrics.items() if k in ("dev_wer", "dev_cer", "dev_nll")}) self.f.write(json.dumps(d) + "\n") say("DEV", {k: (round(v, 2) if isinstance(v, float) else v) for k, v in d.items() if k != "domains"}, {k: round(v["wer"], 1) for k, v in rep.items()}) def on_train_end(self, t, m): if self.f: self.f.close() class Backup(pl.Callback): fut = None def on_train_batch_end(self, t, m, *a): s = t.global_step if not s or s % C["save"]: return if self.fut is not None: self.fut.result(); self.fut = None tmp = R / "last.tmp.ckpt" t.save_checkpoint(str(tmp)) if t.is_global_zero: tmp.replace(R / "last.ckpt") if C["upload"]: try: self.fut = api.upload_file(path_or_fileobj=str(R / "last.ckpt"), path_in_repo="v4/last.ckpt", repo_id=C["repo"], run_as_future=True) except Exception as e: say("yedek yüklenemedi", repr(e)[:120]) def on_train_end(self, t, m): if self.fut is not None: self.fut.result() best = ModelCheckpoint(dirpath=str(R / "best"), filename="best-{step}-{dev_wer:.3f}", monitor="dev_wer", mode="min", save_top_k=C["topk"], save_weights_only=True, enable_version_counter=False) strategy = DDPStrategy(find_unused_parameters=False) if world > 1 else "auto" trainer = pl.Trainer( accelerator="gpu", devices=world, strategy=strategy, precision=PRECISION_STR, max_steps=C["steps"], max_epochs=-1, val_check_interval=C["val"], check_val_every_n_epoch=None, num_sanity_val_steps=0, gradient_clip_val=None, logger=False, enable_progress_bar=False, callbacks=[Report(), best, Backup()]) model.set_trainer(trainer) # Eğer önceki küçük batch'ten kalan last.ckpt varsa optimizer/LR zamanlayıcısı uyuşmazlığı olmaması için # sadece ağırlıklarını yükleyip yeni hızlı 8500 adımlık takvimle devam etmesini sağlıyoruz: last = R / "last.ckpt" ckpt_arg = None if last.exists(): try: ck = torch.load(str(last), map_location="cpu", weights_only=False) prev_step = int(ck.get("global_step", 0)) if prev_step > 0 and prev_step < C["steps"]: # Aynı takvimdeyse doğrudan devam et, eski 30k takvimindeyse ağırlıkları al model.load_state_dict(ck["state_dict"], strict=True) say(f"♻️ Önceki last.ckpt ağırlıkları yüklendi (Eski adım: {prev_step}).") del ck; gc.collect() except Exception as e: say("last.ckpt okuma bilgisi:", repr(e)[:120]) trainer.fit(model, train_dataloaders=train, val_dataloaders=dev, ckpt_path=ckpt_arg) barrier() shutil.rmtree(SCRATCH, ignore_errors=True) if world > 1: torch.distributed.destroy_process_group() if rank != 0: sys.exit(0) # =============================================================== yalnız rank 0: Greedy Soup, test, LM, ONNX SCRATCH.mkdir(parents=True, exist_ok=True) dev0 = torch.device("cuda:0") def infer(m, raw, bs=64): m = m.to(dev0).eval() order = sorted(range(len(raw)), key=lambda i: len(raw[i][0])) out = [None] * len(raw) with torch.inference_mode(), torch.autocast("cuda", dtype=AMP_DTYPE): for k in range(0, len(order), bs): idx = order[k: k + bs] ws = [raw[i][0] for i in idx] x = torch.zeros(len(ws), max(map(len, ws))) for j, w in enumerate(ws): x[j, : len(w)] = torch.from_numpy(w) lp, el, _ = m.forward(input_signal=x.to(dev0, non_blocking=True), input_signal_length=torch.tensor([len(w) for w in ws], device=dev0)) lp = lp.float().cpu().numpy() for j, i in enumerate(idx): out[i] = (lp[j, : int(el[j])].copy(), ref_clean(raw[i][1])) return out def greedy(lps): V_ = lps[0][0].shape[1] - 1 refs, hyps = [], [] for lp, ref in lps: a = lp.argmax(1) ids = [int(v) for k, v in enumerate(a) if v != V_ and (k == 0 or v != a[k - 1])] refs.append(ref); hyps.append(tok.ids_to_text(ids)) we, wn, ce, cn = lib.wer_counts(refs, hyps) return 100 * we / wn, 100 * ce / cn ckpts = sorted(glob.glob(str(R / "best" / "*.ckpt"))) say("🥣 Greedy Soup değerlendirmesi başlıyor, checkpoint sayısı:", len(ckpts)) sd_list = [torch.load(p, map_location="cpu", weights_only=False)["state_dict"] for p in ckpts] final_sd = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()} cands = {"last": final_sd} scores = {} model.load_state_dict(final_sd, strict=True) scores["last"] = greedy(infer(model, rawdev))[0] say("dev WER last:", round(scores["last"], 2)) if sd_list: wers = [float(re.search(r"dev_wer=(\d+\.\d+)", p).group(1)) if re.search(r"dev_wer=(\d+\.\d+)", p) else 1e9 for p in ckpts] order = np.argsort(wers) best_sd = sd_list[order[0]] cands["best_single"] = best_sd model.load_state_dict(best_sd, strict=True) scores["best_single"] = greedy(infer(model, rawdev))[0] say("dev WER best_single:", round(scores["best_single"], 2)) soup_pool = [best_sd] best_soup_wer = scores["best_single"] soup_sd = best_sd for idx in order[1:]: trial_pool = soup_pool + [sd_list[idx]] trial_sd = { k: (sum(sd[k].float() for sd in trial_pool) / len(trial_pool)).to(best_sd[k].dtype) if best_sd[k].is_floating_point() else best_sd[k] for k in best_sd } model.load_state_dict(trial_sd, strict=True) w_score = greedy(infer(model, rawdev))[0] if w_score < best_soup_wer: soup_pool = trial_pool best_soup_wer = w_score soup_sd = trial_sd say(f"greedy soup eklendi ({len(soup_pool)} model) -> dev WER: {w_score:.2f}") cands["soup"] = soup_sd scores["soup"] = best_soup_wer final_name = min(scores, key=scores.get) model.load_state_dict(cands[final_name], strict=True) say("🏆 Seçilen final model:", final_name, "| WER:", round(scores[final_name], 2)) model.save_to(str(R / "turkish_ctc.nemo")) upload(R / "turkish_ctc.nemo", "turkish_ctc.nemo") del sd_list, cands; gc.collect() rawtest = load_fleurs("test") lp_dev, lp_test = infer(model, rawdev), infer(model, rawtest) res = dict(selected=final_name, dev_scores=scores, dev_greedy_wer_cer=greedy(lp_dev), test_greedy_wer_cer=greedy(lp_test)) atomic_json(res, R / "results.json") say("FINAL FLEURS (greedy)", res) sp = tok.tokenizer (R / "tokenizer").mkdir(exist_ok=True) (R / "tokenizer" / "tokenizer.model").write_bytes(sp.serialized_model_proto()) def lse_norm(lp, T): z = lp / T z = z - z.max(1, keepdims=True) return (z - np.log(np.exp(z).sum(1, keepdims=True))).astype("float32") try: bindir = lib.ensure_kenlm(say) assert bindir, "KenLM ikilileri (lmplz) derlenemedi" corpus = Path("/tmp/lm_corpus.txt") extra = [Path(p) for p in C.get("extra_text", []) if Path(p).exists()] def lm_texts(args): s, p = args d = read(p, ["text", "duration"]) out = [] for t, dur in zip(d["text"], d["duration"]): c = lib.clean_text(t, lang_of(s)) if c and c not in blocked and dur and lib.label_ok(c, dur): out.append(lib.lm_line(sp, c)) del d return out jobs = [(s, p) for s in SOURCES for p in groups[s]] say(f"📚 KenLM 6-gram dil modeli için {len(jobs)} shard 12 kolla paralel taranıyor...") n_lines, j_done = 0, 0 with corpus.open("w", encoding="utf-8") as fo, ThreadPoolExecutor(12) as ex: for ch in ex.map(lm_texts, jobs): j_done += 1 if ch: fo.write("\n".join(ch) + "\n") n_lines += len(ch) if j_done % 25 == 0 or j_done == len(jobs): say(f" -> LM metin tarama: [{j_done}/{len(jobs)}] shard ({n_lines:,} satır)") for p in extra: for line in p.read_text(encoding="utf-8", errors="ignore").splitlines(): c = lib.clean_text(line, "tr") if c and c not in blocked: fo.write(lib.lm_line(sp, c) + "\n"); n_lines += 1 say("🔨 KenLM binary (24G RAM ile) derleniyor, toplam satır:", n_lines) lib.build_kenlm(corpus, R / "lm6.bin", bindir, order=C["lm_order"], mem=C["lm_mem"]) corpus.unlink(missing_ok=True) upload(R / "lm6.bin", "lm6.bin") dec = lib.LMDecoder(str(R / "lm6.bin"), V, alpha=0.5, beta=0.0, beam=C["beam"]) say("🔍 LM Decoder motoru:", dec.backend) def lm_wer(lps, alpha, beta, T): dec.set(alpha=alpha, beta=beta) refs, hyps = [], [] for lp, ref in lps: hyps.append(tok.ids_to_text(dec.decode(lse_norm(lp, T)))); refs.append(ref) we, wn, ce, cn = lib.wer_counts(refs, hyps) return 100 * we / wn, 100 * ce / cn grid = {} for alpha in (0.35, 0.55, 0.75): for beta in (-0.2, 0.0, 0.3): for T in (1.0, 1.25): key = f"{alpha}_{beta}_{T}" grid[key] = lm_wer(lp_dev, alpha, beta, T)[0] say("LM dev", key, round(grid[key], 2)) bk = min(grid, key=grid.get) ba, bb, bt = map(float, bk.split("_")) res.update(lm_backend=dec.backend, lm_grid=grid, lm_alpha=ba, lm_beta=bb, lm_T=bt, dev_lm_wer=grid[bk], test_lm_wer_cer=lm_wer(lp_test, ba, bb, bt)) atomic_json(res, R / "results.json") say("FINAL FLEURS (LM)", res) except Exception as e: res["lm_error"] = repr(e)[:300]; atomic_json(res, R / "results.json"); say("LM aşaması atlandı:", repr(e)[:300]) upload(R / "results.json", "results.json") shutil.rmtree(SCRATCH, ignore_errors=True) try: say("📦 ONNX (FP32 + INT8) CPU paketi dışa aktarılıyor...") import onnxruntime as ort from onnxruntime.quantization import quantize_dynamic, QuantType cpu = model.cpu().eval() fe = cpu.preprocessor.featurizer fb = fe.fb[0].detach().numpy().astype("float32"); win = fe.window.detach().numpy().astype("float32") kw = dict(n_fft=int(fe.n_fft), hop=int(fe.hop_length), preemph=float(getattr(fe, "preemph", 0.97) or 0)) onnx_fp, onnx_q = R / "turkish_ctc.onnx", R / "turkish_ctc.int8.onnx" cpu.export(str(onnx_fp)) quantize_dynamic(str(onnx_fp), str(onnx_q), weight_type=QuantType.QInt8) np.savez(R / "frontend.npz", fb=fb, window=win, **{k: np.array(v) for k, v in kw.items()}) w0 = rawdev[0][0] with torch.no_grad(): ref_f, ref_l = cpu.preprocessor(input_signal=torch.from_numpy(w0)[None], length=torch.tensor([len(w0)])) ref_f = ref_f[0, :, : int(ref_l[0])].numpy() parity = {} for pm in ("constant", "reflect"): f = lib.featurize_np(w0, fb, win, pad_mode=pm, **kw) parity[pm] = float(np.abs(f[:, : ref_f.shape[1]] - ref_f[:, : f.shape[1]]).max()) if f.shape[0] == ref_f.shape[0] else 1e9 pad_mode = min(parity, key=parity.get) info = dict(feature_parity_max_abs=parity, pad_mode=pad_mode) def run(path, threads, clips): so = ort.SessionOptions(); so.intra_op_num_threads = threads sess = ort.InferenceSession(str(path), so, providers=["CPUExecutionProvider"]) in_names = [i.name for i in sess.get_inputs()] refs, hyps, dur, t_used = [], [], 0.0, 0.0 for w, ref in clips: f = lib.featurize_np(w, fb, win, pad_mode=pad_mode, **kw) t0 = time.time() feed = {in_names[0]: f[None]} if len(in_names) > 1: feed[in_names[1]] = np.array([f.shape[1]], dtype="int64") o = sess.run(None, feed)[0][0] t_used += time.time() - t0 + 0.0 a = o.argmax(1) ids = [int(v) for k, v in enumerate(a) if v != V and (k == 0 or v != a[k - 1])] refs.append(ref_clean(ref)); hyps.append(tok.ids_to_text(ids)); dur += len(w) / 16000 we, wn, _, _ = lib.wer_counts(refs, hyps) return dict(wer=100 * we / wn, rtfx_model_only=dur / max(t_used, 1e-9)) clips = rawdev[:60] for name, p in (("fp32", onnx_fp), ("int8", onnx_q)): info[name] = dict(size_mb=round(p.stat().st_size / 2 ** 20, 1), threads1=run(p, 1, clips), threads4=run(p, 4, clips)) res["cpu_export"] = info; atomic_json(res, R / "results.json") say("CPU EXPORT", info) for p in (onnx_fp, onnx_q, R / "frontend.npz", R / "tokenizer" / "tokenizer.model"): upload(p, p.name) upload(R / "results.json", "results.json") except Exception as e: res["cpu_export_error"] = repr(e)[:300]; atomic_json(res, R / "results.json"); say("ONNX aşaması atlandı:", repr(e)[:300]) say("🏁 TÜM AŞAMALAR BİTTİ")