Instructions to use batuhan28/tr-fastconformer-stream with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- NeMo
How to use batuhan28/tr-fastconformer-stream with NeMo:
# tag did not correspond to a valid NeMo domain.
- Notebooks
- Google Colab
- Kaggle
Download v4/code/train.py from batuhan28/tr-fastconformer-stream: direct link, hf CLI and curl.
- Browser
- Download file 33.2 kB
-
https://huggingface.co/batuhan28/tr-fastconformer-stream/resolve/main/v4/code/train.py
- Command line
-
hf download hf://batuhan28/tr-fastconformer-stream/v4/code/train.py
-
curl -L -o train.py https://huggingface.co/batuhan28/tr-fastconformer-stream/resolve/main/v4/code/train.py
33.2 kB
| import os, io, re, json, time, math, pickle, sys, subprocess, shutil, datetime, glob, gc, uuid, warnings, logging | |
| from pathlib import Path | |
| from concurrent.futures import ThreadPoolExecutor | |
| warnings.filterwarnings("ignore") | |
| os.environ["HYDRA_FULL_ERROR"] = "0" | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| import soundfile as sf | |
| import sentencepiece as spm | |
| import pyarrow.parquet as pq | |
| from torch.utils.data import IterableDataset, DataLoader | |
| from huggingface_hub import HfApi, HfFileSystem | |
| import lightning.pytorch as pl | |
| from lightning.pytorch.strategies import DDPStrategy | |
| from lightning.pytorch.callbacks import ModelCheckpoint | |
| from nemo.collections.asr.models import EncDecCTCModelBPE | |
| from nemo.utils import logging as nemo_logging | |
| nemo_logging.setLevel(logging.ERROR) | |
| logging.getLogger("lightning.pytorch").setLevel(logging.ERROR) | |
| sys.path.insert(0, str(Path(__file__).parent)) | |
| import lib | |
| R = Path(__file__).parent | |
| STATUS = R / "status.txt" | |
| C = json.loads((R / "config.json").read_text()) | |
| rank = int(os.environ.get("RANK", 0)) | |
| world = int(os.environ.get("WORLD_SIZE", 1)) | |
| api = HfApi(token=os.environ["HF_TOKEN"]) | |
| fs = HfFileSystem(token=os.environ["HF_TOKEN"]) | |
| pl.seed_everything(42, workers=True) | |
| # ============================================================================== | |
| # ⚡ A100 EXTREME DONANIM BAYRAKLARI | |
| # ============================================================================== | |
| torch.set_num_threads(2) | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| torch.backends.cudnn.allow_tf32 = True | |
| torch.backends.cudnn.benchmark = True | |
| torch.set_float32_matmul_precision("high") | |
| USE_BF16 = bool(C.get("bf16", True) and torch.cuda.is_bf16_supported()) | |
| AMP_DTYPE = torch.bfloat16 if USE_BF16 else torch.float16 | |
| PRECISION_STR = "bf16-mixed" if USE_BF16 else "16-mixed" | |
| SOURCES = list(C["share"]) | |
| lang_of = lambda s: "en" if s == "librispeech" else "tr" | |
| SCRATCH = Path(f"/tmp/shard_scratch_r{rank}") | |
| SCRATCH.mkdir(parents=True, exist_ok=True) | |
| def say(*a): | |
| ts = time.strftime("%H:%M:%S") | |
| msg = " ".join(str(x) for x in a) | |
| print(f"[{ts} r{rank}] {msg}", flush=True) | |
| if rank == 0: | |
| try: | |
| STATUS.write_text(f"[{ts}] {msg}", encoding="utf-8") | |
| except Exception: | |
| pass | |
| def read(p, cols): | |
| """64MB Bloklarla Ultra-Hızlı İndir -> Çok Çekirdekli Arrow ile RAM'e Al -> Diskten Anında Sil.""" | |
| err = None | |
| for k in range(4): | |
| tmp_file = SCRATCH / f"{uuid.uuid4().hex}_{Path(p).name}" | |
| try: | |
| with fs.open(f"datasets/{C['data']}/{p}", "rb", block_size=64 * 1024 * 1024) as r_src, open(tmp_file, "wb") as w_dst: | |
| shutil.copyfileobj(r_src, w_dst, length=64 * 1024 * 1024) | |
| table = pq.read_table(tmp_file, columns=cols, use_threads=True) | |
| out = table.to_pydict() | |
| del table | |
| return out | |
| except Exception as e: | |
| err = e | |
| time.sleep(2 * (k + 1)) | |
| finally: | |
| if tmp_file.exists(): | |
| try: | |
| tmp_file.unlink() | |
| except OSError: | |
| pass | |
| raise RuntimeError(f"Shard okunamadı: {p}") from err | |
| def atomic_json(obj, path): | |
| path = Path(path); tmp = path.with_suffix(".tmp") | |
| tmp.write_text(json.dumps(obj, ensure_ascii=False)); tmp.replace(path) | |
| def upload(path, name): | |
| if C["upload"]: | |
| try: | |
| api.upload_file(path_or_fileobj=str(path), path_in_repo=f"v4/{name}", repo_id=C["repo"]) | |
| except Exception as e: | |
| say("yükleme başarısız:", name, repr(e)[:150]) | |
| say(f"🚀 EXTREME A100 Süreci Başlatıldı (Batch={C['seconds']}sn | Clips={C['max_clips']} | Workers={C['workers']} | {PRECISION_STR})...") | |
| if world > 1: | |
| torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) | |
| torch.distributed.init_process_group("nccl", timeout=datetime.timedelta(hours=3)) | |
| def barrier(): | |
| if world > 1: | |
| torch.distributed.barrier() | |
| # ---------------------------------------------------------------- 1) shard listesi | |
| if rank == 0 and not (R / "groups.json").exists(): | |
| say("📂 Adım 1/5: HuggingFace veri deposundaki Parquet shard listesi çekiliyor...") | |
| files = api.list_repo_files(C["data"], repo_type="dataset") | |
| allg = {s: sorted(p for p in files if p.startswith(f"data/{s}/") and p.endswith(".parquet")) for s in SOURCES} | |
| assert all(allg.values()), "Bir eğitim kaynağı eksik: " + str([s for s in allg if not allg[s]]) | |
| atomic_json({"train": {s: v[1:] for s, v in allg.items()}, "dev": {s: v[0] for s, v in allg.items()}}, | |
| R / "groups.json") | |
| say("✅ Shard listesi kaydedildi:", {s: len(v) for s, v in allg.items()}) | |
| barrier() | |
| G = json.loads((R / "groups.json").read_text()) | |
| groups, devshard = G["train"], G["dev"] | |
| # ---------------------------------------------------------------- 2) FLEURS dev/test | |
| def load_fleurs(split): | |
| dest = R / f"fleurs_{split}.pkl" | |
| if dest.exists(): | |
| return pickle.load(dest.open("rb")) | |
| say(f"🌍 Adım 2/5: Google FLEURS Türkçe '{split}' veri seti indiriliyor ve sesleri çözülüyor...") | |
| from datasets import load_dataset, Audio | |
| ds = None | |
| for kwargs in ( | |
| dict(path="google/fleurs", name="tr_tr", split=split, trust_remote_code=True), | |
| dict(path="google/fleurs", name="tr_tr", split=split), | |
| dict(path="xtreme_s", name="fleurs.tr_tr", split=split), | |
| ): | |
| try: | |
| ds = load_dataset(**kwargs) | |
| break | |
| except Exception: | |
| continue | |
| if ds is None: | |
| from huggingface_hub import hf_hub_download | |
| import tarfile, csv | |
| tsv_name = "dev.tsv" if split == "validation" else f"{split}.tsv" | |
| tar_name = "dev.tar.gz" if split == "validation" else f"{split}.tar.gz" | |
| tsv_p = hf_hub_download("google/fleurs", f"data/tr_tr/{tsv_name}", repo_type="dataset") | |
| tar_p = hf_hub_download("google/fleurs", f"data/tr_tr/audio/{tar_name}", repo_type="dataset") | |
| trans_map = {} | |
| with open(tsv_p, "r", encoding="utf-8") as f: | |
| for row in csv.reader(f, delimiter="\t"): | |
| if len(row) >= 3: | |
| trans_map[row[1].strip()] = row[2].strip() | |
| out = [] | |
| with tarfile.open(tar_p, "r:gz") as tar: | |
| members = [m for m in tar.getmembers() if m.isfile() and m.name.endswith(".wav")] | |
| for idx, m in enumerate(members, 1): | |
| fname = Path(m.name).name | |
| if fname in trans_map: | |
| raw = tar.extractfile(m).read() | |
| out.append((lib.decode_audio(raw), trans_map[fname])) | |
| if idx % 200 == 0 or idx == len(members): | |
| say(f" -> FLEURS {split} (doğrudan tar.gz) ses çözümü: {idx}/{len(members)}") | |
| with dest.open("wb") as f: | |
| pickle.dump(out, f) | |
| return out | |
| ds = ds.cast_column("audio", Audio(decode=False)) | |
| out = [] | |
| tot = len(ds) | |
| for idx, ex in enumerate(ds, 1): | |
| a = ex["audio"] | |
| raw = a.get("bytes") or Path(a["path"]).read_bytes() | |
| out.append((lib.decode_audio(raw), ex["transcription"])) | |
| if idx % 200 == 0 or idx == tot: | |
| say(f" -> FLEURS {split} ses çözümü: {idx}/{tot}") | |
| with dest.open("wb") as f: | |
| pickle.dump(out, f) | |
| shutil.rmtree(Path.home() / ".cache/huggingface/datasets", ignore_errors=True) | |
| return out | |
| if rank == 0: | |
| load_fleurs("validation") | |
| atomic_json([t for _, t in load_fleurs("test")], R / "fleurs_test_refs.json") | |
| barrier() | |
| rawdev = load_fleurs("validation") | |
| rawtest_refs = json.loads((R / "fleurs_test_refs.json").read_text()) | |
| ref_clean = lambda t: lib.clean_text(t, "tr", strict=False) | |
| blocked = {ref_clean(t) for _, t in rawdev} | {ref_clean(t) for t in rawtest_refs} | |
| del rawtest_refs | |
| # ---------------------------------------------------------------- 3) tokenizer (BPE) + başlangıç modeli | |
| if rank == 0 and not (R / "init.nemo").exists(): | |
| tokdir = R / "tokenizer"; tokdir.mkdir(exist_ok=True) | |
| rng = np.random.default_rng(0) | |
| per_source = {s: [] for s in SOURCES} | |
| jobs = [] | |
| for s in SOURCES: | |
| paths = groups[s] | |
| pick = [paths[i] for i in np.linspace(0, len(paths) - 1, min(8, len(paths))).astype(int)] | |
| for p in pick: | |
| jobs.append((s, p)) | |
| say(f"🔤 Adım 3/5: Tokenizer eğitimi için {len(jobs)} shard'dan 12 paralel kolla metin örnekleniyor...") | |
| def fetch_tok_texts(item): | |
| s, p = item | |
| d = read(p, ["text"]) | |
| out = [t for t in (lib.clean_text(x, lang_of(s)) for x in d["text"]) if t and t not in blocked] | |
| return s, p, out | |
| done_cnt = 0 | |
| with ThreadPoolExecutor(12) as ex: | |
| for s, p, out in ex.map(fetch_tok_texts, jobs): | |
| per_source[s].extend(out) | |
| done_cnt += 1 | |
| if done_cnt % 4 == 0 or done_cnt == len(jobs): | |
| say(f" -> Tokenizer metin indirme ilerlemesi: [{done_cnt}/{len(jobs)}] shard tamamlandı ({s})") | |
| lines = [] | |
| for s in SOURCES: | |
| n = min(len(per_source[s]), int(C["share"][s] * 2_000_000)) | |
| lines += list(rng.choice(per_source[s], size=n, replace=False)) if n < len(per_source[s]) else per_source[s] | |
| assert lines, "Tokenizer metni yok." | |
| say(f"🔤 SentencePiece BPE ({C['vocab']} token) toplam {len(lines):,} cümle üzerinde eğitiliyor...") | |
| (tokdir / "text.txt").write_text("\n".join(lines), encoding="utf-8") | |
| del per_source, lines; gc.collect() | |
| spm.SentencePieceTrainer.train( | |
| input=str(tokdir / "text.txt"), model_prefix=str(tokdir / "tokenizer"), model_type="bpe", | |
| vocab_size=C["vocab"], character_coverage=1.0, normalization_rule_name="identity", | |
| bos_id=-1, eos_id=-1, pad_id=-1, hard_vocab_limit=False, num_threads=C["cpus"], | |
| required_chars="abcçdefgğhıijklmnoöprsştuüvyzqwx", input_sentence_size=2_000_000, | |
| shuffle_input_sentence=True, minloglevel=2) | |
| (tokdir / "text.txt").unlink(missing_ok=True) | |
| sp0 = spm.SentencePieceProcessor(model_file=str(tokdir / "tokenizer.model")) | |
| (tokdir / "vocab.txt").write_text("\n".join(sp0.id_to_piece(i) for i in range(sp0.vocab_size()))) | |
| say(f"📦 Taban model ({C['base']}) indiriliyor ve Türkçe BPE sözlüğü enjekte ediliyor...") | |
| m = EncDecCTCModelBPE.from_pretrained(C["base"], map_location="cpu") | |
| m.change_vocabulary(new_tokenizer_dir=str(tokdir), new_tokenizer_type="bpe") | |
| m.save_to(str(R / "init.nemo")); del m | |
| say("☁️ init.nemo yedekleniyor...") | |
| upload(R / "init.nemo", "init.nemo") | |
| barrier() | |
| # ---------------------------------------------------------------- 4) model (2400s Dev Batch + OOM Korumalı Forward + FusedAdamW) | |
| say("🧠 Adım 4/5: init.nemo belleğe yükleniyor...") | |
| class ASR(EncDecCTCModelBPE): | |
| group_names = ["fleurs"] + SOURCES + ["fleurs_rest"] | |
| def configure_optimizers(self): | |
| if USE_BF16: | |
| opt = torch.optim.AdamW(self.parameters(), lr=C["lr"], betas=(0.9, 0.98), weight_decay=C["wd"], fused=True) | |
| else: | |
| opt = torch.optim.AdamW(self.parameters(), lr=C["lr"], betas=(0.9, 0.98), weight_decay=C["wd"], foreach=True) | |
| total, warm = C["steps"], C["warmup"] | |
| decay = int(total * 0.25) | |
| def f(s): | |
| if s < warm: | |
| return (s + 1) / warm | |
| if s < total - decay: | |
| return 1.0 | |
| return max(0.05, 1.0 - 0.95 * (s - (total - decay)) / decay) | |
| return {"optimizer": opt, "lr_scheduler": {"scheduler": torch.optim.lr_scheduler.LambdaLR(opt, f), | |
| "interval": "step"}} | |
| def on_before_optimizer_step(self, optimizer): | |
| torch.nn.utils.clip_grad_norm_(self.parameters(), max_norm=1.0) | |
| def _chunk_loss(self, x, xl, y, yl): | |
| proc_x, proc_xl = self.preprocessor(input_signal=x, length=xl) | |
| if self.spec_augmentation is not None and self.training: | |
| proc_x = self.spec_augmentation(input_spec=proc_x, length=proc_xl) | |
| encoded, el = self.encoder(audio_signal=proc_x, length=proc_xl) | |
| lp = self.decoder(encoder_output=encoded) | |
| loss = F.ctc_loss(lp.float().transpose(0, 1), y, el.long(), yl, blank=lp.shape[-1] - 1, | |
| reduction="none", zero_infinity=True) | |
| return loss.sum() | |
| def training_step(self, batch, batch_idx): | |
| x, xl, y, yl = batch | |
| tot_tokens = yl.sum().clamp(min=1) | |
| try: | |
| return self._chunk_loss(x, xl, y, yl) / tot_tokens | |
| except torch.cuda.OutOfMemoryError: | |
| # Eğer 2400 sn'lik ekstrem bir batch 40GB sınırını aşarsa çökmez, ikiye bölüp işler! | |
| torch.cuda.empty_cache() | |
| mid = max(1, x.shape[0] // 2) | |
| l1 = self._chunk_loss(x[:mid], xl[:mid], y[:mid], yl[:mid]) | |
| l2 = self._chunk_loss(x[mid:], xl[mid:], y[mid:], yl[mid:]) | |
| return (l1 + l2) / tot_tokens | |
| def on_validation_epoch_start(self): | |
| self._acc = torch.zeros(len(self.group_names), 6, device=self.device) | |
| self.full_pass = self.global_step % C["full_every"] == 0 | |
| def validation_step(self, batch, batch_idx, dataloader_idx=0): | |
| if dataloader_idx == 2 and not self.full_pass: | |
| return | |
| x, xl, y, yl, g, refs = batch | |
| lp, el, _ = self.forward(input_signal=x, input_signal_length=xl) | |
| lpf = lp.float(); V = lpf.shape[-1] - 1 | |
| nll = F.ctc_loss(lpf.transpose(0, 1), y, el.long(), yl, blank=V, reduction="none", zero_infinity=True) | |
| pred = lpf.argmax(-1).cpu(); elc = el.cpu(); gl = g.tolist() | |
| for i in range(len(refs)): | |
| z = pred[i, : int(elc[i])].unique_consecutive().tolist() | |
| hyp = self.tokenizer.ids_to_text([v for v in z if v != V]) | |
| we, wn, ce, cn = lib.wer_counts([refs[i]], [hyp]) | |
| self._acc[gl[i]] += torch.tensor([we, wn, ce, cn, float(nll[i]), float(yl[i])], device=self.device) | |
| def on_validation_epoch_end(self): | |
| acc = self._acc | |
| if world > 1: | |
| torch.distributed.all_reduce(acc) | |
| rep = {} | |
| for k, name in enumerate(self.group_names): | |
| we, wn, ce, cn, nl, nt = acc[k].tolist() | |
| if wn > 0: | |
| rep[name] = dict(wer=100 * we / wn, cer=100 * ce / max(cn, 1), nll=nl / max(nt, 1)) | |
| if self.full_pass and "fleurs" in rep and "fleurs_rest" in rep: | |
| a, b = acc[0], acc[len(self.group_names) - 1] | |
| s_ = (a + b).tolist() | |
| rep["fleurs_full"] = dict(wer=100 * s_[0] / max(s_[1], 1), cer=100 * s_[2] / max(s_[3], 1), | |
| nll=s_[4] / max(s_[5], 1)) | |
| self.domain_report = rep | |
| f = rep.get("fleurs", dict(wer=100.0, cer=100.0, nll=99.0)) | |
| for key in ("wer", "cer", "nll"): | |
| self.log(f"dev_{key}", float(f[key]), sync_dist=(world > 1), rank_zero_only=False, batch_size=1) | |
| model = ASR.restore_from(str(R / "init.nemo"), map_location="cpu") | |
| tok = model.tokenizer | |
| if hasattr(model.encoder, "sync_max_audio_length"): | |
| model.encoder.sync_max_audio_length = False | |
| V = tok.vocab_size | |
| say("✅ Model yüklendi -> Vokabüler:", V, "| Parametre (M):", round(sum(p.numel() for p in model.parameters()) / 1e6, 1)) | |
| # ---------------------------------------------------------------- 5) veri | |
| def pack(items): | |
| ws, ids = zip(*items) | |
| x = torch.zeros(len(items), max(map(len, ws))) | |
| y = torch.zeros(len(items), max(map(len, ids)), dtype=torch.long) | |
| for i, (w, t) in enumerate(items): | |
| x[i, : len(w)] = torch.from_numpy(w.astype("float32") / 32767.0) if w.dtype == np.int16 else torch.from_numpy(w) | |
| y[i, : len(t)] = torch.tensor(t) | |
| return x, torch.tensor(list(map(len, ws))), y, torch.tensor(list(map(len, ids))) | |
| class Stream(IterableDataset): | |
| def __iter__(self): | |
| wi = torch.utils.data.get_worker_info() | |
| wid, nw = (wi.id, wi.num_workers) if wi else (0, 1) | |
| worker = rank * nw + wid | |
| rng = np.random.default_rng(42 + 1000 * C["launch"] + worker) | |
| cols = ["audio", "text", "duration"] | |
| for b in lib.batch_stream(groups, C["share"], C["hours"], worker, world * nw, rng, | |
| lambda p: read(p, cols), tok.text_to_ids, blocked, C["seconds"], C["max_clips"], | |
| lang_of=lang_of, tel_p=C["tel_p"], edge_p=C["edge_p"], | |
| min_buffer=C.get("min_buffer", 3200), | |
| ram_cache_shards=C.get("ram_cache_shards", 4), | |
| status_fn=say if worker == 0 else None): | |
| yield pack(b) | |
| def dev_items(raw, g): | |
| out = [] | |
| for w, t in raw: | |
| ref = ref_clean(t) | |
| ids = tok.text_to_ids(ref) | |
| if ids: | |
| out.append((w, ids, ref, g)) | |
| out.sort(key=lambda z: len(z[0])) | |
| return out | |
| def pack_dev(items): | |
| x, xl, y, yl = pack([(w, ids) for w, ids, _, _ in items]) | |
| return x, xl, y, yl, torch.tensor([g for *_, g in items]), [r for _, _, r, _ in items] | |
| if rank == 0 and not (R / "domain_dev.pkl").exists(): | |
| out = [] | |
| for idx_s, s in enumerate(SOURCES, 1): | |
| say(f"📊 Adım 5/5: Kaynak bazlı doğrulama kümesi hazırlanıyor [{idx_s}/{len(SOURCES)}]: {s}...") | |
| d = read(devshard[s], ["audio", "text", "duration"]) | |
| rows = [] | |
| for a, t, dur in zip(d["audio"], d["text"], d["duration"]): | |
| text = lib.clean_text(t, lang_of(s)) | |
| if a and a.get("bytes") and dur and 1 <= dur <= 20 and text and lib.label_ok(text, dur) and text not in blocked: | |
| rows.append((a["bytes"], text)) | |
| pick = np.random.default_rng(0).permutation(len(rows))[: C["dev_per_source"]] | |
| out += [(lib.decode_audio(rows[i][0]), rows[i][1], s) for i in pick] | |
| del d, rows; gc.collect() | |
| with (R / "domain_dev.pkl").open("wb") as f: | |
| pickle.dump(out, f) | |
| barrier() | |
| dom_raw = pickle.load((R / "domain_dev.pkl").open("rb")) | |
| dom = [] | |
| for w, text, s in dom_raw: | |
| ids = tok.text_to_ids(text) | |
| if ids: | |
| dom.append((w, ids, text, 1 + SOURCES.index(s))) | |
| dom.sort(key=lambda z: len(z[0])) | |
| del dom_raw | |
| train = DataLoader(Stream(), batch_size=None, num_workers=C["workers"], prefetch_factor=C.get("prefetch", 6), | |
| pin_memory=True, persistent_workers=(C["workers"] > 0)) | |
| fl_all = dev_items(rawdev, 0) | |
| sel = set(lib.pick_representative([len(w) / 16000 for w, *_ in fl_all], [len(r.split()) for _, _, r, _ in fl_all], | |
| [len(r) for _, _, r, _ in fl_all], C["dev_subset"])) | |
| fl_sub = [it for i, it in enumerate(fl_all) if i in sel] | |
| fl_rest = [(w, ids, r, 1 + len(SOURCES)) for i, (w, ids, r, _) in enumerate(fl_all) if i not in sel] | |
| say("🎯 Hazırlık tamamlandı! FLEURS dev:", len(fl_all), "| Alt küme:", len(fl_sub), "-> A100 Extreme Eğitimi başlıyor...") | |
| dev = [DataLoader(fl_sub, batch_size=64, collate_fn=pack_dev, pin_memory=True), | |
| DataLoader(dom, batch_size=64, collate_fn=pack_dev, pin_memory=True), | |
| DataLoader(fl_rest, batch_size=64, collate_fn=pack_dev, pin_memory=True)] | |
| # ---------------------------------------------------------------- 6) callback'ler | |
| class Report(pl.Callback): | |
| def on_train_start(self, t, m): | |
| self.f = (R / "metrics.jsonl").open("a", buffering=1) if t.is_global_zero else None | |
| if t.is_global_zero: | |
| say(f"🔥 A100 EXTREME ({PRECISION_STR} | Batch={C['seconds']}sn) eğitimi aktif!") | |
| def on_train_batch_end(self, t, m, outputs, batch, batch_idx): | |
| if self.f and (t.global_step == 1 or t.global_step % 5 == 0): | |
| loss = outputs["loss"] if isinstance(outputs, dict) else outputs | |
| lr = t.optimizers[0].param_groups[0]["lr"] | |
| self.f.write(json.dumps(dict(step=t.global_step, loss=float(loss.detach()), lr=lr, time=time.time())) + "\n") | |
| if t.global_step <= 15 or t.global_step % 25 == 0: | |
| say(f"⚡ Adım {t.global_step:,}/{C['steps']:,} | Loss: {float(loss.detach()):.3f} | LR: {lr:.2e} | Klip/Batch: {batch[0].shape[0]}") | |
| def on_validation_end(self, t, m): | |
| if t.sanity_checking or not t.is_global_zero or not self.f: | |
| return | |
| rep = getattr(m, "domain_report", {}) | |
| d = dict(step=t.global_step, time=time.time(), domains=rep) | |
| d.update({k: float(v) for k, v in t.callback_metrics.items() if k in ("dev_wer", "dev_cer", "dev_nll")}) | |
| self.f.write(json.dumps(d) + "\n") | |
| say("DEV", {k: (round(v, 2) if isinstance(v, float) else v) for k, v in d.items() if k != "domains"}, | |
| {k: round(v["wer"], 1) for k, v in rep.items()}) | |
| def on_train_end(self, t, m): | |
| if self.f: | |
| self.f.close() | |
| class Backup(pl.Callback): | |
| fut = None | |
| def on_train_batch_end(self, t, m, *a): | |
| s = t.global_step | |
| if not s or s % C["save"]: | |
| return | |
| if self.fut is not None: | |
| self.fut.result(); self.fut = None | |
| tmp = R / "last.tmp.ckpt" | |
| t.save_checkpoint(str(tmp)) | |
| if t.is_global_zero: | |
| tmp.replace(R / "last.ckpt") | |
| if C["upload"]: | |
| try: | |
| self.fut = api.upload_file(path_or_fileobj=str(R / "last.ckpt"), path_in_repo="v4/last.ckpt", | |
| repo_id=C["repo"], run_as_future=True) | |
| except Exception as e: | |
| say("yedek yüklenemedi", repr(e)[:120]) | |
| def on_train_end(self, t, m): | |
| if self.fut is not None: | |
| self.fut.result() | |
| best = ModelCheckpoint(dirpath=str(R / "best"), filename="best-{step}-{dev_wer:.3f}", monitor="dev_wer", mode="min", | |
| save_top_k=C["topk"], save_weights_only=True, enable_version_counter=False) | |
| strategy = DDPStrategy(find_unused_parameters=False) if world > 1 else "auto" | |
| trainer = pl.Trainer( | |
| accelerator="gpu", devices=world, strategy=strategy, | |
| precision=PRECISION_STR, max_steps=C["steps"], max_epochs=-1, val_check_interval=C["val"], | |
| check_val_every_n_epoch=None, num_sanity_val_steps=0, gradient_clip_val=None, logger=False, | |
| enable_progress_bar=False, callbacks=[Report(), best, Backup()]) | |
| model.set_trainer(trainer) | |
| # Eğer önceki küçük batch'ten kalan last.ckpt varsa optimizer/LR zamanlayıcısı uyuşmazlığı olmaması için | |
| # sadece ağırlıklarını yükleyip yeni hızlı 8500 adımlık takvimle devam etmesini sağlıyoruz: | |
| last = R / "last.ckpt" | |
| ckpt_arg = None | |
| if last.exists(): | |
| try: | |
| ck = torch.load(str(last), map_location="cpu", weights_only=False) | |
| prev_step = int(ck.get("global_step", 0)) | |
| if prev_step > 0 and prev_step < C["steps"]: | |
| # Aynı takvimdeyse doğrudan devam et, eski 30k takvimindeyse ağırlıkları al | |
| model.load_state_dict(ck["state_dict"], strict=True) | |
| say(f"♻️ Önceki last.ckpt ağırlıkları yüklendi (Eski adım: {prev_step}).") | |
| del ck; gc.collect() | |
| except Exception as e: | |
| say("last.ckpt okuma bilgisi:", repr(e)[:120]) | |
| trainer.fit(model, train_dataloaders=train, val_dataloaders=dev, ckpt_path=ckpt_arg) | |
| barrier() | |
| shutil.rmtree(SCRATCH, ignore_errors=True) | |
| if world > 1: | |
| torch.distributed.destroy_process_group() | |
| if rank != 0: | |
| sys.exit(0) | |
| # =============================================================== yalnız rank 0: Greedy Soup, test, LM, ONNX | |
| SCRATCH.mkdir(parents=True, exist_ok=True) | |
| dev0 = torch.device("cuda:0") | |
| def infer(m, raw, bs=64): | |
| m = m.to(dev0).eval() | |
| order = sorted(range(len(raw)), key=lambda i: len(raw[i][0])) | |
| out = [None] * len(raw) | |
| with torch.inference_mode(), torch.autocast("cuda", dtype=AMP_DTYPE): | |
| for k in range(0, len(order), bs): | |
| idx = order[k: k + bs] | |
| ws = [raw[i][0] for i in idx] | |
| x = torch.zeros(len(ws), max(map(len, ws))) | |
| for j, w in enumerate(ws): | |
| x[j, : len(w)] = torch.from_numpy(w) | |
| lp, el, _ = m.forward(input_signal=x.to(dev0, non_blocking=True), | |
| input_signal_length=torch.tensor([len(w) for w in ws], device=dev0)) | |
| lp = lp.float().cpu().numpy() | |
| for j, i in enumerate(idx): | |
| out[i] = (lp[j, : int(el[j])].copy(), ref_clean(raw[i][1])) | |
| return out | |
| def greedy(lps): | |
| V_ = lps[0][0].shape[1] - 1 | |
| refs, hyps = [], [] | |
| for lp, ref in lps: | |
| a = lp.argmax(1) | |
| ids = [int(v) for k, v in enumerate(a) if v != V_ and (k == 0 or v != a[k - 1])] | |
| refs.append(ref); hyps.append(tok.ids_to_text(ids)) | |
| we, wn, ce, cn = lib.wer_counts(refs, hyps) | |
| return 100 * we / wn, 100 * ce / cn | |
| ckpts = sorted(glob.glob(str(R / "best" / "*.ckpt"))) | |
| say("🥣 Greedy Soup değerlendirmesi başlıyor, checkpoint sayısı:", len(ckpts)) | |
| sd_list = [torch.load(p, map_location="cpu", weights_only=False)["state_dict"] for p in ckpts] | |
| final_sd = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()} | |
| cands = {"last": final_sd} | |
| scores = {} | |
| model.load_state_dict(final_sd, strict=True) | |
| scores["last"] = greedy(infer(model, rawdev))[0] | |
| say("dev WER last:", round(scores["last"], 2)) | |
| if sd_list: | |
| wers = [float(re.search(r"dev_wer=(\d+\.\d+)", p).group(1)) if re.search(r"dev_wer=(\d+\.\d+)", p) else 1e9 for p in ckpts] | |
| order = np.argsort(wers) | |
| best_sd = sd_list[order[0]] | |
| cands["best_single"] = best_sd | |
| model.load_state_dict(best_sd, strict=True) | |
| scores["best_single"] = greedy(infer(model, rawdev))[0] | |
| say("dev WER best_single:", round(scores["best_single"], 2)) | |
| soup_pool = [best_sd] | |
| best_soup_wer = scores["best_single"] | |
| soup_sd = best_sd | |
| for idx in order[1:]: | |
| trial_pool = soup_pool + [sd_list[idx]] | |
| trial_sd = { | |
| k: (sum(sd[k].float() for sd in trial_pool) / len(trial_pool)).to(best_sd[k].dtype) | |
| if best_sd[k].is_floating_point() else best_sd[k] | |
| for k in best_sd | |
| } | |
| model.load_state_dict(trial_sd, strict=True) | |
| w_score = greedy(infer(model, rawdev))[0] | |
| if w_score < best_soup_wer: | |
| soup_pool = trial_pool | |
| best_soup_wer = w_score | |
| soup_sd = trial_sd | |
| say(f"greedy soup eklendi ({len(soup_pool)} model) -> dev WER: {w_score:.2f}") | |
| cands["soup"] = soup_sd | |
| scores["soup"] = best_soup_wer | |
| final_name = min(scores, key=scores.get) | |
| model.load_state_dict(cands[final_name], strict=True) | |
| say("🏆 Seçilen final model:", final_name, "| WER:", round(scores[final_name], 2)) | |
| model.save_to(str(R / "turkish_ctc.nemo")) | |
| upload(R / "turkish_ctc.nemo", "turkish_ctc.nemo") | |
| del sd_list, cands; gc.collect() | |
| rawtest = load_fleurs("test") | |
| lp_dev, lp_test = infer(model, rawdev), infer(model, rawtest) | |
| res = dict(selected=final_name, dev_scores=scores, | |
| dev_greedy_wer_cer=greedy(lp_dev), test_greedy_wer_cer=greedy(lp_test)) | |
| atomic_json(res, R / "results.json") | |
| say("FINAL FLEURS (greedy)", res) | |
| sp = tok.tokenizer | |
| (R / "tokenizer").mkdir(exist_ok=True) | |
| (R / "tokenizer" / "tokenizer.model").write_bytes(sp.serialized_model_proto()) | |
| def lse_norm(lp, T): | |
| z = lp / T | |
| z = z - z.max(1, keepdims=True) | |
| return (z - np.log(np.exp(z).sum(1, keepdims=True))).astype("float32") | |
| try: | |
| bindir = lib.ensure_kenlm(say) | |
| assert bindir, "KenLM ikilileri (lmplz) derlenemedi" | |
| corpus = Path("/tmp/lm_corpus.txt") | |
| extra = [Path(p) for p in C.get("extra_text", []) if Path(p).exists()] | |
| def lm_texts(args): | |
| s, p = args | |
| d = read(p, ["text", "duration"]) | |
| out = [] | |
| for t, dur in zip(d["text"], d["duration"]): | |
| c = lib.clean_text(t, lang_of(s)) | |
| if c and c not in blocked and dur and lib.label_ok(c, dur): | |
| out.append(lib.lm_line(sp, c)) | |
| del d | |
| return out | |
| jobs = [(s, p) for s in SOURCES for p in groups[s]] | |
| say(f"📚 KenLM 6-gram dil modeli için {len(jobs)} shard 12 kolla paralel taranıyor...") | |
| n_lines, j_done = 0, 0 | |
| with corpus.open("w", encoding="utf-8") as fo, ThreadPoolExecutor(12) as ex: | |
| for ch in ex.map(lm_texts, jobs): | |
| j_done += 1 | |
| if ch: | |
| fo.write("\n".join(ch) + "\n") | |
| n_lines += len(ch) | |
| if j_done % 25 == 0 or j_done == len(jobs): | |
| say(f" -> LM metin tarama: [{j_done}/{len(jobs)}] shard ({n_lines:,} satır)") | |
| for p in extra: | |
| for line in p.read_text(encoding="utf-8", errors="ignore").splitlines(): | |
| c = lib.clean_text(line, "tr") | |
| if c and c not in blocked: | |
| fo.write(lib.lm_line(sp, c) + "\n"); n_lines += 1 | |
| say("🔨 KenLM binary (24G RAM ile) derleniyor, toplam satır:", n_lines) | |
| lib.build_kenlm(corpus, R / "lm6.bin", bindir, order=C["lm_order"], mem=C["lm_mem"]) | |
| corpus.unlink(missing_ok=True) | |
| upload(R / "lm6.bin", "lm6.bin") | |
| dec = lib.LMDecoder(str(R / "lm6.bin"), V, alpha=0.5, beta=0.0, beam=C["beam"]) | |
| say("🔍 LM Decoder motoru:", dec.backend) | |
| def lm_wer(lps, alpha, beta, T): | |
| dec.set(alpha=alpha, beta=beta) | |
| refs, hyps = [], [] | |
| for lp, ref in lps: | |
| hyps.append(tok.ids_to_text(dec.decode(lse_norm(lp, T)))); refs.append(ref) | |
| we, wn, ce, cn = lib.wer_counts(refs, hyps) | |
| return 100 * we / wn, 100 * ce / cn | |
| grid = {} | |
| for alpha in (0.35, 0.55, 0.75): | |
| for beta in (-0.2, 0.0, 0.3): | |
| for T in (1.0, 1.25): | |
| key = f"{alpha}_{beta}_{T}" | |
| grid[key] = lm_wer(lp_dev, alpha, beta, T)[0] | |
| say("LM dev", key, round(grid[key], 2)) | |
| bk = min(grid, key=grid.get) | |
| ba, bb, bt = map(float, bk.split("_")) | |
| res.update(lm_backend=dec.backend, lm_grid=grid, lm_alpha=ba, lm_beta=bb, lm_T=bt, | |
| dev_lm_wer=grid[bk], test_lm_wer_cer=lm_wer(lp_test, ba, bb, bt)) | |
| atomic_json(res, R / "results.json") | |
| say("FINAL FLEURS (LM)", res) | |
| except Exception as e: | |
| res["lm_error"] = repr(e)[:300]; atomic_json(res, R / "results.json"); say("LM aşaması atlandı:", repr(e)[:300]) | |
| upload(R / "results.json", "results.json") | |
| shutil.rmtree(SCRATCH, ignore_errors=True) | |
| try: | |
| say("📦 ONNX (FP32 + INT8) CPU paketi dışa aktarılıyor...") | |
| import onnxruntime as ort | |
| from onnxruntime.quantization import quantize_dynamic, QuantType | |
| cpu = model.cpu().eval() | |
| fe = cpu.preprocessor.featurizer | |
| fb = fe.fb[0].detach().numpy().astype("float32"); win = fe.window.detach().numpy().astype("float32") | |
| kw = dict(n_fft=int(fe.n_fft), hop=int(fe.hop_length), preemph=float(getattr(fe, "preemph", 0.97) or 0)) | |
| onnx_fp, onnx_q = R / "turkish_ctc.onnx", R / "turkish_ctc.int8.onnx" | |
| cpu.export(str(onnx_fp)) | |
| quantize_dynamic(str(onnx_fp), str(onnx_q), weight_type=QuantType.QInt8) | |
| np.savez(R / "frontend.npz", fb=fb, window=win, **{k: np.array(v) for k, v in kw.items()}) | |
| w0 = rawdev[0][0] | |
| with torch.no_grad(): | |
| ref_f, ref_l = cpu.preprocessor(input_signal=torch.from_numpy(w0)[None], length=torch.tensor([len(w0)])) | |
| ref_f = ref_f[0, :, : int(ref_l[0])].numpy() | |
| parity = {} | |
| for pm in ("constant", "reflect"): | |
| f = lib.featurize_np(w0, fb, win, pad_mode=pm, **kw) | |
| parity[pm] = float(np.abs(f[:, : ref_f.shape[1]] - ref_f[:, : f.shape[1]]).max()) if f.shape[0] == ref_f.shape[0] else 1e9 | |
| pad_mode = min(parity, key=parity.get) | |
| info = dict(feature_parity_max_abs=parity, pad_mode=pad_mode) | |
| def run(path, threads, clips): | |
| so = ort.SessionOptions(); so.intra_op_num_threads = threads | |
| sess = ort.InferenceSession(str(path), so, providers=["CPUExecutionProvider"]) | |
| in_names = [i.name for i in sess.get_inputs()] | |
| refs, hyps, dur, t_used = [], [], 0.0, 0.0 | |
| for w, ref in clips: | |
| f = lib.featurize_np(w, fb, win, pad_mode=pad_mode, **kw) | |
| t0 = time.time() | |
| feed = {in_names[0]: f[None]} | |
| if len(in_names) > 1: | |
| feed[in_names[1]] = np.array([f.shape[1]], dtype="int64") | |
| o = sess.run(None, feed)[0][0] | |
| t_used += time.time() - t0 + 0.0 | |
| a = o.argmax(1) | |
| ids = [int(v) for k, v in enumerate(a) if v != V and (k == 0 or v != a[k - 1])] | |
| refs.append(ref_clean(ref)); hyps.append(tok.ids_to_text(ids)); dur += len(w) / 16000 | |
| we, wn, _, _ = lib.wer_counts(refs, hyps) | |
| return dict(wer=100 * we / wn, rtfx_model_only=dur / max(t_used, 1e-9)) | |
| clips = rawdev[:60] | |
| for name, p in (("fp32", onnx_fp), ("int8", onnx_q)): | |
| info[name] = dict(size_mb=round(p.stat().st_size / 2 ** 20, 1), | |
| threads1=run(p, 1, clips), threads4=run(p, 4, clips)) | |
| res["cpu_export"] = info; atomic_json(res, R / "results.json") | |
| say("CPU EXPORT", info) | |
| for p in (onnx_fp, onnx_q, R / "frontend.npz", R / "tokenizer" / "tokenizer.model"): | |
| upload(p, p.name) | |
| upload(R / "results.json", "results.json") | |
| except Exception as e: | |
| res["cpu_export_error"] = repr(e)[:300]; atomic_json(res, R / "results.json"); say("ONNX aşaması atlandı:", repr(e)[:300]) | |
| say("🏁 TÜM AŞAMALAR BİTTİ") | |