batuhan28 commited on
Commit
350b3ee
·
verified ·
1 Parent(s): 4338ccf

Upload v4/code/train.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. v4/code/train.py +15 -7
v4/code/train.py CHANGED
@@ -1,6 +1,11 @@
1
- import os, io, re, json, time, math, pickle, sys, subprocess, shutil, datetime, glob, gc, uuid, traceback
2
  from pathlib import Path
3
  from concurrent.futures import ThreadPoolExecutor
 
 
 
 
 
4
  import numpy as np
5
  import torch
6
  import torch.nn.functional as F
@@ -13,6 +18,10 @@ import lightning.pytorch as pl
13
  from lightning.pytorch.strategies import DDPStrategy
14
  from lightning.pytorch.callbacks import ModelCheckpoint
15
  from nemo.collections.asr.models import EncDecCTCModelBPE
 
 
 
 
16
 
17
  sys.path.insert(0, str(Path(__file__).parent))
18
  import lib
@@ -230,13 +239,12 @@ if rank == 0 and not (R / "init.nemo").exists():
230
  barrier()
231
 
232
 
233
- # ---------------------------------------------------------------- 4) model (FP16 GradClip ile tam uyumlu AdamW)
234
  say("🧠 Adım 4/5: init.nemo belleğe yükleniyor...")
235
  class ASR(EncDecCTCModelBPE):
236
  group_names = ["fleurs"] + SOURCES + ["fleurs_rest"]
237
 
238
  def configure_optimizers(self):
239
- # NOT: precision="16-mixed" + gradient_clip_val=1.0 kullanılırken fused=False olmalıdır!
240
  opt = torch.optim.AdamW(self.parameters(), lr=C["lr"], betas=(0.9, 0.98), weight_decay=C["wd"], foreach=True)
241
  total, warm = C["steps"], C["warmup"]
242
  decay = int(total * 0.25)
@@ -296,7 +304,7 @@ class ASR(EncDecCTCModelBPE):
296
  self.domain_report = rep
297
  f = rep.get("fleurs", dict(wer=100.0, cer=100.0, nll=99.0))
298
  for key in ("wer", "cer", "nll"):
299
- self.log(f"dev_{key}", float(f[key]), sync_dist=False, rank_zero_only=False, batch_size=1)
300
 
301
 
302
  model = ASR.restore_from(str(R / "init.nemo"), map_location="cpu")
@@ -390,15 +398,15 @@ class Report(pl.Callback):
390
  def on_train_start(self, t, m):
391
  self.f = (R / "metrics.jsonl").open("a", buffering=1) if t.is_global_zero else None
392
  if t.is_global_zero:
393
- say("🔥 GPU eğitimi aktif! İlk eğitim batch'i bekleniyor...")
394
 
395
  def on_train_batch_end(self, t, m, outputs, batch, batch_idx):
396
  if self.f and (t.global_step == 1 or t.global_step % 5 == 0):
397
  loss = outputs["loss"] if isinstance(outputs, dict) else outputs
398
  lr = t.optimizers[0].param_groups[0]["lr"]
399
  self.f.write(json.dumps(dict(step=t.global_step, loss=float(loss.detach()), lr=lr, time=time.time())) + "\n")
400
- if t.global_step <= 15:
401
- say(f"✅ Adım {t.global_step} tamamlandı | Loss: {float(loss.detach()):.3f}")
402
 
403
  def on_validation_end(self, t, m):
404
  if t.sanity_checking or not t.is_global_zero or not self.f:
 
1
+ import os, io, re, json, time, math, pickle, sys, subprocess, shutil, datetime, glob, gc, uuid, warnings, logging
2
  from pathlib import Path
3
  from concurrent.futures import ThreadPoolExecutor
4
+
5
+ # Gereksiz Lightning / NeMo uyarılarını tamamen sustur
6
+ warnings.filterwarnings("ignore")
7
+ os.environ["HYDRA_FULL_ERROR"] = "0"
8
+
9
  import numpy as np
10
  import torch
11
  import torch.nn.functional as F
 
18
  from lightning.pytorch.strategies import DDPStrategy
19
  from lightning.pytorch.callbacks import ModelCheckpoint
20
  from nemo.collections.asr.models import EncDecCTCModelBPE
21
+ from nemo.utils import logging as nemo_logging
22
+
23
+ nemo_logging.setLevel(logging.ERROR)
24
+ logging.getLogger("lightning.pytorch").setLevel(logging.ERROR)
25
 
26
  sys.path.insert(0, str(Path(__file__).parent))
27
  import lib
 
239
  barrier()
240
 
241
 
242
+ # ---------------------------------------------------------------- 4) model (FP16 GradClip + sync_dist=True tam uyumlu)
243
  say("🧠 Adım 4/5: init.nemo belleğe yükleniyor...")
244
  class ASR(EncDecCTCModelBPE):
245
  group_names = ["fleurs"] + SOURCES + ["fleurs_rest"]
246
 
247
  def configure_optimizers(self):
 
248
  opt = torch.optim.AdamW(self.parameters(), lr=C["lr"], betas=(0.9, 0.98), weight_decay=C["wd"], foreach=True)
249
  total, warm = C["steps"], C["warmup"]
250
  decay = int(total * 0.25)
 
304
  self.domain_report = rep
305
  f = rep.get("fleurs", dict(wer=100.0, cer=100.0, nll=99.0))
306
  for key in ("wer", "cer", "nll"):
307
+ self.log(f"dev_{key}", float(f[key]), sync_dist=(world > 1), rank_zero_only=False, batch_size=1)
308
 
309
 
310
  model = ASR.restore_from(str(R / "init.nemo"), map_location="cpu")
 
398
  def on_train_start(self, t, m):
399
  self.f = (R / "metrics.jsonl").open("a", buffering=1) if t.is_global_zero else None
400
  if t.is_global_zero:
401
+ say("🔥 GPU eğitimi aktif! İlk eğitim batch'i işleniyor...")
402
 
403
  def on_train_batch_end(self, t, m, outputs, batch, batch_idx):
404
  if self.f and (t.global_step == 1 or t.global_step % 5 == 0):
405
  loss = outputs["loss"] if isinstance(outputs, dict) else outputs
406
  lr = t.optimizers[0].param_groups[0]["lr"]
407
  self.f.write(json.dumps(dict(step=t.global_step, loss=float(loss.detach()), lr=lr, time=time.time())) + "\n")
408
+ if t.global_step <= 15 or t.global_step % 50 == 0:
409
+ say(f"⚡ Adım {t.global_step:,}/{C['steps']:,} | Loss: {float(loss.detach()):.3f} | LR: {lr:.2e}")
410
 
411
  def on_validation_end(self, t, m):
412
  if t.sanity_checking or not t.is_global_zero or not self.f: