Upload sft_03_train.py with huggingface_hub

sft_03_train.py

@@ -305,8 +305,10 @@ def main():
         for _ in range(GRAD_ACCUM_STEPS):
             x, y, m = train_ds.get_batch(rng)
             with torch.amp.autocast(device_type="cuda", dtype=DTYPE):
-                logits, _ = model(x)  # targets=None → logits only (forward optimize)
-                loss = masked_ce_loss(logits, y, m, softcap=cfg.logit_softcap)
+                # targets=y geciyoruz → full (B,T,V) logits + softcap uygulanmis
+                # Donen loss'u atip kendi mask'li CE'mizi hesapliyoruz
+                logits, _ = model(x, y)
+                loss = masked_ce_loss(logits, y, m, softcap=0.0)
             if torch.isnan(loss) or loss.item() == 0.0:
                 continue
             (loss / GRAD_ACCUM_STEPS).backward()
@@ -351,8 +353,8 @@ def main():
                 with torch.amp.autocast(device_type="cuda", dtype=DTYPE):
                     for _ in range(20):
                         x, y, m = val_ds.get_batch(val_rng)
-                        logits, _ = model(x)
-                        vl = masked_ce_loss(logits, y, m, softcap=cfg.logit_softcap)
+                        logits, _ = model(x, y)
+                        vl = masked_ce_loss(logits, y, m, softcap=0.0)
                         if not torch.isnan(vl) and vl.item() > 0:
                             val_losses.append(vl.item())
             model.train()