Upload sft_03_train.py with huggingface_hub

sft_03_train.py

@@ -24,6 +24,10 @@ import sys
 import time
 from pathlib import Path
 
+# Liger CE training'de logits'i in-place modifiye ettigi icin SFT'de devre disi
+# Bu IMPORT'tan ONCE set edilmeli
+os.environ["NANOGPT_NO_LIGER"] = "1"
+
 import numpy as np
 import torch
 import torch.nn.functional as F
@@ -249,13 +253,37 @@ def main():
     model = GPTV5(cfg).to(DEVICE)
     ckpt = torch.load(args.base, map_location=DEVICE, weights_only=False)
     if "model" in ckpt:
-        model.load_state_dict(ckpt["model"])
+        sd = ckpt["model"]
         log(f"  Base step: {ckpt.get('step', '?')}  best_val: "
             f"{ckpt.get('best_val', '?')}")
     else:
-        model.load_state_dict(ckpt)
+        sd = ckpt
+
+    # _orig_mod. prefix'i kaldir (compile ile kaydedilmis ise)
+    has_prefix = any(k.startswith("_orig_mod.") for k in sd.keys())
+    if has_prefix:
+        log(f"  ! _orig_mod. prefix tespit edildi, kaldiriliyor")
+        sd = {k.replace("_orig_mod.", "", 1): v for k, v in sd.items()}
+
+    missing, unexpected = model.load_state_dict(sd, strict=False)
+    log(f"  Yuklendi: {len(sd) - len(unexpected)}/{len(sd)} key")
+    if missing:
+        log(f"  ! Missing keys ({len(missing)}): {missing[:3]}...")
+    if unexpected:
+        log(f"  ! Unexpected keys ({len(unexpected)}): {unexpected[:3]}...")
     log(f"  Params: {sum(p.numel() for p in model.parameters())/1e6:.1f}M")
 
+    # Sanity check — random tokens uzerinde forward, ortalama top-1 olasilik
+    with torch.no_grad():
+        with torch.amp.autocast(device_type="cuda", dtype=DTYPE):
+            test_x = torch.randint(0, cfg.vocab_size, (2, 64), device=DEVICE)
+            test_logits, _ = model(test_x, test_x)
+            probs = F.softmax(test_logits[0, 0].float(), dim=-1)
+            top1_p = probs.max().item()
+            entropy = -(probs * torch.log(probs + 1e-12)).sum().item()
+    log(f"  Sanity: top1_p={top1_p:.4f}, entropy={entropy:.3f} "
+        f"(uniform ~= {math.log(cfg.vocab_size):.2f}, trained <~ 5)")
+
     # Resume?
     start_step = 0
     sft_latest = out_dir / "sft_latest.pt"