FİXED

.gitignore

@@ -1 +1,6 @@
 *.pyc
+Model_Architecture/wandb
+*.pt
+Model_Architecture/checkpoints/step_8000_expert_best.pt.old
+Model_Architecture/data/train.txt
+Model_Architecture/data/val.txt

Model_Architecture/config.json

@@ -1,6 +1,6 @@
 {
     "model": {
-        "max_batch_size": 8,
+        "max_batch_size": 16,
         "max_seq_len": 512,
         "dtype": "fp32",
         "scale_fmt": null,
@@ -45,7 +45,9 @@
         "save_dir": "./checkpoints",
         "log_every": 100,
         "dtype": "fp32",
-        "compile": false
+        "compile": false,
+        "max_val_batches": 50,
+        "val_batch_size_multiplier": 4
     },
     "data": {
         "train_file": "./data/train.txt",

Model_Architecture/data.py

@@ -7,6 +7,7 @@ from tqdm import tqdm
 import mmap
 import numpy as np
 import os
+import json
 
 from model import ModelArgs
 
@@ -257,6 +258,8 @@ def create_dataloader(
     max_samples: Optional[int] = None,
     use_turkish_tokenizer: bool = True,
     use_memory_efficient: bool = True,  # NEW: Use memory-efficient loading by default
+    is_val: bool = True 
+
 ) -> DataLoader:
 
     # Select tokenizer based on user preference
@@ -310,10 +313,21 @@ def create_dataloader(
     except Exception as e:
         raise RuntimeError(f"Failed to create dataset: {e}")
 
+    config_path = Path("config.json")
+    
+    with open(config_path,"r") as f:
+        config = json.load(f)
+        val_batch_size = config["model"]["max_batch_size"] #* config["training"].get("val_batch_size_multiplier", 4)
+
+    if is_val:
+        batch_size = val_batch_size
+    else:
+        batch_size = args.max_batch_size
+
     # Create DataLoader with optimized settings
     dataloader = DataLoader(
         dataset,
-        batch_size=args.max_batch_size,
+        batch_size=batch_size,
         shuffle=shuffle,
         drop_last=drop_last,
         num_workers=num_workers,

Model_Architecture/data/dataset_info.json

@@ -0,0 +1,12 @@
+{
+  "dataset": "uonlp/CulturaX",
+  "subset": "tr",
+  "use_small": true,
+  "total_documents": 1471979,
+  "train_samples": 1398380,
+  "val_samples": 73599,
+  "train_ratio": 0.95,
+  "seed": 2357,
+  "train_file": "/mnt/2tb_ssd/ismAIl/Model_Architecture/data/train.txt",
+  "val_file": "/mnt/2tb_ssd/ismAIl/Model_Architecture/data/val.txt"
+}

Model_Architecture/train.py

@@ -243,6 +243,7 @@ def load_data(config):
         shuffle=False,
         drop_last=True,
         use_memory_efficient=True,  # Use memory-efficient loading
+        is_val = True 
     )
 
     print(f"✅ Train batches: {len(train_loader)}")
@@ -256,25 +257,38 @@ def evaluate(model, val_loader, device, config):
     model.eval()
     total_loss = 0.0
     total_tokens = 0
-
+    max_batches = config["training"].get("max_val_batches", 50)  # Only 50 batches
+    
+    # Add progress bar
+    from tqdm import tqdm
+    pbar = tqdm(total=max_batches, desc="📊 Validating", ncols=80)
+    
+    val_dtype = config["training"]["dtype"]
     with torch.no_grad():
-        for input_ids, target_ids in val_loader:
-            input_ids = input_ids.to(device)
-            target_ids = target_ids.to(device)
-
-            # Model returns just logits in eval mode (no lb_loss)
-            output = model(input_ids, start_pos=0)
-            logits = output if not isinstance(output, tuple) else output[0]
-
-            loss = F.cross_entropy(
-                logits.view(-1, logits.size(-1)),
-                target_ids.view(-1),
-                ignore_index=-1,
-            )
-
+        for i, (input_ids, target_ids) in enumerate(val_loader):
+            if i >= max_batches:
+                break
+                
+            input_ids = input_ids.to(device, non_blocking=True)
+            target_ids = target_ids.to(device, non_blocking=True)
+            
+            # Use autocast for speed
+            with torch.amp.autocast(device_type='cuda', enabled=(val_dtype == 'bf16')):
+                output = model(input_ids, start_pos=0)
+                logits = output[0] if isinstance(output, tuple) else output
+                
+                loss = F.cross_entropy(
+                    logits.view(-1, logits.size(-1)),
+                    target_ids.view(-1),
+                    ignore_index=-1,
+                )
+            
             total_loss += loss.item() * target_ids.numel()
             total_tokens += target_ids.numel()
-
+            pbar.update(1)
+            pbar.set_postfix({'loss': f'{loss.item():.3f}'})
+    
+    pbar.close()
     model.train()
     return total_loss / total_tokens
 
@@ -284,17 +298,16 @@ def save_checkpoint(model, optimizer, step, config, expert_idx=None):
     save_dir = Path(config["training"]["save_dir"])
     save_dir.mkdir(parents=True, exist_ok=True)
     
-    # Create checkpoint name
-    if expert_idx is not None:
-        ckpt_name = f"step_{step}_expert_{expert_idx}.pt"
-    else:
-        ckpt_name = f"step_{step}.pt"
-    
+    ckpt_name = f"step_{step}_expert_{expert_idx}.pt" if expert_idx is not None else f"step_{step}.pt"
     ckpt_path = save_dir / ckpt_name
     
+    # 🔥 Exclude cache buffers - they should be reinitialized from config
+    state_dict = model.state_dict()
+    filtered_state_dict = {k: v for k, v in state_dict.items() if 'cache' not in k.lower()}
+    
     checkpoint = {
         "step": step,
-        "model_state_dict": model.state_dict(),
+        "model_state_dict": filtered_state_dict,
         "optimizer_state_dict": optimizer.state_dict(),
         "config": config,
     }
@@ -406,11 +419,33 @@ def main():
     
     # Resume from checkpoint
     if args.resume:
+        print(f"📥 Loading checkpoint from {args.resume}...")
         ckpt = torch.load(args.resume, map_location=device)
-        model.load_state_dict(ckpt["model_state_dict"])
+        
+        # Create model with current config (ensures correct cache sizes)
+        model, model_args = setup_model(config, device)
+        
+        # Load state dict but skip/resize mismatched buffers
+        model_state_dict = model.state_dict()
+        loaded_state_dict = ckpt["model_state_dict"]
+        
+        skip_count = 0
+        for name, param in loaded_state_dict.items():
+            if name in model_state_dict:
+                if model_state_dict[name].shape != param.shape:
+                    if "cache" in name:  # Skip cache buffers
+                        skip_count += 1
+                        continue
+                    else:
+                        raise RuntimeError(f"Shape mismatch {name}: {param.shape} vs {model_state_dict[name].shape}")
+                model_state_dict[name].copy_(param)
+            else:
+                print(f"⚠️  Unexpected parameter: {name}")
+        
+        model.load_state_dict(model_state_dict, strict=False)
         optimizer.load_state_dict(ckpt["optimizer_state_dict"])
         step = ckpt["step"]
-        print(f"✅ Resumed from step {step}\n")
+        print(f"✅ Resumed from step {step} (skipped {skip_count} cache buffers)\n")
     
     # ✅ FIX: Only create scaler for FP16, not BF16 or FP32
     training_dtype = config["training"]["dtype"].lower()
@@ -570,4 +605,8 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()
+    
+    
+    
+