import torch import torch.nn as nn import torch.nn.functional as F import math from torch.utils.data import DataLoader, Dataset from datasets import load_from_disk from transformers import AutoTokenizer import sacrebleu import wandb import argparse import json from pathlib import Path import yaml from tqdm import tqdm from .model import DecoderOnlyTransformer from .optimizers import CustomAdamW, CustomCautious, CustomLion, CustomScion, CustomSophia class TextDataset(Dataset): def __init__(self, texts, tokenizer, seq_len = 256): self.seq_len = seq_len all_tokens = [] for text in texts: token_ids = tokenizer.encode(text) all_tokens.extend(token_ids) all_tokens.append(tokenizer.eos_token_id) self.tokens = torch.tensor(all_tokens, dtype = torch.long) self.num_chunks = len(self.tokens) // (self.seq_len + 1) def __len__(self): return self.num_chunks def __getitem__(self, idx): start = idx * (self.seq_len + 1) end = start + self.seq_len + 1 chunk = self.tokens[start:end] return{ "input_ids": chunk[:-1], "targets": chunk[1:] } def compute_bleu(model, test_samples, tokenizer, prompt_len = 32, gen_len = 32, device = "cuda"): model.eval() scores = [] with torch.no_grad(): for test in test_samples: token_ids = tokenizer.encode(test) if len(token_ids) < prompt_len + 5: continue prompt_ids = torch.tensor([token_ids[:prompt_len]], dtype = torch.long, device = device) reference_ids = token_ids[prompt_len : prompt_len + gen_len] reference_text = tokenizer.decode(reference_ids, skip_special_tokens = True) current_ids = prompt_ids.clone() for _ in range(gen_len): logits = model(current_ids) next_token_id = torch.argmax(logits[:, -1, :], dim = -1, keepdim = True) current_ids = torch.cat([current_ids, next_token_id], dim = -1) generated_ids = current_ids[0, prompt_len:].tolist() generated_text = tokenizer.decode(generated_ids, skip_special_tokens = True) bleu = sacrebleu.sentence_bleu(generated_text, [reference_text]).score scores.append(bleu) avg_bleu = sum(scores) / max(1, len(scores)) return avg_bleu def evaluate(model, val_loader, criterion, device = "cuda"): model.eval() total_loss = 0.0 total_batches = 0 with torch.no_grad(): for batch in val_loader: input_ids = batch["input_ids"].to(device) targets = batch["targets"].to(device) logits = model(input_ids) loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1)) total_loss += loss.item() total_batches += 1 avg_loss = total_loss / max(1, total_batches) return avg_loss, math.exp(avg_loss) def train(args): run = wandb.init( project = "ANLP-Assignment-2-Part2", name = f"{args.optimizer}_lr{args.lr}_wd{args.weight_decay}", config = vars(args), ) device = "cuda" if torch.cuda.is_available() else "cpu" tokenizer = AutoTokenizer.from_pretrained("gpt2") tokenizer.pad_token = tokenizer.eos_token dataset = load_from_disk("src/part2/dataset") splits = dataset.train_test_split(test_size = 0.1, seed = 42) val_test = splits["test"].train_test_split(test_size = 0.5, seed = 42) train_ds = TextDataset(splits["train"]["text"], tokenizer, seq_len = args.seq_len) val_ds = TextDataset(val_test["train"]["text"], tokenizer, seq_len = args.seq_len) test_ds = val_test["test"]["text"] train_loader = DataLoader(train_ds, batch_size = args.batch_size, shuffle = True) val_loader = DataLoader(val_ds, batch_size = args.batch_size, shuffle = False) model = DecoderOnlyTransformer( vocab_size = tokenizer.vocab_size, d_model = args.d_model, n_heads = args.n_heads, d_ff = args.d_ff, n_layers = args.n_layers, max_seq_len = args.seq_len, dropout = args.dropout, ).to(device) checkpoint_dir = Path(args.save_dir) / args.optimizer / run.id checkpoint_dir.mkdir(parents = True, exist_ok = False) metadata = { "config": vars(args), "model_config": { "vocab_size": tokenizer.vocab_size, "d_model": args.d_model, "n_heads": args.n_heads, "d_ff": args.d_ff, "n_layers": args.n_layers, "max_seq_len": args.seq_len, "dropout": args.dropout, }, "tokenizer": "gpt2", "wandb_run_id": run.id, "wandb_url": run.url, "evaluations": [], } best_val_loss = float("inf") print(f"Checkpoints: {checkpoint_dir}") if args.optimizer == "adamw": optimizer = CustomAdamW(model.parameters(), lr = args.lr, weight_decay = args.weight_decay) elif args.optimizer == "cautious": optimizer = CustomCautious(model.parameters(), lr = args.lr, weight_decay = args.weight_decay) elif args.optimizer == "lion": optimizer = CustomLion(model.parameters(), lr = args.lr, weight_decay = args.weight_decay) elif args.optimizer == "scion": optimizer = CustomScion(model.parameters(), lr = args.lr, weight_decay = args.weight_decay) elif args.optimizer == "sophia": optimizer = CustomSophia(model.parameters(), lr = args.lr, weight_decay = args.weight_decay, rho = getattr(args, "rho", 0.04)) else: raise ValueError(f"unknown optimizer: {args.optimizer}") criterion = nn.CrossEntropyLoss() tokens_per_batch = args.batch_size * args.seq_len total_tokens_budget = len(train_ds) * args.seq_len eval_token_interval = total_tokens_budget * 0.1 total_tokens_processed = 0 next_eval_token = eval_token_interval step_count = 0 model.train() print(f"Starting training with {args.optimizer} | Budget: {total_tokens_budget:,} tokens") for epoch in range(args.epochs): pbar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{args.epochs}", dynamic_ncols=True) for batch in pbar: input_ids = batch["input_ids"].to(device) targets = batch["targets"].to(device) if args.optimizer == "sophia" and (step_count % 10 == 0): optimizer.zero_grad() with torch.no_grad(): h_logits = model(input_ids) probs = torch.softmax(h_logits, dim = -1) sampled_targets = torch.multinomial(probs.view(-1, probs.size(-1)), num_samples = 1).view_as(targets) h_loss = criterion(model(input_ids).view(-1, probs.size(-1)), sampled_targets.view(-1)) h_loss.backward() optimizer.update_hessian() optimizer.zero_grad() optimizer.zero_grad() logits = model(input_ids) loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1)) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm = 1.0) optimizer.step() step_count += 1 total_tokens_processed += tokens_per_batch pbar.set_postfix(loss=f"{loss.item():.4f}", tokens=f"{total_tokens_processed:,}") wandb.log({ "train/loss": loss.item(), "tokens": total_tokens_processed, }) if total_tokens_processed >= next_eval_token: val_loss, val_ppl = evaluate(model, val_loader, criterion, device) bleu = compute_bleu(model, test_ds[:50], tokenizer, device = device) fraction = total_tokens_processed / total_tokens_budget print(f"\n[{fraction:.1f}x tokens] Val Loss: {val_loss:.4f} | PPL: {val_ppl:.2f} | BLEU: {bleu:.4f}") wandb.log({ "val/loss": val_loss, "val/perplexity": val_ppl, "val/bleu": bleu, "tokens": total_tokens_processed, "fraction": fraction, }) metrics = { "step": step_count, "tokens": total_tokens_processed, "fraction": fraction, "val_loss": val_loss, "val_perplexity": val_ppl, "val_bleu": bleu, } metadata["evaluations"].append(metrics) if val_loss < best_val_loss: best_val_loss = val_loss save_weights(model, checkpoint_dir / "best.pt") metadata["best"] = metrics save_weights(model, checkpoint_dir / "last.pt") metadata["last"] = metrics (checkpoint_dir / "metrics.json").write_text(json.dumps(metadata, indent = 2) + "\n") next_eval_token += eval_token_interval model.train() if total_tokens_processed >= total_tokens_budget: break if total_tokens_processed >= total_tokens_budget: break save_weights(model, checkpoint_dir / "last.pt") metadata["last_checkpoint_step"] = step_count metadata["last_checkpoint_tokens"] = total_tokens_processed metadata["complete"] = True (checkpoint_dir / "metrics.json").write_text(json.dumps(metadata, indent = 2) + "\n") print(f"Saved weights and metrics to {checkpoint_dir}") wandb.finish() def save_weights(model, path): # Replace only after the entire state dict has been written. temporary_path = path.with_suffix(".pt.tmp") torch.save(model.state_dict(), temporary_path) temporary_path.replace(path) def parse_args(): parser = argparse.ArgumentParser(description="Train transformer with specified optimizer config(s)") parser.add_argument("--config", type=str, default=None, help="Path to YAML config file") parser.add_argument("--all", action="store_true", help="Train all 5 optimizer configs sequentially") parser.add_argument("--lr", type=float, default=None) parser.add_argument("--batch_size", type=int, default=None) parser.add_argument("--save_dir", type=str, default="checkpoints_part2", help="Root directory for per-run weights and metrics") cli_args = parser.parse_args() if not cli_args.config and not cli_args.all: parser.error("You must specify either --config or --all") return cli_args def build_config(config_path, cli_args): with open(config_path, "r") as f: config = yaml.safe_load(f) if cli_args.lr is not None: config["lr"] = cli_args.lr if cli_args.batch_size is not None: config["batch_size"] = cli_args.batch_size config["save_dir"] = cli_args.save_dir return argparse.Namespace(**config) if __name__ == "__main__": cli_args = parse_args() if cli_args.all: all_configs = [ "configs/part2_adamw.yaml", "configs/part2_cautious.yaml", "configs/part2_lion.yaml", "configs/part2_scion.yaml", "configs/part2_sophia.yaml", ] for config_path in all_configs: print("\n" + "=" * 60) print(f"Starting run for: {config_path}") print("=" * 60) args = build_config(config_path, cli_args) train(args) if torch.cuda.is_available(): torch.cuda.empty_cache() else: args = build_config(cli_args.config, cli_args) train(args)