import torch import torch.nn as nn import torch.nn.functional as F import math from torch.utils.data import DataLoader from datasets import load_from_disk import sacrebleu import wandb import argparse from tqdm import tqdm import yaml import os from .model import DecoderOnlyTransformer from .dataset import train_or_load_tokenizer, prepare_data, Translation_Dataset def compute_translation_bleu(model, test_pairs, tokenizer, max_samples=50, max_gen_len=40, device="cuda"): model.eval() scores = [] samples = test_pairs[:max_samples] with torch.no_grad(): for lang, src_text, ref_text in tqdm(samples, desc = "Computing BLEU", leave = False): prompt_str = f"<|{lang}|> {src_text} <|en|>" prompt_ids = tokenizer.encode(prompt_str, add_special_tokens=False) current_ids = torch.tensor([prompt_ids], dtype=torch.long, device=device) prompt_len = current_ids.size(1) for _ in range(max_gen_len): logits = model(current_ids) next_token_id = torch.argmax(logits[:, -1, :], dim = -1, keepdim = True) current_ids = torch.cat([current_ids, next_token_id], dim = -1) if next_token_id.item() == tokenizer.eos_token_id: break generated_ids = current_ids[0, prompt_len:].tolist() generated_text = tokenizer.decode(generated_ids, skip_special_tokens = True) bleu = sacrebleu.sentence_bleu(generated_text, [ref_text]).score scores.append(bleu) avg_bleu = sum(scores) / max(1, len(scores)) return avg_bleu def evaluate(model, val_loader, device = "cuda"): model.eval() total_loss = 0.0 total_batches = 0 criterion = nn.CrossEntropyLoss(ignore_index = -100) with torch.no_grad(): for batch in tqdm(val_loader, desc = "Evaluating Val Loss", leave = False): input_ids = batch["input_ids"].to(device) targets = batch["targets"].to(device) logits = model(input_ids) loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1)) total_loss += loss.item() total_batches += 1 avg_loss = total_loss / max(1, total_batches) return avg_loss, math.exp(avg_loss) def train(args): device = "cuda" if torch.cuda.is_available() else "cpu" print(f"using device: {device} | training variant {args.variant}") wandb.init( project = "ANLP-Assignment-2-Part1", name = f"variant_{args.variant}", config = vars(args) ) print("Preparing dataset and tokenizer...") train_pairs, test_pairs, raw_ds = prepare_data(num_samples = args.num_samples, test_size = args.test_size) tokenizer = train_or_load_tokenizer(raw_ds, vocab_size = args.vocab_size) train_ds = Translation_Dataset(train_pairs, tokenizer, max_seq_len = args.seq_len) val_ds = Translation_Dataset(test_pairs, tokenizer, max_seq_len = args.seq_len) train_loader = DataLoader(train_ds, batch_size = args.batch_size, shuffle = True) val_loader = DataLoader(val_ds, batch_size = args.batch_size, shuffle = False) model = DecoderOnlyTransformer(vocab_size = len(tokenizer), d_model = args.d_model, n_heads = args.n_heads, d_ff = args.d_ff, n_layers = args.n_layers, variant = args.variant, max_seq_len = args.seq_len, dropout = args.dropout).to(device) optimizer = torch.optim.AdamW(model.parameters(), lr = args.lr, weight_decay = args.weight_decay) criterion = nn.CrossEntropyLoss(ignore_index = -100) scaler = torch.amp.GradScaler("cuda") tokens_trained = 0 epoch = 0 best_val_loss = float("inf") last_val_ppl = None last_val_bleu = None pbar = tqdm(total = args.max_tokens, desc = f"Variant {args.variant}", unit = "tok") while tokens_trained < args.max_tokens: epoch += 1 for batch in train_loader: model.train() optimizer.zero_grad() input_ids = batch["input_ids"].to(device) targets = batch["targets"].to(device) with torch.amp.autocast("cuda"): logits = model(input_ids) loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1)) scaler.scale(loss).backward() scaler.step(optimizer) scaler.update() num_tokens = input_ids.numel() tokens_trained += num_tokens pbar.update(num_tokens) postfix = {"loss": f"{loss.item(): .4f}"} if last_val_ppl is not None: postfix["val_ppl"] = f"{last_val_ppl:.1f}" if last_val_bleu is not None: postfix["bleu"] = f"{last_val_bleu:.1f}" pbar.set_postfix(postfix) if tokens_trained % args.eval_every_tokens < num_tokens or tokens_trained >= args.max_tokens: val_loss, val_ppl = evaluate(model, val_loader, device = device) val_bleu = compute_translation_bleu(model, test_pairs, tokenizer, device = device) last_val_ppl = val_ppl last_val_bleu = val_bleu tqdm.write(f"\n[Tokens: {tokens_trained:,}] Loss: {val_loss:.4f} | PPL: {val_ppl:.2f} | BLEU: {val_bleu:.2f}") wandb.log({ "tokens_trained": tokens_trained, "train/loss": loss.item(), "val/loss": val_loss, "val/perplexity": val_ppl, "val/bleu": val_bleu, "epoch": epoch, }) if val_loss < best_val_loss: best_val_loss = val_loss os.makedirs(args.save_dir, exist_ok = True) torch.save(model.state_dict(), f"{args.save_dir}/best_variant_{args.variant}.pt") if tokens_trained >= args.max_tokens: break pbar.close() print(f"training for variant {args.variant} complete") wandb.finish() if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--config", type=str, default=None, help="Path to config YAML") parser.add_argument("--all", action="store_true", help="Train all 5 variants sequentially") parser.add_argument("--variant", type=int, default=1, choices=[1, 2, 3, 4, 5]) parser.add_argument("--d_model", type=int, default=384) parser.add_argument("--n_heads", type=int, default=6) parser.add_argument("--d_ff", type=int, default=1536) parser.add_argument("--n_layers", type=int, default=6) parser.add_argument("--vocab_size", type=int, default=8192) parser.add_argument("--seq_len", type=int, default=128) parser.add_argument("--batch_size", type=int, default=32) parser.add_argument("--lr", type=float, default=5e-4) parser.add_argument("--weight_decay", type=float, default=0.01) parser.add_argument("--dropout", type=float, default=0.1) parser.add_argument("--max_tokens", type=int, default=12_000_000) parser.add_argument("--num_samples", type=int, default=50000) parser.add_argument("--test_size", type=int, default=2000) parser.add_argument("--eval_every_tokens", type=int, default=1_000_000) parser.add_argument("--save_dir", type=str, default="checkpoints_part1") args = parser.parse_args() if args.all: all_configs = [f"configs/part1_var{i}.yaml" for i in range(1, 6)] for cfg_path in all_configs: print("\n" + "=" * 60) print(f"Starting run for: {cfg_path}") print("=" * 60) with open(cfg_path, "r") as f: cfg = yaml.safe_load(f) run_args = argparse.Namespace(**vars(args)) for k, v in cfg.items(): setattr(run_args, k, v) train(run_args) if torch.cuda.is_available(): torch.cuda.empty_cache() elif args.config: with open(args.config, "r") as f: cfg = yaml.safe_load(f) for k, v in cfg.items(): setattr(args, k, v) train(args) else: train(args)