Download src/part1/train.py from Chechaaa09/anlp-a2-models: direct link, hf CLI and curl.
- Browser
- Download file 8.14 kB
-
https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part1/train.py
- Command line
-
hf download hf://Chechaaa09/anlp-a2-models/src/part1/train.py
-
curl -L -o train.py https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part1/train.py
8.14 kB
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import math | |
| from torch.utils.data import DataLoader | |
| from datasets import load_from_disk | |
| import sacrebleu | |
| import wandb | |
| import argparse | |
| from tqdm import tqdm | |
| import yaml | |
| import os | |
| from .model import DecoderOnlyTransformer | |
| from .dataset import train_or_load_tokenizer, prepare_data, Translation_Dataset | |
| def compute_translation_bleu(model, test_pairs, tokenizer, max_samples=50, max_gen_len=40, device="cuda"): | |
| model.eval() | |
| scores = [] | |
| samples = test_pairs[:max_samples] | |
| with torch.no_grad(): | |
| for lang, src_text, ref_text in tqdm(samples, desc = "Computing BLEU", leave = False): | |
| prompt_str = f"<|{lang}|> {src_text} <|en|>" | |
| prompt_ids = tokenizer.encode(prompt_str, add_special_tokens=False) | |
| current_ids = torch.tensor([prompt_ids], dtype=torch.long, device=device) | |
| prompt_len = current_ids.size(1) | |
| for _ in range(max_gen_len): | |
| logits = model(current_ids) | |
| next_token_id = torch.argmax(logits[:, -1, :], dim = -1, keepdim = True) | |
| current_ids = torch.cat([current_ids, next_token_id], dim = -1) | |
| if next_token_id.item() == tokenizer.eos_token_id: | |
| break | |
| generated_ids = current_ids[0, prompt_len:].tolist() | |
| generated_text = tokenizer.decode(generated_ids, skip_special_tokens = True) | |
| bleu = sacrebleu.sentence_bleu(generated_text, [ref_text]).score | |
| scores.append(bleu) | |
| avg_bleu = sum(scores) / max(1, len(scores)) | |
| return avg_bleu | |
| def evaluate(model, val_loader, device = "cuda"): | |
| model.eval() | |
| total_loss = 0.0 | |
| total_batches = 0 | |
| criterion = nn.CrossEntropyLoss(ignore_index = -100) | |
| with torch.no_grad(): | |
| for batch in tqdm(val_loader, desc = "Evaluating Val Loss", leave = False): | |
| input_ids = batch["input_ids"].to(device) | |
| targets = batch["targets"].to(device) | |
| logits = model(input_ids) | |
| loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1)) | |
| total_loss += loss.item() | |
| total_batches += 1 | |
| avg_loss = total_loss / max(1, total_batches) | |
| return avg_loss, math.exp(avg_loss) | |
| def train(args): | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| print(f"using device: {device} | training variant {args.variant}") | |
| wandb.init( | |
| project = "ANLP-Assignment-2-Part1", | |
| name = f"variant_{args.variant}", | |
| config = vars(args) | |
| ) | |
| print("Preparing dataset and tokenizer...") | |
| train_pairs, test_pairs, raw_ds = prepare_data(num_samples = args.num_samples, test_size = args.test_size) | |
| tokenizer = train_or_load_tokenizer(raw_ds, vocab_size = args.vocab_size) | |
| train_ds = Translation_Dataset(train_pairs, tokenizer, max_seq_len = args.seq_len) | |
| val_ds = Translation_Dataset(test_pairs, tokenizer, max_seq_len = args.seq_len) | |
| train_loader = DataLoader(train_ds, batch_size = args.batch_size, shuffle = True) | |
| val_loader = DataLoader(val_ds, batch_size = args.batch_size, shuffle = False) | |
| model = DecoderOnlyTransformer(vocab_size = len(tokenizer), d_model = args.d_model, n_heads = args.n_heads, d_ff = args.d_ff, n_layers = args.n_layers, variant = args.variant, max_seq_len = args.seq_len, dropout = args.dropout).to(device) | |
| optimizer = torch.optim.AdamW(model.parameters(), lr = args.lr, weight_decay = args.weight_decay) | |
| criterion = nn.CrossEntropyLoss(ignore_index = -100) | |
| scaler = torch.amp.GradScaler("cuda") | |
| tokens_trained = 0 | |
| epoch = 0 | |
| best_val_loss = float("inf") | |
| last_val_ppl = None | |
| last_val_bleu = None | |
| pbar = tqdm(total = args.max_tokens, desc = f"Variant {args.variant}", unit = "tok") | |
| while tokens_trained < args.max_tokens: | |
| epoch += 1 | |
| for batch in train_loader: | |
| model.train() | |
| optimizer.zero_grad() | |
| input_ids = batch["input_ids"].to(device) | |
| targets = batch["targets"].to(device) | |
| with torch.amp.autocast("cuda"): | |
| logits = model(input_ids) | |
| loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1)) | |
| scaler.scale(loss).backward() | |
| scaler.step(optimizer) | |
| scaler.update() | |
| num_tokens = input_ids.numel() | |
| tokens_trained += num_tokens | |
| pbar.update(num_tokens) | |
| postfix = {"loss": f"{loss.item(): .4f}"} | |
| if last_val_ppl is not None: | |
| postfix["val_ppl"] = f"{last_val_ppl:.1f}" | |
| if last_val_bleu is not None: | |
| postfix["bleu"] = f"{last_val_bleu:.1f}" | |
| pbar.set_postfix(postfix) | |
| if tokens_trained % args.eval_every_tokens < num_tokens or tokens_trained >= args.max_tokens: | |
| val_loss, val_ppl = evaluate(model, val_loader, device = device) | |
| val_bleu = compute_translation_bleu(model, test_pairs, tokenizer, device = device) | |
| last_val_ppl = val_ppl | |
| last_val_bleu = val_bleu | |
| tqdm.write(f"\n[Tokens: {tokens_trained:,}] Loss: {val_loss:.4f} | PPL: {val_ppl:.2f} | BLEU: {val_bleu:.2f}") | |
| wandb.log({ | |
| "tokens_trained": tokens_trained, | |
| "train/loss": loss.item(), | |
| "val/loss": val_loss, | |
| "val/perplexity": val_ppl, | |
| "val/bleu": val_bleu, | |
| "epoch": epoch, | |
| }) | |
| if val_loss < best_val_loss: | |
| best_val_loss = val_loss | |
| os.makedirs(args.save_dir, exist_ok = True) | |
| torch.save(model.state_dict(), f"{args.save_dir}/best_variant_{args.variant}.pt") | |
| if tokens_trained >= args.max_tokens: | |
| break | |
| pbar.close() | |
| print(f"training for variant {args.variant} complete") | |
| wandb.finish() | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--config", type=str, default=None, help="Path to config YAML") | |
| parser.add_argument("--all", action="store_true", help="Train all 5 variants sequentially") | |
| parser.add_argument("--variant", type=int, default=1, choices=[1, 2, 3, 4, 5]) | |
| parser.add_argument("--d_model", type=int, default=384) | |
| parser.add_argument("--n_heads", type=int, default=6) | |
| parser.add_argument("--d_ff", type=int, default=1536) | |
| parser.add_argument("--n_layers", type=int, default=6) | |
| parser.add_argument("--vocab_size", type=int, default=8192) | |
| parser.add_argument("--seq_len", type=int, default=128) | |
| parser.add_argument("--batch_size", type=int, default=32) | |
| parser.add_argument("--lr", type=float, default=5e-4) | |
| parser.add_argument("--weight_decay", type=float, default=0.01) | |
| parser.add_argument("--dropout", type=float, default=0.1) | |
| parser.add_argument("--max_tokens", type=int, default=12_000_000) | |
| parser.add_argument("--num_samples", type=int, default=50000) | |
| parser.add_argument("--test_size", type=int, default=2000) | |
| parser.add_argument("--eval_every_tokens", type=int, default=1_000_000) | |
| parser.add_argument("--save_dir", type=str, default="checkpoints_part1") | |
| args = parser.parse_args() | |
| if args.all: | |
| all_configs = [f"configs/part1_var{i}.yaml" for i in range(1, 6)] | |
| for cfg_path in all_configs: | |
| print("\n" + "=" * 60) | |
| print(f"Starting run for: {cfg_path}") | |
| print("=" * 60) | |
| with open(cfg_path, "r") as f: | |
| cfg = yaml.safe_load(f) | |
| run_args = argparse.Namespace(**vars(args)) | |
| for k, v in cfg.items(): | |
| setattr(run_args, k, v) | |
| train(run_args) | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| elif args.config: | |
| with open(args.config, "r") as f: | |
| cfg = yaml.safe_load(f) | |
| for k, v in cfg.items(): | |
| setattr(args, k, v) | |
| train(args) | |
| else: | |
| train(args) |