anlp-a2-models / src /part1 /train.py
Chechaaa09's picture
Upload Task 1 checkpoints, tokenizer, configurations and source
ad63418 verified
Raw History Blame Contribute Delete
8.14 kB
import torch
import torch.nn as nn
import torch.nn.functional as F
import math
from torch.utils.data import DataLoader
from datasets import load_from_disk
import sacrebleu
import wandb
import argparse
from tqdm import tqdm
import yaml
import os
from .model import DecoderOnlyTransformer
from .dataset import train_or_load_tokenizer, prepare_data, Translation_Dataset
def compute_translation_bleu(model, test_pairs, tokenizer, max_samples=50, max_gen_len=40, device="cuda"):
model.eval()
scores = []
samples = test_pairs[:max_samples]
with torch.no_grad():
for lang, src_text, ref_text in tqdm(samples, desc = "Computing BLEU", leave = False):
prompt_str = f"<|{lang}|> {src_text} <|en|>"
prompt_ids = tokenizer.encode(prompt_str, add_special_tokens=False)
current_ids = torch.tensor([prompt_ids], dtype=torch.long, device=device)
prompt_len = current_ids.size(1)
for _ in range(max_gen_len):
logits = model(current_ids)
next_token_id = torch.argmax(logits[:, -1, :], dim = -1, keepdim = True)
current_ids = torch.cat([current_ids, next_token_id], dim = -1)
if next_token_id.item() == tokenizer.eos_token_id:
break
generated_ids = current_ids[0, prompt_len:].tolist()
generated_text = tokenizer.decode(generated_ids, skip_special_tokens = True)
bleu = sacrebleu.sentence_bleu(generated_text, [ref_text]).score
scores.append(bleu)
avg_bleu = sum(scores) / max(1, len(scores))
return avg_bleu
def evaluate(model, val_loader, device = "cuda"):
model.eval()
total_loss = 0.0
total_batches = 0
criterion = nn.CrossEntropyLoss(ignore_index = -100)
with torch.no_grad():
for batch in tqdm(val_loader, desc = "Evaluating Val Loss", leave = False):
input_ids = batch["input_ids"].to(device)
targets = batch["targets"].to(device)
logits = model(input_ids)
loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1))
total_loss += loss.item()
total_batches += 1
avg_loss = total_loss / max(1, total_batches)
return avg_loss, math.exp(avg_loss)
def train(args):
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"using device: {device} | training variant {args.variant}")
wandb.init(
project = "ANLP-Assignment-2-Part1",
name = f"variant_{args.variant}",
config = vars(args)
)
print("Preparing dataset and tokenizer...")
train_pairs, test_pairs, raw_ds = prepare_data(num_samples = args.num_samples, test_size = args.test_size)
tokenizer = train_or_load_tokenizer(raw_ds, vocab_size = args.vocab_size)
train_ds = Translation_Dataset(train_pairs, tokenizer, max_seq_len = args.seq_len)
val_ds = Translation_Dataset(test_pairs, tokenizer, max_seq_len = args.seq_len)
train_loader = DataLoader(train_ds, batch_size = args.batch_size, shuffle = True)
val_loader = DataLoader(val_ds, batch_size = args.batch_size, shuffle = False)
model = DecoderOnlyTransformer(vocab_size = len(tokenizer), d_model = args.d_model, n_heads = args.n_heads, d_ff = args.d_ff, n_layers = args.n_layers, variant = args.variant, max_seq_len = args.seq_len, dropout = args.dropout).to(device)
optimizer = torch.optim.AdamW(model.parameters(), lr = args.lr, weight_decay = args.weight_decay)
criterion = nn.CrossEntropyLoss(ignore_index = -100)
scaler = torch.amp.GradScaler("cuda")
tokens_trained = 0
epoch = 0
best_val_loss = float("inf")
last_val_ppl = None
last_val_bleu = None
pbar = tqdm(total = args.max_tokens, desc = f"Variant {args.variant}", unit = "tok")
while tokens_trained < args.max_tokens:
epoch += 1
for batch in train_loader:
model.train()
optimizer.zero_grad()
input_ids = batch["input_ids"].to(device)
targets = batch["targets"].to(device)
with torch.amp.autocast("cuda"):
logits = model(input_ids)
loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1))
scaler.scale(loss).backward()
scaler.step(optimizer)
scaler.update()
num_tokens = input_ids.numel()
tokens_trained += num_tokens
pbar.update(num_tokens)
postfix = {"loss": f"{loss.item(): .4f}"}
if last_val_ppl is not None:
postfix["val_ppl"] = f"{last_val_ppl:.1f}"
if last_val_bleu is not None:
postfix["bleu"] = f"{last_val_bleu:.1f}"
pbar.set_postfix(postfix)
if tokens_trained % args.eval_every_tokens < num_tokens or tokens_trained >= args.max_tokens:
val_loss, val_ppl = evaluate(model, val_loader, device = device)
val_bleu = compute_translation_bleu(model, test_pairs, tokenizer, device = device)
last_val_ppl = val_ppl
last_val_bleu = val_bleu
tqdm.write(f"\n[Tokens: {tokens_trained:,}] Loss: {val_loss:.4f} | PPL: {val_ppl:.2f} | BLEU: {val_bleu:.2f}")
wandb.log({
"tokens_trained": tokens_trained,
"train/loss": loss.item(),
"val/loss": val_loss,
"val/perplexity": val_ppl,
"val/bleu": val_bleu,
"epoch": epoch,
})
if val_loss < best_val_loss:
best_val_loss = val_loss
os.makedirs(args.save_dir, exist_ok = True)
torch.save(model.state_dict(), f"{args.save_dir}/best_variant_{args.variant}.pt")
if tokens_trained >= args.max_tokens:
break
pbar.close()
print(f"training for variant {args.variant} complete")
wandb.finish()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=str, default=None, help="Path to config YAML")
parser.add_argument("--all", action="store_true", help="Train all 5 variants sequentially")
parser.add_argument("--variant", type=int, default=1, choices=[1, 2, 3, 4, 5])
parser.add_argument("--d_model", type=int, default=384)
parser.add_argument("--n_heads", type=int, default=6)
parser.add_argument("--d_ff", type=int, default=1536)
parser.add_argument("--n_layers", type=int, default=6)
parser.add_argument("--vocab_size", type=int, default=8192)
parser.add_argument("--seq_len", type=int, default=128)
parser.add_argument("--batch_size", type=int, default=32)
parser.add_argument("--lr", type=float, default=5e-4)
parser.add_argument("--weight_decay", type=float, default=0.01)
parser.add_argument("--dropout", type=float, default=0.1)
parser.add_argument("--max_tokens", type=int, default=12_000_000)
parser.add_argument("--num_samples", type=int, default=50000)
parser.add_argument("--test_size", type=int, default=2000)
parser.add_argument("--eval_every_tokens", type=int, default=1_000_000)
parser.add_argument("--save_dir", type=str, default="checkpoints_part1")
args = parser.parse_args()
if args.all:
all_configs = [f"configs/part1_var{i}.yaml" for i in range(1, 6)]
for cfg_path in all_configs:
print("\n" + "=" * 60)
print(f"Starting run for: {cfg_path}")
print("=" * 60)
with open(cfg_path, "r") as f:
cfg = yaml.safe_load(f)
run_args = argparse.Namespace(**vars(args))
for k, v in cfg.items():
setattr(run_args, k, v)
train(run_args)
if torch.cuda.is_available():
torch.cuda.empty_cache()
elif args.config:
with open(args.config, "r") as f:
cfg = yaml.safe_load(f)
for k, v in cfg.items():
setattr(args, k, v)
train(args)
else:
train(args)