anlp-a2-models / src /part2 /train.py
Chechaaa09's picture
Add completed Task 2 checkpoints and evaluation metrics
6c5b105 verified
Raw History Blame Contribute Delete
11.7 kB
import torch
import torch.nn as nn
import torch.nn.functional as F
import math
from torch.utils.data import DataLoader, Dataset
from datasets import load_from_disk
from transformers import AutoTokenizer
import sacrebleu
import wandb
import argparse
import json
from pathlib import Path
import yaml
from tqdm import tqdm
from .model import DecoderOnlyTransformer
from .optimizers import CustomAdamW, CustomCautious, CustomLion, CustomScion, CustomSophia
class TextDataset(Dataset):
def __init__(self, texts, tokenizer, seq_len = 256):
self.seq_len = seq_len
all_tokens = []
for text in texts:
token_ids = tokenizer.encode(text)
all_tokens.extend(token_ids)
all_tokens.append(tokenizer.eos_token_id)
self.tokens = torch.tensor(all_tokens, dtype = torch.long)
self.num_chunks = len(self.tokens) // (self.seq_len + 1)
def __len__(self):
return self.num_chunks
def __getitem__(self, idx):
start = idx * (self.seq_len + 1)
end = start + self.seq_len + 1
chunk = self.tokens[start:end]
return{
"input_ids": chunk[:-1],
"targets": chunk[1:]
}
def compute_bleu(model, test_samples, tokenizer, prompt_len = 32, gen_len = 32, device = "cuda"):
model.eval()
scores = []
with torch.no_grad():
for test in test_samples:
token_ids = tokenizer.encode(test)
if len(token_ids) < prompt_len + 5:
continue
prompt_ids = torch.tensor([token_ids[:prompt_len]], dtype = torch.long, device = device)
reference_ids = token_ids[prompt_len : prompt_len + gen_len]
reference_text = tokenizer.decode(reference_ids, skip_special_tokens = True)
current_ids = prompt_ids.clone()
for _ in range(gen_len):
logits = model(current_ids)
next_token_id = torch.argmax(logits[:, -1, :], dim = -1, keepdim = True)
current_ids = torch.cat([current_ids, next_token_id], dim = -1)
generated_ids = current_ids[0, prompt_len:].tolist()
generated_text = tokenizer.decode(generated_ids, skip_special_tokens = True)
bleu = sacrebleu.sentence_bleu(generated_text, [reference_text]).score
scores.append(bleu)
avg_bleu = sum(scores) / max(1, len(scores))
return avg_bleu
def evaluate(model, val_loader, criterion, device = "cuda"):
model.eval()
total_loss = 0.0
total_batches = 0
with torch.no_grad():
for batch in val_loader:
input_ids = batch["input_ids"].to(device)
targets = batch["targets"].to(device)
logits = model(input_ids)
loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1))
total_loss += loss.item()
total_batches += 1
avg_loss = total_loss / max(1, total_batches)
return avg_loss, math.exp(avg_loss)
def train(args):
run = wandb.init(
project = "ANLP-Assignment-2-Part2",
name = f"{args.optimizer}_lr{args.lr}_wd{args.weight_decay}",
config = vars(args),
)
device = "cuda" if torch.cuda.is_available() else "cpu"
tokenizer = AutoTokenizer.from_pretrained("gpt2")
tokenizer.pad_token = tokenizer.eos_token
dataset = load_from_disk("src/part2/dataset")
splits = dataset.train_test_split(test_size = 0.1, seed = 42)
val_test = splits["test"].train_test_split(test_size = 0.5, seed = 42)
train_ds = TextDataset(splits["train"]["text"], tokenizer, seq_len = args.seq_len)
val_ds = TextDataset(val_test["train"]["text"], tokenizer, seq_len = args.seq_len)
test_ds = val_test["test"]["text"]
train_loader = DataLoader(train_ds, batch_size = args.batch_size, shuffle = True)
val_loader = DataLoader(val_ds, batch_size = args.batch_size, shuffle = False)
model = DecoderOnlyTransformer(
vocab_size = tokenizer.vocab_size,
d_model = args.d_model,
n_heads = args.n_heads,
d_ff = args.d_ff,
n_layers = args.n_layers,
max_seq_len = args.seq_len,
dropout = args.dropout,
).to(device)
checkpoint_dir = Path(args.save_dir) / args.optimizer / run.id
checkpoint_dir.mkdir(parents = True, exist_ok = False)
metadata = {
"config": vars(args),
"model_config": {
"vocab_size": tokenizer.vocab_size,
"d_model": args.d_model,
"n_heads": args.n_heads,
"d_ff": args.d_ff,
"n_layers": args.n_layers,
"max_seq_len": args.seq_len,
"dropout": args.dropout,
},
"tokenizer": "gpt2",
"wandb_run_id": run.id,
"wandb_url": run.url,
"evaluations": [],
}
best_val_loss = float("inf")
print(f"Checkpoints: {checkpoint_dir}")
if args.optimizer == "adamw":
optimizer = CustomAdamW(model.parameters(), lr = args.lr, weight_decay = args.weight_decay)
elif args.optimizer == "cautious":
optimizer = CustomCautious(model.parameters(), lr = args.lr, weight_decay = args.weight_decay)
elif args.optimizer == "lion":
optimizer = CustomLion(model.parameters(), lr = args.lr, weight_decay = args.weight_decay)
elif args.optimizer == "scion":
optimizer = CustomScion(model.parameters(), lr = args.lr, weight_decay = args.weight_decay)
elif args.optimizer == "sophia":
optimizer = CustomSophia(model.parameters(), lr = args.lr, weight_decay = args.weight_decay, rho = getattr(args, "rho", 0.04))
else:
raise ValueError(f"unknown optimizer: {args.optimizer}")
criterion = nn.CrossEntropyLoss()
tokens_per_batch = args.batch_size * args.seq_len
total_tokens_budget = len(train_ds) * args.seq_len
eval_token_interval = total_tokens_budget * 0.1
total_tokens_processed = 0
next_eval_token = eval_token_interval
step_count = 0
model.train()
print(f"Starting training with {args.optimizer} | Budget: {total_tokens_budget:,} tokens")
for epoch in range(args.epochs):
pbar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{args.epochs}", dynamic_ncols=True)
for batch in pbar:
input_ids = batch["input_ids"].to(device)
targets = batch["targets"].to(device)
if args.optimizer == "sophia" and (step_count % 10 == 0):
optimizer.zero_grad()
with torch.no_grad():
h_logits = model(input_ids)
probs = torch.softmax(h_logits, dim = -1)
sampled_targets = torch.multinomial(probs.view(-1, probs.size(-1)), num_samples = 1).view_as(targets)
h_loss = criterion(model(input_ids).view(-1, probs.size(-1)), sampled_targets.view(-1))
h_loss.backward()
optimizer.update_hessian()
optimizer.zero_grad()
optimizer.zero_grad()
logits = model(input_ids)
loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1))
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm = 1.0)
optimizer.step()
step_count += 1
total_tokens_processed += tokens_per_batch
pbar.set_postfix(loss=f"{loss.item():.4f}", tokens=f"{total_tokens_processed:,}")
wandb.log({
"train/loss": loss.item(),
"tokens": total_tokens_processed,
})
if total_tokens_processed >= next_eval_token:
val_loss, val_ppl = evaluate(model, val_loader, criterion, device)
bleu = compute_bleu(model, test_ds[:50], tokenizer, device = device)
fraction = total_tokens_processed / total_tokens_budget
print(f"\n[{fraction:.1f}x tokens] Val Loss: {val_loss:.4f} | PPL: {val_ppl:.2f} | BLEU: {bleu:.4f}")
wandb.log({
"val/loss": val_loss,
"val/perplexity": val_ppl,
"val/bleu": bleu,
"tokens": total_tokens_processed,
"fraction": fraction,
})
metrics = {
"step": step_count,
"tokens": total_tokens_processed,
"fraction": fraction,
"val_loss": val_loss,
"val_perplexity": val_ppl,
"val_bleu": bleu,
}
metadata["evaluations"].append(metrics)
if val_loss < best_val_loss:
best_val_loss = val_loss
save_weights(model, checkpoint_dir / "best.pt")
metadata["best"] = metrics
save_weights(model, checkpoint_dir / "last.pt")
metadata["last"] = metrics
(checkpoint_dir / "metrics.json").write_text(json.dumps(metadata, indent = 2) + "\n")
next_eval_token += eval_token_interval
model.train()
if total_tokens_processed >= total_tokens_budget:
break
if total_tokens_processed >= total_tokens_budget:
break
save_weights(model, checkpoint_dir / "last.pt")
metadata["last_checkpoint_step"] = step_count
metadata["last_checkpoint_tokens"] = total_tokens_processed
metadata["complete"] = True
(checkpoint_dir / "metrics.json").write_text(json.dumps(metadata, indent = 2) + "\n")
print(f"Saved weights and metrics to {checkpoint_dir}")
wandb.finish()
def save_weights(model, path):
# Replace only after the entire state dict has been written.
temporary_path = path.with_suffix(".pt.tmp")
torch.save(model.state_dict(), temporary_path)
temporary_path.replace(path)
def parse_args():
parser = argparse.ArgumentParser(description="Train transformer with specified optimizer config(s)")
parser.add_argument("--config", type=str, default=None, help="Path to YAML config file")
parser.add_argument("--all", action="store_true", help="Train all 5 optimizer configs sequentially")
parser.add_argument("--lr", type=float, default=None)
parser.add_argument("--batch_size", type=int, default=None)
parser.add_argument("--save_dir", type=str, default="checkpoints_part2", help="Root directory for per-run weights and metrics")
cli_args = parser.parse_args()
if not cli_args.config and not cli_args.all:
parser.error("You must specify either --config <path> or --all")
return cli_args
def build_config(config_path, cli_args):
with open(config_path, "r") as f:
config = yaml.safe_load(f)
if cli_args.lr is not None:
config["lr"] = cli_args.lr
if cli_args.batch_size is not None:
config["batch_size"] = cli_args.batch_size
config["save_dir"] = cli_args.save_dir
return argparse.Namespace(**config)
if __name__ == "__main__":
cli_args = parse_args()
if cli_args.all:
all_configs = [
"configs/part2_adamw.yaml",
"configs/part2_cautious.yaml",
"configs/part2_lion.yaml",
"configs/part2_scion.yaml",
"configs/part2_sophia.yaml",
]
for config_path in all_configs:
print("\n" + "=" * 60)
print(f"Starting run for: {config_path}")
print("=" * 60)
args = build_config(config_path, cli_args)
train(args)
if torch.cuda.is_available():
torch.cuda.empty_cache()
else:
args = build_config(cli_args.config, cli_args)
train(args)