Download src/part2/train.py from Chechaaa09/anlp-a2-models: direct link, hf CLI and curl.
- Browser
- Download file 11.7 kB
-
https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part2/train.py
- Command line
-
hf download hf://Chechaaa09/anlp-a2-models/src/part2/train.py
-
curl -L -o train.py https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part2/train.py
11.7 kB
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import math | |
| from torch.utils.data import DataLoader, Dataset | |
| from datasets import load_from_disk | |
| from transformers import AutoTokenizer | |
| import sacrebleu | |
| import wandb | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import yaml | |
| from tqdm import tqdm | |
| from .model import DecoderOnlyTransformer | |
| from .optimizers import CustomAdamW, CustomCautious, CustomLion, CustomScion, CustomSophia | |
| class TextDataset(Dataset): | |
| def __init__(self, texts, tokenizer, seq_len = 256): | |
| self.seq_len = seq_len | |
| all_tokens = [] | |
| for text in texts: | |
| token_ids = tokenizer.encode(text) | |
| all_tokens.extend(token_ids) | |
| all_tokens.append(tokenizer.eos_token_id) | |
| self.tokens = torch.tensor(all_tokens, dtype = torch.long) | |
| self.num_chunks = len(self.tokens) // (self.seq_len + 1) | |
| def __len__(self): | |
| return self.num_chunks | |
| def __getitem__(self, idx): | |
| start = idx * (self.seq_len + 1) | |
| end = start + self.seq_len + 1 | |
| chunk = self.tokens[start:end] | |
| return{ | |
| "input_ids": chunk[:-1], | |
| "targets": chunk[1:] | |
| } | |
| def compute_bleu(model, test_samples, tokenizer, prompt_len = 32, gen_len = 32, device = "cuda"): | |
| model.eval() | |
| scores = [] | |
| with torch.no_grad(): | |
| for test in test_samples: | |
| token_ids = tokenizer.encode(test) | |
| if len(token_ids) < prompt_len + 5: | |
| continue | |
| prompt_ids = torch.tensor([token_ids[:prompt_len]], dtype = torch.long, device = device) | |
| reference_ids = token_ids[prompt_len : prompt_len + gen_len] | |
| reference_text = tokenizer.decode(reference_ids, skip_special_tokens = True) | |
| current_ids = prompt_ids.clone() | |
| for _ in range(gen_len): | |
| logits = model(current_ids) | |
| next_token_id = torch.argmax(logits[:, -1, :], dim = -1, keepdim = True) | |
| current_ids = torch.cat([current_ids, next_token_id], dim = -1) | |
| generated_ids = current_ids[0, prompt_len:].tolist() | |
| generated_text = tokenizer.decode(generated_ids, skip_special_tokens = True) | |
| bleu = sacrebleu.sentence_bleu(generated_text, [reference_text]).score | |
| scores.append(bleu) | |
| avg_bleu = sum(scores) / max(1, len(scores)) | |
| return avg_bleu | |
| def evaluate(model, val_loader, criterion, device = "cuda"): | |
| model.eval() | |
| total_loss = 0.0 | |
| total_batches = 0 | |
| with torch.no_grad(): | |
| for batch in val_loader: | |
| input_ids = batch["input_ids"].to(device) | |
| targets = batch["targets"].to(device) | |
| logits = model(input_ids) | |
| loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1)) | |
| total_loss += loss.item() | |
| total_batches += 1 | |
| avg_loss = total_loss / max(1, total_batches) | |
| return avg_loss, math.exp(avg_loss) | |
| def train(args): | |
| run = wandb.init( | |
| project = "ANLP-Assignment-2-Part2", | |
| name = f"{args.optimizer}_lr{args.lr}_wd{args.weight_decay}", | |
| config = vars(args), | |
| ) | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| tokenizer = AutoTokenizer.from_pretrained("gpt2") | |
| tokenizer.pad_token = tokenizer.eos_token | |
| dataset = load_from_disk("src/part2/dataset") | |
| splits = dataset.train_test_split(test_size = 0.1, seed = 42) | |
| val_test = splits["test"].train_test_split(test_size = 0.5, seed = 42) | |
| train_ds = TextDataset(splits["train"]["text"], tokenizer, seq_len = args.seq_len) | |
| val_ds = TextDataset(val_test["train"]["text"], tokenizer, seq_len = args.seq_len) | |
| test_ds = val_test["test"]["text"] | |
| train_loader = DataLoader(train_ds, batch_size = args.batch_size, shuffle = True) | |
| val_loader = DataLoader(val_ds, batch_size = args.batch_size, shuffle = False) | |
| model = DecoderOnlyTransformer( | |
| vocab_size = tokenizer.vocab_size, | |
| d_model = args.d_model, | |
| n_heads = args.n_heads, | |
| d_ff = args.d_ff, | |
| n_layers = args.n_layers, | |
| max_seq_len = args.seq_len, | |
| dropout = args.dropout, | |
| ).to(device) | |
| checkpoint_dir = Path(args.save_dir) / args.optimizer / run.id | |
| checkpoint_dir.mkdir(parents = True, exist_ok = False) | |
| metadata = { | |
| "config": vars(args), | |
| "model_config": { | |
| "vocab_size": tokenizer.vocab_size, | |
| "d_model": args.d_model, | |
| "n_heads": args.n_heads, | |
| "d_ff": args.d_ff, | |
| "n_layers": args.n_layers, | |
| "max_seq_len": args.seq_len, | |
| "dropout": args.dropout, | |
| }, | |
| "tokenizer": "gpt2", | |
| "wandb_run_id": run.id, | |
| "wandb_url": run.url, | |
| "evaluations": [], | |
| } | |
| best_val_loss = float("inf") | |
| print(f"Checkpoints: {checkpoint_dir}") | |
| if args.optimizer == "adamw": | |
| optimizer = CustomAdamW(model.parameters(), lr = args.lr, weight_decay = args.weight_decay) | |
| elif args.optimizer == "cautious": | |
| optimizer = CustomCautious(model.parameters(), lr = args.lr, weight_decay = args.weight_decay) | |
| elif args.optimizer == "lion": | |
| optimizer = CustomLion(model.parameters(), lr = args.lr, weight_decay = args.weight_decay) | |
| elif args.optimizer == "scion": | |
| optimizer = CustomScion(model.parameters(), lr = args.lr, weight_decay = args.weight_decay) | |
| elif args.optimizer == "sophia": | |
| optimizer = CustomSophia(model.parameters(), lr = args.lr, weight_decay = args.weight_decay, rho = getattr(args, "rho", 0.04)) | |
| else: | |
| raise ValueError(f"unknown optimizer: {args.optimizer}") | |
| criterion = nn.CrossEntropyLoss() | |
| tokens_per_batch = args.batch_size * args.seq_len | |
| total_tokens_budget = len(train_ds) * args.seq_len | |
| eval_token_interval = total_tokens_budget * 0.1 | |
| total_tokens_processed = 0 | |
| next_eval_token = eval_token_interval | |
| step_count = 0 | |
| model.train() | |
| print(f"Starting training with {args.optimizer} | Budget: {total_tokens_budget:,} tokens") | |
| for epoch in range(args.epochs): | |
| pbar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{args.epochs}", dynamic_ncols=True) | |
| for batch in pbar: | |
| input_ids = batch["input_ids"].to(device) | |
| targets = batch["targets"].to(device) | |
| if args.optimizer == "sophia" and (step_count % 10 == 0): | |
| optimizer.zero_grad() | |
| with torch.no_grad(): | |
| h_logits = model(input_ids) | |
| probs = torch.softmax(h_logits, dim = -1) | |
| sampled_targets = torch.multinomial(probs.view(-1, probs.size(-1)), num_samples = 1).view_as(targets) | |
| h_loss = criterion(model(input_ids).view(-1, probs.size(-1)), sampled_targets.view(-1)) | |
| h_loss.backward() | |
| optimizer.update_hessian() | |
| optimizer.zero_grad() | |
| optimizer.zero_grad() | |
| logits = model(input_ids) | |
| loss = criterion(logits.view(-1, logits.size(-1)), targets.view(-1)) | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm = 1.0) | |
| optimizer.step() | |
| step_count += 1 | |
| total_tokens_processed += tokens_per_batch | |
| pbar.set_postfix(loss=f"{loss.item():.4f}", tokens=f"{total_tokens_processed:,}") | |
| wandb.log({ | |
| "train/loss": loss.item(), | |
| "tokens": total_tokens_processed, | |
| }) | |
| if total_tokens_processed >= next_eval_token: | |
| val_loss, val_ppl = evaluate(model, val_loader, criterion, device) | |
| bleu = compute_bleu(model, test_ds[:50], tokenizer, device = device) | |
| fraction = total_tokens_processed / total_tokens_budget | |
| print(f"\n[{fraction:.1f}x tokens] Val Loss: {val_loss:.4f} | PPL: {val_ppl:.2f} | BLEU: {bleu:.4f}") | |
| wandb.log({ | |
| "val/loss": val_loss, | |
| "val/perplexity": val_ppl, | |
| "val/bleu": bleu, | |
| "tokens": total_tokens_processed, | |
| "fraction": fraction, | |
| }) | |
| metrics = { | |
| "step": step_count, | |
| "tokens": total_tokens_processed, | |
| "fraction": fraction, | |
| "val_loss": val_loss, | |
| "val_perplexity": val_ppl, | |
| "val_bleu": bleu, | |
| } | |
| metadata["evaluations"].append(metrics) | |
| if val_loss < best_val_loss: | |
| best_val_loss = val_loss | |
| save_weights(model, checkpoint_dir / "best.pt") | |
| metadata["best"] = metrics | |
| save_weights(model, checkpoint_dir / "last.pt") | |
| metadata["last"] = metrics | |
| (checkpoint_dir / "metrics.json").write_text(json.dumps(metadata, indent = 2) + "\n") | |
| next_eval_token += eval_token_interval | |
| model.train() | |
| if total_tokens_processed >= total_tokens_budget: | |
| break | |
| if total_tokens_processed >= total_tokens_budget: | |
| break | |
| save_weights(model, checkpoint_dir / "last.pt") | |
| metadata["last_checkpoint_step"] = step_count | |
| metadata["last_checkpoint_tokens"] = total_tokens_processed | |
| metadata["complete"] = True | |
| (checkpoint_dir / "metrics.json").write_text(json.dumps(metadata, indent = 2) + "\n") | |
| print(f"Saved weights and metrics to {checkpoint_dir}") | |
| wandb.finish() | |
| def save_weights(model, path): | |
| # Replace only after the entire state dict has been written. | |
| temporary_path = path.with_suffix(".pt.tmp") | |
| torch.save(model.state_dict(), temporary_path) | |
| temporary_path.replace(path) | |
| def parse_args(): | |
| parser = argparse.ArgumentParser(description="Train transformer with specified optimizer config(s)") | |
| parser.add_argument("--config", type=str, default=None, help="Path to YAML config file") | |
| parser.add_argument("--all", action="store_true", help="Train all 5 optimizer configs sequentially") | |
| parser.add_argument("--lr", type=float, default=None) | |
| parser.add_argument("--batch_size", type=int, default=None) | |
| parser.add_argument("--save_dir", type=str, default="checkpoints_part2", help="Root directory for per-run weights and metrics") | |
| cli_args = parser.parse_args() | |
| if not cli_args.config and not cli_args.all: | |
| parser.error("You must specify either --config <path> or --all") | |
| return cli_args | |
| def build_config(config_path, cli_args): | |
| with open(config_path, "r") as f: | |
| config = yaml.safe_load(f) | |
| if cli_args.lr is not None: | |
| config["lr"] = cli_args.lr | |
| if cli_args.batch_size is not None: | |
| config["batch_size"] = cli_args.batch_size | |
| config["save_dir"] = cli_args.save_dir | |
| return argparse.Namespace(**config) | |
| if __name__ == "__main__": | |
| cli_args = parse_args() | |
| if cli_args.all: | |
| all_configs = [ | |
| "configs/part2_adamw.yaml", | |
| "configs/part2_cautious.yaml", | |
| "configs/part2_lion.yaml", | |
| "configs/part2_scion.yaml", | |
| "configs/part2_sophia.yaml", | |
| ] | |
| for config_path in all_configs: | |
| print("\n" + "=" * 60) | |
| print(f"Starting run for: {config_path}") | |
| print("=" * 60) | |
| args = build_config(config_path, cli_args) | |
| train(args) | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| else: | |
| args = build_config(cli_args.config, cli_args) | |
| train(args) | |