import os import torch import wandb import numpy as np from datasets import load_dataset from transformers import DistilBertTokenizer, DistilBertForSequenceClassification from torch.utils.data import DataLoader, Dataset from torch.optim import AdamW from transformers import get_linear_schedule_with_warmup from sklearn.metrics import f1_score, accuracy_score, confusion_matrix import matplotlib.pyplot as plt import seaborn as sns from dotenv import load_dotenv load_dotenv() # ── CONFIG ───────────────────────────── MODEL_NAME = "distilbert-base-uncased" MAX_LEN = 128 BATCH_SIZE = 16 EPOCHS = 3 LR = 2e-5 DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") print(f"Using device: {DEVICE}") # ── WANDB ────────────────────────────── wandb.init( project="customer-support-classifier", config={ "model": MODEL_NAME, "epochs": EPOCHS, "batch_size": BATCH_SIZE, "learning_rate": LR, "max_len": MAX_LEN } ) # ── DATASET ──────────────────────────── print("Loading Banking77 dataset...") dataset = load_dataset("PolyAI/banking77") train_data = dataset["train"] test_data = dataset["test"] num_labels = 77 print(f"Train size: {len(train_data)} | Test size: {len(test_data)}") # ── TOKENIZER ────────────────────────── tokenizer = DistilBertTokenizer.from_pretrained(MODEL_NAME) # ── CUSTOM DATASET CLASS ─────────────── class BankingDataset(Dataset): def __init__(self, data, tokenizer, max_len): self.data = data self.tokenizer = tokenizer self.max_len = max_len def __len__(self): return len(self.data) def __getitem__(self, idx): text = self.data[idx]["text"] label = self.data[idx]["label"] encoding = self.tokenizer( text, max_length=self.max_len, padding="max_length", truncation=True, return_tensors="pt" ) return { "input_ids": encoding["input_ids"].squeeze(), "attention_mask": encoding["attention_mask"].squeeze(), "label": torch.tensor(label, dtype=torch.long) } # ── DATALOADERS ──────────────────────── train_dataset = BankingDataset(train_data, tokenizer, MAX_LEN) test_dataset = BankingDataset(test_data, tokenizer, MAX_LEN) train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True) test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False) # ── MODEL ────────────────────────────── print("Loading DistilBERT model...") model = DistilBertForSequenceClassification.from_pretrained( MODEL_NAME, num_labels=num_labels ) model.to(DEVICE) # ── OPTIMIZER + SCHEDULER ────────────── optimizer = AdamW(model.parameters(), lr=LR, weight_decay=0.01) total_steps = len(train_loader) * EPOCHS scheduler = get_linear_schedule_with_warmup( optimizer, num_warmup_steps=total_steps // 10, num_training_steps=total_steps ) # ── LOSS ─────────────────────────────── criterion = torch.nn.CrossEntropyLoss() # ── TRAINING LOOP ────────────────────── def train_epoch(model, loader, optimizer, scheduler, criterion): model.train() total_loss = 0 all_preds = [] all_labels = [] for batch in loader: input_ids = batch["input_ids"].to(DEVICE) attention_mask = batch["attention_mask"].to(DEVICE) labels = batch["label"].to(DEVICE) optimizer.zero_grad() outputs = model(input_ids=input_ids, attention_mask=attention_mask) logits = outputs.logits loss = criterion(logits, labels) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) optimizer.step() scheduler.step() total_loss += loss.item() preds = torch.argmax(logits, dim=1).cpu().numpy() all_preds.extend(preds) all_labels.extend(labels.cpu().numpy()) avg_loss = total_loss / len(loader) acc = accuracy_score(all_labels, all_preds) f1 = f1_score(all_labels, all_preds, average="weighted") return avg_loss, acc, f1 # ── EVALUATION LOOP ──────────────────── def evaluate(model, loader, criterion): model.eval() total_loss = 0 all_preds = [] all_labels = [] with torch.no_grad(): for batch in loader: input_ids = batch["input_ids"].to(DEVICE) attention_mask = batch["attention_mask"].to(DEVICE) labels = batch["label"].to(DEVICE) outputs = model(input_ids=input_ids, attention_mask=attention_mask) logits = outputs.logits loss = criterion(logits, labels) total_loss += loss.item() preds = torch.argmax(logits, dim=1).cpu().numpy() all_preds.extend(preds) all_labels.extend(labels.cpu().numpy()) avg_loss = total_loss / len(loader) acc = accuracy_score(all_labels, all_preds) f1 = f1_score(all_labels, all_preds, average="weighted") return avg_loss, acc, f1, all_preds, all_labels # ── RUN TRAINING ─────────────────────── print("\nStarting training...") best_f1 = 0 for epoch in range(EPOCHS): print(f"\nEpoch {epoch+1}/{EPOCHS}") train_loss, train_acc, train_f1 = train_epoch( model, train_loader, optimizer, scheduler, criterion ) val_loss, val_acc, val_f1, preds, labels = evaluate( model, test_loader, criterion ) print(f"Train — Loss: {train_loss:.4f} | Acc: {train_acc:.4f} | F1: {train_f1:.4f}") print(f"Val — Loss: {val_loss:.4f} | Acc: {val_acc:.4f} | F1: {val_f1:.4f}") wandb.log({ "epoch": epoch + 1, "train_loss": train_loss, "train_acc": train_acc, "train_f1": train_f1, "val_loss": val_loss, "val_acc": val_acc, "val_f1": val_f1 }) if val_f1 > best_f1: best_f1 = val_f1 model.save_pretrained("best_model") tokenizer.save_pretrained("best_model") print(f"✅ Best model saved! F1: {best_f1:.4f}") # ── CONFUSION MATRIX ─────────────────── print("\nGenerating confusion matrix...") _, _, _, final_preds, final_labels = evaluate(model, test_loader, criterion) cm = confusion_matrix(final_labels, final_preds) plt.figure(figsize=(20, 20)) sns.heatmap(cm, annot=False, fmt="d", cmap="Blues") plt.title("Confusion Matrix — Banking77") plt.ylabel("True Label") plt.xlabel("Predicted Label") plt.tight_layout() plt.savefig("confusion_matrix.png") print("✅ Saved confusion_matrix.png") wandb.log({"confusion_matrix": wandb.Image("confusion_matrix.png")}) wandb.finish() print(f"\n🎉 Training complete! Best F1: {best_f1:.4f}")