import os import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import torch import logging import argparse import numpy as np import random from sklearn.model_selection import train_test_split from sklearn.preprocessing import LabelEncoder from sklearn.utils.class_weight import compute_class_weight from sklearn.metrics import accuracy_score, precision_recall_fscore_support from transformers import ( AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer, EarlyStoppingCallback ) from torch.utils.data import Dataset import torch.nn as nn from src.data_loader import load_and_preprocess_data # ============================= # CONFIG # ============================= MODEL_NAME = "google/muril-base-cased" MAX_LEN = 192 OUTPUT_DIR = "model_output" SEED = 42 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger(__name__) # ============================= # REPRODUCIBILITY # ============================= def set_seed(seed=SEED): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) # ============================= # DATASET CLASS # ============================= class ToxicDataset(Dataset): def __init__(self, encodings, labels): self.encodings = encodings self.labels = labels def __getitem__(self, idx): item = {k: torch.tensor(v[idx]) for k, v in self.encodings.items()} item["labels"] = torch.tensor(self.labels[idx]) return item def __len__(self): return len(self.labels) # ============================= # METRICS # ============================= def compute_metrics(pred): labels = pred.label_ids preds = pred.predictions.argmax(-1) precision, recall, f1, _ = precision_recall_fscore_support( labels, preds, average="macro", zero_division=0 ) acc = accuracy_score(labels, preds) return { "accuracy": acc, "macro_f1": f1, "macro_precision": precision, "macro_recall": recall } # ============================= # WEIGHTED TRAINER # ============================= class WeightedTrainer(Trainer): def __init__(self, class_weights=None, *args, **kwargs): super().__init__(*args, **kwargs) self.class_weights = class_weights def compute_loss(self, model, inputs, return_outputs=False, **kwargs): labels = inputs.get("labels") outputs = model(**inputs) logits = outputs.get("logits") loss_fct = nn.CrossEntropyLoss( weight=self.class_weights.to(logits.device) ) loss = loss_fct( logits.view(-1, model.config.num_labels), labels.view(-1) ) return (loss, outputs) if return_outputs else loss # ============================= # TRAIN FUNCTION # ============================= def train_model( data_path=".", epochs=5, train_batch_size=16, eval_batch_size=16, smoke_test=False ): set_seed() logger.info("Loading dataset...") df = load_and_preprocess_data(data_path, augment=False) # Safety check logger.info(f"Total rows: {len(df)}") logger.info(f"Unique cleaned: {df['cleaned_text'].nunique()}") logger.info(f"Label distribution:\n{df['label'].value_counts()}") if len(df) < 2000: raise ValueError("Dataset too small for transformer training.") # Label encoding le = LabelEncoder() df["label_encoded"] = le.fit_transform(df["label"]) # Save label encoder later num_labels = len(le.classes_) # Split X_train, X_val, y_train, y_val = train_test_split( df["cleaned_text"].tolist(), df["label_encoded"].tolist(), test_size=0.2, stratify=df["label_encoded"], random_state=SEED ) if smoke_test: logger.warning("Running smoke test mode") X_train, y_train = X_train[:100], y_train[:100] X_val, y_val = X_val[:30], y_val[:30] epochs = 1 logger.info(f"Training samples: {len(X_train)}") logger.info(f"Validation samples: {len(X_val)}") # Tokenizer tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) train_encodings = tokenizer( X_train, truncation=True, padding=True, max_length=MAX_LEN ) val_encodings = tokenizer( X_val, truncation=True, padding=True, max_length=MAX_LEN ) train_dataset = ToxicDataset(train_encodings, y_train) val_dataset = ToxicDataset(val_encodings, y_val) # Model model = AutoModelForSequenceClassification.from_pretrained( MODEL_NAME, num_labels=num_labels ) # Class weights class_weights = compute_class_weight( class_weight="balanced", classes=np.unique(df["label_encoded"]), y=df["label_encoded"] ) class_weights = torch.tensor(class_weights, dtype=torch.float) # Training arguments training_args = TrainingArguments( output_dir=OUTPUT_DIR, num_train_epochs=epochs, per_device_train_batch_size=train_batch_size, per_device_eval_batch_size=eval_batch_size, eval_strategy="epoch", save_strategy="epoch", load_best_model_at_end=True, metric_for_best_model="macro_f1", greater_is_better=True, weight_decay=0.01, warmup_steps=100, logging_steps=20, save_total_limit=2, seed=SEED, fp16=torch.cuda.is_available(), report_to=[] ) trainer = WeightedTrainer( model=model, args=training_args, train_dataset=train_dataset, eval_dataset=val_dataset, compute_metrics=compute_metrics, class_weights=class_weights, callbacks=[EarlyStoppingCallback(early_stopping_patience=2)] ) logger.info("Starting training...") trainer.train() logger.info("Saving model...") trainer.save_model(OUTPUT_DIR) tokenizer.save_pretrained(OUTPUT_DIR) import joblib joblib.dump(le, os.path.join(OUTPUT_DIR, "label_encoder.joblib")) logger.info("Training complete. Model saved.") # ============================= # ENTRYPOINT # ============================= if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--epochs", type=int, default=5) parser.add_argument("--batch_size", type=int, default=16) parser.add_argument("--smoke_test", action="store_true") args = parser.parse_args() train_model( epochs=args.epochs, train_batch_size=args.batch_size, smoke_test=args.smoke_test )