Improved Logistic Regression with hyperparameter tuning and TF-IDF enhancements

logistic_reg.py

@@ -1,11 +1,12 @@
 from datasets import load_dataset
 import pandas as pd
-import numpy as np
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.preprocessing import MultiLabelBinarizer
-from sklearn.metrics import hamming_loss, f1_score, classification_report
-import xgboost as xgb
-from joblib import dump, load
+from sklearn.linear_model import LogisticRegression
+from sklearn.multiclass import OneVsRestClassifier
+from sklearn.metrics import classification_report, hamming_loss
+from sklearn.model_selection import GridSearchCV
+from joblib import dump
 
 # Step 1: Load the Dataset Repository
 dataset = load_dataset("meriemm6/commit-classification-dataset", data_files={"train": "training.csv", "validation": "validation.csv"})
@@ -27,43 +28,44 @@ validation_data['Ground truth'] = validation_data['Ground truth'].fillna("mainte
 train_data['Ground truth'] = train_data['Ground truth'].apply(lambda x: x.split(', '))
 validation_data['Ground truth'] = validation_data['Ground truth'].apply(lambda x: x.split(', '))
 
-# Encode the labels
-mlb = MultiLabelBinarizer()
-y_train_encoded = mlb.fit_transform(train_data['Ground truth'])
-y_val_encoded = mlb.transform(validation_data['Ground truth'])
-
-# Step 3: TF-IDF Vectorization (Increased Features)
-tfidf_vectorizer = TfidfVectorizer(max_features=10000, stop_words="english")
+# Step 3: TF-IDF Vectorization (Enhanced Features)
+tfidf_vectorizer = TfidfVectorizer(max_features=10000, stop_words='english', ngram_range=(1, 2))
 X_train_tfidf = tfidf_vectorizer.fit_transform(train_data['Message'])
 X_val_tfidf = tfidf_vectorizer.transform(validation_data['Message'])
 
+# Step 4: MultiLabel Encoding
+mlb = MultiLabelBinarizer()
+y_train_encoded = mlb.fit_transform(train_data['Ground truth'])
+y_val_encoded = mlb.transform(validation_data['Ground truth'])
 
+# Step 5: Hyperparameter Tuning for Logistic Regression
+log_reg = LogisticRegression(class_weight='balanced', max_iter=5000, random_state=42)
+multi_log_reg = OneVsRestClassifier(log_reg)
 
-# Save the TF-IDF vectorizer
-dump(tfidf_vectorizer, "tfidf_vectorizer_xgboost.joblib")
+param_grid = {
+    'estimator__C': [0.1, 1, 10],  # Regularization strength
+    'estimator__solver': ['lbfgs', 'liblinear'],  # Optimizers
+}
+grid_search = GridSearchCV(
+    estimator=multi_log_reg,
+    param_grid=param_grid,
+    scoring='f1_weighted',
+    cv=3,
+    verbose=2,
+    n_jobs=-1
+)
+grid_search.fit(X_train_tfidf, y_train_encoded)
+best_model = grid_search.best_estimator_
 
-# Step 4: Add Class Weighting
-label_counts = y_train_encoded.sum(axis=0)
-scale_pos_weight = (len(y_train_encoded) - label_counts) / label_counts
+# Step 6: Validation Metrics
+y_val_pred = best_model.predict(X_val_tfidf)
+print("Validation Metrics:")
+print(f"F1 Score: {classification_report(y_val_encoded, y_val_pred, target_names=mlb.classes_, zero_division=0)}")
+print(f"Hamming Loss: {hamming_loss(y_val_encoded, y_val_pred):.4f}")
 
-# Step 5: Train XGBoost Models with Class Weighting and Dynamic Parameters
-models = []
-for i in range(y_train_encoded.shape[1]):
-    model = xgb.XGBClassifier(
-        objective="binary:logistic",
-        use_label_encoder=False,
-        eval_metric="logloss",
-        scale_pos_weight=scale_pos_weight[i],  # Class weights
-        max_depth=6,  # Reduced to prevent overfitting
-        learning_rate=0.03,  # Lower learning rate for better generalization
-        n_estimators=300,  # Increased estimators for better performance
-        subsample=0.8,
-        colsample_bytree=0.8,
-        min_child_weight=1  # Prevents overfitting on small datasets
-    )
-    model.fit(X_train_tfidf, y_train_encoded[:, i])
-    models.append(model)
+# Step 7: Save the Model and Preprocessing Artifacts
+dump(best_model, "optimized_logistic_model.joblib")  # Save the optimized Logistic Regression model
+dump(tfidf_vectorizer, "tfidf_vectorizer.joblib")  # Save the TF-IDF vectorizer
+dump(mlb, "label_binarizer.joblib")  # Save the MultiLabelBinarizer
 
-# Save the models
-for idx, model in enumerate(models):
-    dump(model, f"xgboost_model_label_{idx}.joblib")
+print("Optimized model and preprocessing files saved successfully.")