File size: 5,087 Bytes
e69d4e4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | """Perform cross validation using a variety of algorithms."""
import os
import pandas as pd
import numpy as np
from lenusml import splits, plots
# Model training and evaluation
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_validate, cross_val_predict
from imblearn.ensemble import BalancedRandomForestClassifier, BalancedBaggingClassifier
from interpret.glassbox import ExplainableBoostingClassifier
import lightgbm as lgb
import xgboost as xgb
import mlflow
data_dir = '../data/models/model1/'
cohort_info_dir = '../data/cohort_info/'
output_dir = '../data/models/model1/output'
# Load CV folds and train data
fold_patients = np.load(os.path.join(cohort_info_dir, 'fold_patients.npy'),
allow_pickle=True)
train_data = pd.read_pickle(os.path.join(data_dir, 'train_data_cv.pkl'))
# Cross check fold patients with train data
cross_validation_fold_indices = splits.custom_cv_fold_indices(fold_patients=fold_patients,
train_data=train_data,
id_column='StudyId')
# Create list of model features
cols_to_drop = ['StudyId', 'IsExac']
features_list = [col for col in train_data.columns if col not in cols_to_drop]
# Separate features from target
features = train_data[features_list].astype('float')
target = train_data.IsExac.astype('float')
scale_pos_weight = target.value_counts()[0] / target.value_counts()[1]
mlflow.set_tracking_uri("sqlite:///mlruns.sqlite")
mlflow.set_experiment('model_drop2')
# Set CV scoring strategies and any model parameters
scoring = ['f1', 'balanced_accuracy', 'accuracy', 'precision', 'recall', 'roc_auc',
'average_precision', 'neg_brier_score']
scale_pos_weight = target.value_counts()[0] / target.value_counts()[1]
models = []
models.append((RandomForestClassifier(random_state=0), 'random_forest'))
models.append((RandomForestClassifier(random_state=0, class_weight='balanced'),
'random_forest_class_weight'))
models.append((BalancedBaggingClassifier(random_state=0),
'balanced_bagging'))
models.append((BalancedRandomForestClassifier(random_state=0), 'balanced_random_forest'))
models.append((xgb.XGBClassifier(random_state=0, use_label_encoder=False,
eval_metric='logloss'), 'xgb'))
models.append((lgb.LGBMClassifier(random_state=0), 'lgbm'))
models.append((xgb.XGBClassifier(random_state=0, use_label_encoder=False,
eval_metric='logloss', scale_pos_weight=scale_pos_weight), 'xgb_spw'))
models.append((ExplainableBoostingClassifier(random_state=0), 'ebm'))
with mlflow.start_run(run_name='model_selection'):
# Perform K-fold cross validation with custom folds
for model in models:
with mlflow.start_run(run_name=model[1], nested=True):
# Create the artifacts directory if it doesn't exist
artifact_dir = './tmp'
os.makedirs(artifact_dir, exist_ok=True)
# Remove any existing directory contents to not mix files between different
# runs
for f in os.listdir(artifact_dir):
os.remove(os.path.join(artifact_dir, f))
crossval = cross_validate(model[0], features, target,
cv=cross_validation_fold_indices,
return_estimator=True, scoring=scoring)
# Get the predicted probabilities from each models
probabilities_cv = cross_val_predict(model[0], features, target,
cv=cross_validation_fold_indices,
method='predict_proba')[:, 1]
model_scores = pd.DataFrame({'model_score': probabilities_cv,
'true_label': target})
# Log metrics averaged across folds
for score in scoring:
mlflow.log_metric(score, crossval['test_' + score].mean())
# Log model parameters
params = model[0].get_params()
for param in params:
mlflow.log_param(param, params[param])
plots.plot_lift_curve(scores=model_scores, savefig=True,
output_dir=artifact_dir, figname='lift_curve.png')
plots.plot_cumulative_gains_curve(scores=model_scores, savefig=True,
output_dir=artifact_dir,
figname='cumulative_gains_curve.png')
# Plot distribution of model scores (histogram plus KDE)
plots.plot_score_distribution(scores=model_scores, postive_class_name='Exac',
negative_class_name='No exac', savefig=True,
output_dir=artifact_dir,
figname='model_score_distribution.png')
# Log artifacts
mlflow.log_artifacts(artifact_dir)
mlflow.end_run()
|