Upload fibropred_model.py

scripts/fibropred_model.py

@@ -9,12 +9,18 @@ from sklearn.metrics import classification_report, accuracy_score, roc_curve, au
 from sklearn.feature_selection import SelectFromModel
 import matplotlib.pyplot as plt
 import seaborn as sns
+from scipy.stats import zscore
+from imblearn.over_sampling import SMOTE
+import os
+os.environ["LOKY_MAX_CPU_COUNT"] = "4"  # Cambia "4" por el número de núcleos deseado
+
 
 # Load dataset
 def load_data(file_path):
     df = pd.read_excel(file_path, header=1)
     return df
 
+
 # Preprocess data including categorical variables
 def preprocess_data_with_categoricals(df):
     # Replace -9 with NaN for missing values
@@ -24,10 +30,6 @@ def preprocess_data_with_categoricals(df):
     missing_percentage = df.isnull().sum() / len(df) * 100
     df = df.drop(columns=missing_percentage[missing_percentage > 50].index)
 
-    # Drop specific columns
-    drop_columns = ['ProgressiveDisease', 'Final diagnosis', 'Transplantation date', 'Cause of death', 'Date of death', 'COD NUMBER']
-    df = df.drop(columns=[col for col in drop_columns if col in df.columns])
-    
     # Impute missing values
     imputer = SimpleImputer(strategy='median')
     numeric_cols = df.select_dtypes(include=['number']).columns
@@ -59,15 +61,22 @@ def apply_one_hot_encoding(df):
     categorical_cols = df.select_dtypes(include=['object']).columns
     df = pd.get_dummies(df, columns=categorical_cols, drop_first=True)
     return df
+# Remove outliers based on Z-score
+def remove_outliers(df, numeric_cols, z_threshold=4):
+    for col in numeric_cols:
+        z_scores = zscore(df[col])
+        df = df[(np.abs(z_scores) < z_threshold) | (pd.isnull(z_scores))]
+    return df
 
 # Select predictors using feature importance
 def select_important_features(X, y, threshold=0.03):
     model = RandomForestClassifier(random_state=42)
     model.fit(X, y)
     selector = SelectFromModel(model, threshold=threshold, prefit=True)
-    X_reduced = selector.transform(X)
-    selected_features = X.columns[selector.get_support()]
-    return pd.DataFrame(X_reduced, columns=selected_features), selected_features
+    selected_mask = selector.get_support()
+    selected_features = X.columns[selected_mask]
+    X_reduced = X.loc[:, selected_features]
+    return X_reduced, selected_features
 
 # Visualize feature importance
 def plot_feature_importance(model, features, target):
@@ -122,33 +131,56 @@ def plot_roc_auc(model, X_test, y_test, target):
 
 # Save trained model
 def save_model(model, target, selected_features):
-
     if not os.path.exists("models"):
         os.makedirs("models")
     file_name = f"models/{target}_random_forest_model.pkl"
     joblib.dump({'model': model, 'features': selected_features}, file_name)
     print(f"Model and features saved to {file_name}")
 
-
 # Main pipeline
 def main():
     file_path = 'FibroPredCODIFICADA.xlsx'
     df = load_data(file_path)
 
-    # Target columns
-    target_columns = ['Death', 'Progressive disease', 'Necessity of transplantation']
+    # Include 'ProgressiveDisease' in target columns
+    target_columns = ['Death', 'Binary diagnosis', 'Necessity of transplantation', 'Progressive disease']
+
+    # Define predictors to remove for each target
+    predictors_to_remove_dict = {
+        'Death': ['Final diagnosis', 'Transplantation date', 'Cause of death', 'Date of death', 'COD NUMBER','FVC (L) 1 year after diagnosis',
+                  'FVC (%) 1 year after diagnosis','DLCO (%) 1 year after diagnosis'],
+        'Binary diagnosis': ['ProgressiveDisease', 'Final diagnosis', 'Transplantation date', 'Cause of death', 'Date of death', 'COD NUMBER','Pirfenidone','Nintedanib',
+                             'Antifibrotic Drug','Prednisone','Mycophenolate','FVC (L) 1 year after diagnosis','FVC (%) 1 year after diagnosis',
+                             'DLCO (%) 1 year after diagnosis','RadioWorsening2y'],
+        'Necessity of transplantation': ['ProgressiveDisease', 'Final diagnosis', 'Transplantation date', 'Cause of death', 'Date of death', 'COD NUMBER','Age at diagnosis'],
+        'Progressive disease': ['ProgressiveDisease', 'Final diagnosis', 'Transplantation date', 'Cause of death', 'Date of death', 'COD NUMBER', 'FVC (L) 1 year after diagnosis',
+                               'FVC (%) 1 year after diagnosis','DLCO (%) 1 year after diagnosis','RadioWorsening2y']
+    }
 
     # Preprocess data
     df, numeric_cols, categorical_cols = preprocess_data_with_categoricals(df)
 
     for target in target_columns:
         print(f"Processing target: {target}")
-        X = df[numeric_cols].drop(columns=target_columns, errors='ignore')  # Ensure target variables are excluded
+        # Apply outlier removal only for specific targets
+        if target in ['Necessity of transplantation', 'Progressive disease']:
+            print(f"Removing outliers for target: {target}")
+            df = remove_outliers(df, numeric_cols)
+
+        # Get predictors to remove for the current target
+        predictors_to_remove = predictors_to_remove_dict.get(target, [])
+
+        X = df[numeric_cols].drop(columns=target_columns + predictors_to_remove, errors='ignore')
         y = df[target]
 
         # Split data
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 
+        # Apply SMOTE only for specific targets
+        if target in ['Binary diagnosis', 'Necessity of transplantation']:
+            print(f"Applying SMOTE to balance the training set for target: {target}")
+            smote = SMOTE(random_state=42)
+            X_train, y_train = smote.fit_resample(X_train, y_train)
         # Select important features
         X_train_selected, selected_features = select_important_features(X_train, y_train)
         X_test_selected = X_test[selected_features]
@@ -166,9 +198,9 @@ def main():
         model.fit(X_train_selected, y_train)
 
         # Cross-validation to check overfitting
-        cv = StratifiedKFold(n_splits=15, shuffle=True, random_state=42)
+        cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
         cv_scores = cross_val_score(model, X_train_selected, y_train, cv=cv, scoring='accuracy')
-        train_scores = cross_val_score(model, X_train_selected, y_train, cv=15, scoring='accuracy')
+        train_scores = cross_val_score(model, X_train_selected, y_train, cv=10, scoring='accuracy')
         y_pred_test = model.predict(X_test_selected)
         test_score = accuracy_score(y_test, y_pred_test)