Update src/preprocess_utils.py

src/preprocess_utils.py

@@ -2133,6 +2133,13 @@ def preprocess_pipeline(df, target_col=None) -> pd.DataFrame:
         "HLA_mismatch_weighted",
         "OS_time_days",
         "OS_time_months",
+        "CD34_DOSE_X10E6_PER_KG",
+        "CMV_RECIPIENT_POSITIVE",
+        "CMV_DONOR_POSITIVE",
+        "ALEMTUZUMAB_USED",
+        "ATG_USED",
+        "T_CELL_DEPLETION",
+        "CD34_SELECTION",
     }
     
     num_fill_cols = [c for c in num_cols if c not in keep_nan_numeric]
@@ -2180,43 +2187,7 @@ def preprocess_pipeline(df, target_col=None) -> pd.DataFrame:
         df["Hematological Diagnosis_Grouped"] = np.nan
         df["Hematological Diagnosis_Malignant"] = np.nan
 
-    if columns_have_enough_data(
-        df,
-        ["Donor_gender", "Recipient_gender", "Source of cells", "Donor_type", "GVHD_Prophylaxis_Cat", "Conditioning_intensity"],
-        min_non_null_fraction=0.05
-    ):
-        df = add_chronic_gvhd_features(df)
-        important_sparse_cols = [
-            "HLAmatchratio",
-            "HLA_Class_I_mismatch",
-            "HLA_Class_II_mismatch",
-            "HLA_total_match",
-            "HLA_total_mismatch",
-            "Donor_type",
-            "Conditioning_intensity",
-            "GVHD_Prophylaxis_Cat",
-            "Source of cells",
-            "Recipient_Blood group before HSCT_MergePlusMinus",
-            "D_Blood group_MergePlusMinus",
-            "R_Age_at_transplant",
-            "D_Age_at_transplant",
-        ]
-        df = add_missing_indicators(df, important_sparse_cols)
-    else:
-        chronic_cols = [
-            "Female_to_Male",
-            "PBSC",
-            "BONE_MARROW",
-            "CORD",
-            "Older_Donor",
-            "Older_Recipient",
-            "HLA_mismatch_weighted",
-            "DonorType_Source",
-            "DonorType_Proph",
-            "DonorType_Conditioning",
-        ]
-        for c in chronic_cols:
-            df[c] = np.nan
+    
     
     # Do NOT globally replace UNKNOWN_TOKEN across all columns
     # Only coerce numeric columns where needed
@@ -2224,9 +2195,29 @@ def preprocess_pipeline(df, target_col=None) -> pd.DataFrame:
         if pd.api.types.is_numeric_dtype(df[col]):
             df[col] = pd.to_numeric(df[col], errors="coerce")
 
-       
+    # Always build engineered features; add_chronic_gvhd_features already has
+    # per-feature >=5% guards, so a sparse column no longer nulls the whole block.
+    df = add_chronic_gvhd_features(df)
+    important_sparse_cols = [
+        "HLAmatchratio",
+        "HLA_Class_I_mismatch",
+        "HLA_Class_II_mismatch",
+        "HLA_total_match",
+        "HLA_total_mismatch",
+        "Donor_type",
+        "Conditioning_intensity",
+        "GVHD_Prophylaxis_Cat",
+        "Source of cells",
+        "Recipient_Blood group before HSCT_MergePlusMinus",
+        "D_Blood group_MergePlusMinus",
+        "R_Age_at_transplant",
+        "D_Age_at_transplant",
+    ]
+    df = add_missing_indicators(df, important_sparse_cols)   
+    
+    print("POST-FIX check:",
+          df[["PBSC", "BONE_MARROW", "CORD"]].notna().mean().round(3).to_dict())
     
-
     # Ensure newly added categorical model features exist with sensible defaults
     categorical_defaults = {
         "Recipient_gender": UNKNOWN_TOKEN,