initial commit

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+src/saved_models/*.pkl filter=lfs diff=lfs merge=lfs -text

requirements.txt

@@ -1,3 +1,12 @@
-altair
-pandas
-streamlit
+catboost==1.2.8
+huggingface_hub==0.33.2
+numpy==1.26.4
+pandas==2.3.0
+pyarrow==16.1.0
+PyYAML==6.0.2
+scikit_learn==1.5.1
+streamlit==1.46.1
+
+# altair
+# pandas
+# streamlit

src/GVHD_Predictions_App.py

@@ -0,0 +1,4 @@
+import streamlit as st
+
+st.set_page_config(page_title="GVHD Predictions", layout="wide")
+st.title("GVHD Predictions App")

src/inference_utils.py

@@ -0,0 +1,38 @@
+import pandas as pd
+import streamlit as st
+from sklearn.metrics import roc_auc_score, f1_score, accuracy_score, precision_score, recall_score, brier_score_loss, log_loss
+
+def compute_metrics(y_true, y_pred_proba, threshold=0.5):
+    y_pred = (y_pred_proba >= threshold).astype(int)
+    return {
+        "AUC": roc_auc_score(y_true, y_pred_proba),
+        "F1": f1_score(y_true, y_pred),
+        "Accuracy": accuracy_score(y_true, y_pred),
+        "Precision": precision_score(y_true, y_pred),
+        "Recall": recall_score(y_true, y_pred),
+        "BrierScore": brier_score_loss(y_true, y_pred_proba),
+        "Logloss": log_loss(y_true, y_pred_proba),
+    }
+
+def add_predictions(df, probs):
+    df['Predicted Probability'] = probs
+    df['GVHD Prediction'] = ['POSITIVE' if p > 0.5 else 'NEGATIVE' for p in probs]
+
+    df_with_gt = df[['Predicted Probability', 'GVHD Prediction']].join(st.session_state.targets_df)
+    
+    # Define cell-level styling
+    def highlight_prediction(val):
+        if val == "POSITIVE":
+            return "background-color: #d4edda; color: #155724; text-align: center;"
+        elif val == "NEGATIVE":
+            return "background-color: #f8d7da; color: #721c24; text-align: center;"
+        return "text-align: center;"
+
+    # Apply color and alignment
+    df_styled = (
+        df_with_gt.style
+        .applymap(highlight_prediction, subset=["GVHD Prediction"])
+        .set_properties(**{'text-align': 'center'})  # Apply center alignment to all cells
+    )
+
+    return df_styled

src/model_utils.py

@@ -0,0 +1,304 @@
+import streamlit as st
+import pickle
+from datetime import datetime
+from pathlib import Path
+
+from catboost import CatBoostClassifier
+# from xgboost import XGBClassifier
+# from lightgbm import LGBMClassifier
+from sklearn.ensemble import RandomForestClassifier
+
+import json
+import uuid
+import io
+from datetime import datetime
+from typing import Any, Dict, Optional
+import pickle
+import pyarrow as pa
+import pyarrow.parquet as pq
+from huggingface_hub import CommitScheduler
+
+MODEL_DIR = Path("saved_models")
+MODEL_DIR.mkdir(exist_ok=True)
+
+import yaml
+
+def load_model_params(model_type, mode="ensemble", path=Path("params") / "model_params.yaml"):
+    if mode not in ["ensemble", "single_model"]:
+        raise ValueError("mode must be either 'ensemble' or 'single_model'")
+
+    if model_type not in ["CatBoost", "XGBoost", "LightGBM", "RandomForest"]:
+        raise ValueError("model_type must be one of 'CatBoost', 'XGBoost', 'LightGBM', or 'RandomForest'")
+
+    with open(path, "r") as f:
+        all_params = yaml.safe_load(f)
+
+    params = all_params[model_type][mode]
+    if "random_seed" in params:
+        st.session_state.random_seed = params["random_seed"]
+
+    return params
+
+def get_model(model_type, mode="ensemble", best_iter=None):
+    params = load_model_params(model_type, mode)
+
+    # iter is set for single_model mode, where 
+    if best_iter is not None:
+        params['iterations'] = best_iter
+    # if "random_seed" in st.session_state:
+    #     random_seed = st.session_state.random_seed
+
+    if model_type == "CatBoost":
+        return CatBoostClassifier(**params)
+    # elif model_type == "XGBoost":
+    #     return XGBClassifier(**params, use_label_encoder=False, eval_metric="logloss")
+    # elif model_type == "LightGBM":
+    #     return LGBMClassifier(**params)
+    elif model_type == "RandomForest":
+        return RandomForestClassifier(**params)
+    else:
+        raise ValueError(f"Unsupported model type: {model_type}")
+
+# def save_model(model, user_model_name, metrics_result_single=None):
+#     timestamp = datetime.now().strftime("%y%m%d_%H%M%S")
+#     filename = f"{timestamp}_{user_model_name}_single.pkl"
+#     filepath = MODEL_DIR / filename
+
+#     single_model_data = {
+#         "timestamp": timestamp,
+#         "model_name": user_model_name,
+#         "target_col": st.session_state.target_col if "target_col" in st.session_state else "UNKNOWN",
+#         "model": model,
+#         "best_iteration": st.session_state.best_iteration,
+#         "metrics_result_single": metrics_result_single
+#     }
+
+#     with open(filepath, "wb") as f:
+#         pickle.dump(single_model_data, f)
+#     return filename
+
+def save_model(model, user_model_name, metrics_result_single=None):
+    from datetime import datetime
+    import io
+    import uuid
+    import pickle
+    import json
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+    from huggingface_hub import CommitScheduler
+
+    timestamp = datetime.now().strftime("%y%m%d_%H%M%S")
+    filename = f"{timestamp}_{user_model_name}_single.pkl"
+
+    # Prepare model dict (same as before)
+    model_data = {
+        "timestamp": timestamp,
+        "model_name": user_model_name,
+        "target_col": st.session_state.get("target_col", "UNKNOWN"),
+        "model": model,
+        "best_iteration": st.session_state.get("best_iteration"),
+        "metrics_result_single": metrics_result_single,
+    }
+
+    # Serialize (pickle) to bytes
+    model_bytes = pickle.dumps(model_data)
+
+    # Prepare Parquet row
+    row = {
+        "filename": filename,
+        "timestamp": timestamp,
+        "type": "single",
+        "model_file": {"path": filename, "bytes": model_bytes},
+    }
+
+    table = pa.Table.from_pylist([row])
+    table = table.replace_schema_metadata({
+        "huggingface": json.dumps({"info": {
+            "features": {
+                "filename": {"_type": "Value", "dtype": "string"},
+                "timestamp": {"_type": "Value", "dtype": "string"},
+                "type": {"_type": "Value", "dtype": "string"},
+                "model_file": {"_type": "Value", "dtype": "binary"},
+            }
+        }})
+    })
+
+    # Write to in-memory buffer
+    buf = io.BytesIO()
+    pq.write_table(table, buf)
+    buf.seek(0)
+
+    # Upload to HF dataset
+    scheduler = CommitScheduler(
+        repo_id=st.secrets["HF_REPO_ID"],
+        repo_type="dataset",
+        path_in_repo="models",
+        token=st.secrets["HF_TOKEN"],
+        private=True,
+        folder_path="dummy"
+    )
+    scheduler.api.upload_file(
+        repo_id=st.secrets["HF_REPO_ID"],
+        repo_type="dataset",
+        path_in_repo=f"models/{uuid.uuid4()}.parquet",
+        path_or_fileobj=buf
+    )
+
+    return filename
+
+# def save_model_ensemble(models, user_model_name, best_iterations=None, fold_scores=None, metrics_result_ensemble=None):
+#     timestamp = datetime.now().strftime("%y%m%d_%H%M%S")
+#     filename = f"{timestamp}_{user_model_name}_ensemble.pkl"
+#     filepath = MODEL_DIR / filename
+
+#     ensemble_data = {
+#         "timestamp": timestamp,
+#         "model_name": user_model_name,
+#         "target_col": st.session_state.target_col if "target_col" in st.session_state else "UNKNOWN",
+#         "models": models,
+#         "best_iterations": best_iterations,
+#         "fold_scores": fold_scores,
+#         "metrics_result_ensemble": metrics_result_ensemble
+#     }
+
+#     with open(filepath, "wb") as f:
+#         pickle.dump(ensemble_data, f)
+#     return filename
+
+def save_model_ensemble(models, user_model_name, best_iterations=None, fold_scores=None, metrics_result_ensemble=None):
+    from datetime import datetime
+    import io
+    import uuid
+    import pickle
+    import json
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+    from huggingface_hub import CommitScheduler
+
+    timestamp = datetime.now().strftime("%y%m%d_%H%M%S")
+    filename = f"{timestamp}_{user_model_name}_ensemble.pkl"
+
+    ensemble_data = {
+        "timestamp": timestamp,
+        "model_name": user_model_name,
+        "target_col": st.session_state.get("target_col", "UNKNOWN"),
+        "models": models,
+        "best_iterations": best_iterations,
+        "fold_scores": fold_scores,
+        "metrics_result_ensemble": metrics_result_ensemble,
+    }
+
+    model_bytes = pickle.dumps(ensemble_data)
+
+    row = {
+        "filename": filename,
+        "timestamp": timestamp,
+        "type": "ensemble",
+        "model_file": {"path": filename, "bytes": model_bytes},
+    }
+
+    table = pa.Table.from_pylist([row])
+    table = table.replace_schema_metadata({
+        "huggingface": json.dumps({"info": {
+            "features": {
+                "filename": {"_type": "Value", "dtype": "string"},
+                "timestamp": {"_type": "Value", "dtype": "string"},
+                "type": {"_type": "Value", "dtype": "string"},
+                "model_file": {"_type": "Value", "dtype": "binary"},
+            }
+        }})
+    })
+
+    buf = io.BytesIO()
+    pq.write_table(table, buf)
+    buf.seek(0)
+
+    scheduler = CommitScheduler(
+        repo_id=st.secrets["HF_REPO_ID"],
+        repo_type="dataset",
+        path_in_repo="models",
+        token=st.secrets["HF_TOKEN"],
+        private=True,
+        folder_path="dummy"
+    )
+    scheduler.api.upload_file(
+        repo_id=st.secrets["HF_REPO_ID"],
+        repo_type="dataset",
+        path_in_repo=f"models/{uuid.uuid4()}.parquet",
+        path_or_fileobj=buf
+    )
+
+    return filename
+
+
+# def load_model(model_name):
+#     filepath = MODEL_DIR / f"{model_name}.pkl"
+#     if not filepath.exists():
+#         raise FileNotFoundError(f"Model file not found: {filepath}")
+
+#     with open(filepath, "rb") as f:
+#         single_model_data = pickle.load(f)
+
+#     return single_model_data
+
+def load_model(model_name):
+    from huggingface_hub import hf_hub_download
+    import pyarrow.parquet as pq
+    import pickle
+
+    files = hf_hub_download(
+        repo_id=st.secrets["HF_REPO_ID"],
+        repo_type="dataset",
+        token=st.secrets["HF_TOKEN"],
+        filename=None,  # Get whole repo listing
+        cache_dir=None,
+        local_dir=None,
+        local_dir_use_symlinks=False,
+        force_download=False,
+        resume_download=True
+    )
+
+    from huggingface_hub import HfApi
+    api = HfApi(token=st.secrets["HF_TOKEN"])
+    all_files = api.list_repo_files(repo_id=st.secrets["HF_REPO_ID"], repo_type="dataset")
+    model_files = [f for f in all_files if f.startswith("models/") and f.endswith(".parquet")]
+
+    # Find matching filename
+    target_file = None
+    for f in model_files:
+        downloaded = hf_hub_download(
+            repo_id=st.secrets["HF_REPO_ID"],
+            repo_type="dataset",
+            filename=f,
+            token=st.secrets["HF_TOKEN"]
+        )
+        table = pq.read_table(downloaded)
+        row = table.to_pylist()[0]
+        if row["filename"] == model_name:
+            target_file = downloaded
+            break
+
+    if not target_file:
+        raise FileNotFoundError(f"Model {model_name} not found in repo.")
+
+    model_bytes = row["model_file"]["bytes"]
+    return pickle.loads(model_bytes)
+
+
+# def load_model_ensemble(filename):
+#     filepath = MODEL_DIR / f"{filename}.pkl"
+#     if not filepath.exists():
+#         raise FileNotFoundError(f"Model file not found: {filepath}")
+
+#     with open(filepath, "rb") as f:
+#         ensemble_data = pickle.load(f)
+
+#     return ensemble_data
+
+def load_model_ensemble(filename):
+    return load_model(filename)
+
+
+def ensemble_predict(models, X, cat_features):
+    preds = sum([model.predict_proba(X)[:, 1] for model in models]) / len(models)
+    return preds

src/model_utils_ori.py

@@ -0,0 +1,114 @@
+import streamlit as st
+import pickle
+import catboost
+from datetime import datetime
+from pathlib import Path
+
+from catboost import CatBoostClassifier
+# from xgboost import XGBClassifier
+# from lightgbm import LGBMClassifier
+from sklearn.ensemble import RandomForestClassifier
+
+MODEL_DIR = Path("saved_models")
+MODEL_DIR.mkdir(exist_ok=True)
+
+import yaml
+
+def load_model_params(model_type, mode="ensemble", path=Path("params") / "model_params.yaml"):
+    if mode not in ["ensemble", "single_model"]:
+        raise ValueError("mode must be either 'ensemble' or 'single_model'")
+
+    if model_type not in ["CatBoost", "XGBoost", "LightGBM", "RandomForest"]:
+        raise ValueError("model_type must be one of 'CatBoost', 'XGBoost', 'LightGBM', or 'RandomForest'")
+
+    with open(path, "r") as f:
+        all_params = yaml.safe_load(f)
+
+    params = all_params[model_type][mode]
+    if "random_seed" in params:
+        st.session_state.random_seed = params["random_seed"]
+
+    return params
+
+def get_model(model_type, mode="ensemble", best_iter=None):
+    params = load_model_params(model_type, mode)
+
+    # iter is set for single_model mode, where 
+    if best_iter is not None:
+        params['iterations'] = best_iter
+    # if "random_seed" in st.session_state:
+    #     random_seed = st.session_state.random_seed
+
+    if model_type == "CatBoost":
+        return CatBoostClassifier(**params)
+    # elif model_type == "XGBoost":
+    #     return XGBClassifier(**params, use_label_encoder=False, eval_metric="logloss")
+    # elif model_type == "LightGBM":
+    #     return LGBMClassifier(**params)
+    elif model_type == "RandomForest":
+        return RandomForestClassifier(**params)
+    else:
+        raise ValueError(f"Unsupported model type: {model_type}")
+
+def save_model(model, user_model_name, metrics_result_single=None):
+    timestamp = datetime.now().strftime("%y%m%d_%H%M%S")
+    filename = f"{timestamp}_{user_model_name}_single.pkl"
+    filepath = MODEL_DIR / filename
+
+    single_model_data = {
+        "timestamp": timestamp,
+        "model_name": user_model_name,
+        "target_col": st.session_state.target_col if "target_col" in st.session_state else "UNKNOWN",
+        "model": model,
+        "best_iteration": st.session_state.best_iteration,
+        "metrics_result_single": metrics_result_single
+    }
+
+    with open(filepath, "wb") as f:
+        pickle.dump(single_model_data, f)
+    return filename
+
+def load_model(model_name):
+    filepath = MODEL_DIR / f"{model_name}.pkl"
+    if not filepath.exists():
+        raise FileNotFoundError(f"Model file not found: {filepath}")
+
+    with open(filepath, "rb") as f:
+        single_model_data = pickle.load(f)
+
+    return single_model_data
+
+def save_model_ensemble(models, user_model_name, best_iterations=None, fold_scores=None, metrics_result_ensemble=None):
+    timestamp = datetime.now().strftime("%y%m%d_%H%M%S")
+    filename = f"{timestamp}_{user_model_name}_ensemble.pkl"
+    filepath = MODEL_DIR / filename
+
+    ensemble_data = {
+        "timestamp": timestamp,
+        "model_name": user_model_name,
+        "target_col": st.session_state.target_col if "target_col" in st.session_state else "UNKNOWN",
+        "models": models,
+        "best_iterations": best_iterations,
+        "fold_scores": fold_scores,
+        "metrics_result_ensemble": metrics_result_ensemble
+    }
+
+    with open(filepath, "wb") as f:
+        pickle.dump(ensemble_data, f)
+    return filename
+
+
+def load_model_ensemble(filename):
+    filepath = MODEL_DIR / f"{filename}.pkl"
+    if not filepath.exists():
+        raise FileNotFoundError(f"Model file not found: {filepath}")
+
+    with open(filepath, "rb") as f:
+        ensemble_data = pickle.load(f)
+
+    return ensemble_data
+
+
+def ensemble_predict(models, X, cat_features):
+    preds = sum([model.predict_proba(X)[:, 1] for model in models]) / len(models)
+    return preds

src/pages/1_Individual_Predictions.py

@@ -0,0 +1,153 @@
+import streamlit as st
+import pandas as pd
+from model_utils import load_model, load_model_ensemble, ensemble_predict
+from preprocess_utils import load_train_features
+from preprocess_utils import preprocess_pipeline as preprocess
+from inference_utils import add_predictions
+from sidebar import sidebar
+
+# Initialize sidebar
+sidebar()
+
+st.title("👤 Individual Patient Prediction")
+
+with st.form("individual_form"):
+    st.subheader("Recipient Information", divider=True)
+    gender = st.radio("Recipient Gender", ['MALE', 'FEMALE'], index=None)
+    dob = st.date_input("Recipient DOB", value="2000-01-31", format="DD/MM/YYYY")
+    nationality = st.selectbox("Recipient Nationality", sorted([
+        'EMIRATI', 'EGYPTIAN', 'BANGLADESHI', 'AFGHAN', 'SYRIAN', 'INDIAN', 'PAKISTANI',
+        'YEMENI', 'JORDANIAN', 'OMANI', 'FILIPINO', 'SUDANESE', 'MOROCCAN',
+        'PALESTINIAN', 'ETHIOPIAN', 'AMERICAN', 'ALGERIAN', 'INDONESIAN', 'LEBANESE',
+        'SAUDI', 'SRI LANKAN', 'SOMALI', 'FIJI', 'NEW ZEALANDER', 'COMORAN',
+        'MAURITANIA', 'KUWAIT', 'BRITISH', 'UZBEKISTANI', 'ERITREAN', 'IRAQI'
+    ]), index=None)
+    diagnosis = st.selectbox("Hematological Diagnosis", sorted([
+        'ACUTE MYELOID LEUKEMIA', 'ALPHA THALSSEMIA', 'AMYLOIDOSIS', 'APLASTIC ANEMIA', 'BALL',
+        'BETA THALESSEMIA', 'BLASTIC PLASMACYTOID DENDRITRIC CELL NEOPLASM',
+        'CHRONIC GRANULOMATOUS DISEASE', 'CHRONIC LYMPHOCYTIC LEUKEMIA', 'CML',
+        'COMBINED VARIABLE IMMUNODEFICIENCY', 'DYSKERATOSIS CONGENTIA', 'FANCONI ANEMIA',
+        'GLANZMANN THROMBASTHENIA', 'HEMOPHAGOCYTIC LYMPHOHISTIOCYTOSIS (HLH)',
+        'HEREDITARY SPHEROCYTOSIS', 'HODGKIN LYMPHOMA', 'HYPOGAMMAGLOBULINEMIA',
+        'LANGERHANS CELL HISTIOCYTOSIS', 'MYELODYSPLASTIC SYNDROME', 'MEDULLOBLASTOMA',
+        'MULTIPLE MYELOMA', 'MYELOFIBROSIS', 'MYELOPROLIFERATIVE DISORDER',
+        'NEUROBLASTOMA', 'NON HODGKIN LYMPHOMA', 'OTHER', 'PAROXYSMAL NOCTURNAL HEMOGLOBINURIA',
+        'PLASMA CELL LEUKEMIA', 'SCID', 'SICKLE CELL DISEASE', 'TALL', 'X-LINKED HYPER IGM SYNDROME'
+    ]), index=None)
+    diagnosis_date = st.date_input("Date of First Diagnosis / BMBx", value="2000-01-31", format="DD/MM/YYYY")
+
+    recipient_blood_group = st.radio("Recipient Blood Group", ['A+', 'A-', 'B+', 'B-', 'O+', 'O-', 'AB+', 'AB-', 'Unknown'], key="recipient_blood_group", index=None)
+
+    st.markdown("###### Recipient HLA Alleles")
+    r_hla_a = st.multiselect("R_HLA_A", options=['Unknown', 'SELF'], max_selections=2, accept_new_options=True)
+    r_hla_b = st.multiselect("R_HLA_B", options=['Unknown', 'SELF'], max_selections=2, accept_new_options=True)
+    r_hla_c = st.multiselect("R_HLA_C", options=['Unknown', 'SELF'], max_selections=2, accept_new_options=True)
+    r_hla_dr = st.multiselect("R_HLA_DR", options=['Unknown', 'SELF'], max_selections=2, accept_new_options=True)
+    r_hla_dq = st.multiselect("R_HLA_DQ", options=['Unknown', 'SELF'], max_selections=2, accept_new_options=True)
+
+    st.subheader("Donor Information", divider=True)
+    donor_relation = st.radio("Donor Relation to Recipient", [
+        'SELF', 'SIBLING', 'FIRST DEGREE RELATIVE', 'SECOND DEGREE RELATIVE', 'RELATED', 'UNRELATED', 'Unknown', 
+    ], index=None)
+
+    if donor_relation == 'SELF':
+        # If the donor is the recipient, set the donor
+        st.session_state.SELF = True
+    else:
+        st.session_state.SELF = False
+
+    donor_gender = st.radio("Donor Gender", ['MALE', 'FEMALE'], index=None)
+
+    donor_dob = st.date_input("Donor DOB", value="2000-01-31", format="DD/MM/YYYY")
+
+    donor_blood_group = st.radio("Donor Blood Group", ['A+', 'A-', 'B+', 'B-', 'O+', 'O-', 'AB+', 'AB-', 'Unknown'], key="donor_blood_group", index=None)
+
+    st.markdown("###### Donor HLA Alleles")
+    d_hla_a = st.multiselect("D_HLA_A", options=['Unknown', 'SELF'], max_selections=2, accept_new_options=True)
+    d_hla_b = st.multiselect("D_HLA_B", options=['Unknown', 'SELF'], max_selections=2, accept_new_options=True)
+    d_hla_c = st.multiselect("D_HLA_C", options=['Unknown', 'SELF'], max_selections=2, accept_new_options=True)
+    d_hla_dr = st.multiselect("D_HLA_DR", options=['Unknown', 'SELF'], max_selections=2, accept_new_options=True)
+    d_hla_dq = st.multiselect("D_HLA_DQ", options=['Unknown', 'SELF'], max_selections=2, accept_new_options=True)
+
+    st.subheader("Treatment Details", divider=True)
+    lines_of_rx = st.selectbox("Number of Lines of Rx Before HSCT", [0, 1, 2, 3, 4, 5, 6, 7, 'Unknown'], index=None)
+    conditioning = st.multiselect("Pre-HSCT Conditioning Regimen", sorted([
+        'ALEMTUZUMAB', 'ATG', 'BEAM', 'BUSULFAN', 'CAMPATH', 'CARMUSTINE', 'CLOFARABINE',
+        'CYCLOPHOSPHAMIDE', 'CYCLOSPORIN', 'CYTARABINE', 'ETOPOSIDE', 'FLUDARABINE',
+        'GEMCITABINE', 'MELPHALAN', 'MTX', 'OTHER', 'RANIMUSTINE', 'REDUCEDCONDITIONING',
+        'RITUXIMAB', 'SIROLIMUS', 'TBI', 'THIOTEPA', 'TREOSULFAN', 'UA', 'VORNOSTAT'
+    ]), placeholder="Choose an option(s)")
+
+    st.subheader("HSCT Details", divider=True)
+    hsct_date = st.date_input("HSCT Date", value="2000-01-31", format="DD/MM/YYYY")
+    cell_source = st.radio("Source of Cells", sorted(['BONE MARROW', 'PERIPHERAL BLOOD', 'UMBILICAL CORD', 'PBSC', 'Unknown']), index=None)
+    hla_match = st.radio("HLA Match Ratio", ['FULL', 'PARTIAL', 'HAPLOIDENTICAL', 'Unknown'], index=None)
+
+    st.subheader("Post-HSCT Treatment and GVHD Prophylaxis", divider=True)
+    post_hsct_regimen = st.radio("Post-HSCT Regimen", ['YES', 'NO', 'IVIG', 'Unknown'], index=None)
+
+    gvhd_prophylaxis = st.multiselect("First GVHD Prophylaxis", [
+        'NONE'] + sorted(['ABATACEPT', 'ALEMTUZUMAB', 'ATG', 'CYCLOPHOSPHAMIDE', #'CYCLOSPOPRIN', 'CYCLOSPRIN', 
+        'CYCLOSPORIN', 'IMATINIB', 'LEFLUNOMIDE', 'MMF', 'MTX',
+        'RUXOLITINIB', 'SIROLIMUS', 'STEROID', 'TAC'
+    ]), placeholder="Choose an option(s)")
+
+    submitted = st.form_submit_button("PREDICT", type="primary")
+
+if submitted:
+    # single model
+    model = load_model(st.session_state.selected_model)
+
+    # Collect input values in a dict
+    input_dict = {
+        "Recipient_gender": gender,
+        "Recepient_DOB": dob.strftime("%d/%m/%Y"),
+        "Recepient_Nationality": nationality,
+        "Hematological Diagnosis": diagnosis,
+        "Date of first diagnosis/BMBx date": diagnosis_date.strftime("%d/%m/%Y"),
+        "Recepient_Blood group before HSCT": recipient_blood_group if recipient_blood_group != "Unknown" else "X",
+        "Donor_DOB": donor_dob.strftime("%d/%m/%Y"),
+        "Donor_gender": donor_gender,
+        "D_Blood group": donor_blood_group if donor_blood_group != "Unknown" else "X",
+        "R_HLA_A": r_hla_a,
+        "R_HLA _B": r_hla_b,
+        "R_HLA _C": r_hla_c,
+        "R_HLA _DR": r_hla_dr,
+        "R_HLA _DQ": r_hla_dq,
+        "D_HLA_A": d_hla_a,
+        "D_HLA _B": d_hla_b,
+        "D_HLA_C": d_hla_c,
+        "D_HLA_DR": d_hla_dr,
+        "D_HLA _DQ": d_hla_dq,
+        "Number of lines of Rx before HSCT": lines_of_rx,
+        "PreHSCT conditioning regimen+/-ATG+/-TBI": conditioning,
+        "HSCT_date": hsct_date.strftime("%d/%m/%Y"),
+        "Source of cells": cell_source,
+        "Donor_relation to recipient": donor_relation,
+        "HLA match ratio": hla_match,
+        "Post HSCT regimen": post_hsct_regimen,
+        "First_GVHD prophylaxis": gvhd_prophylaxis
+    }
+
+    # You will need to transform these values into proper numeric or encoded inputs for your model
+    X = pd.DataFrame([input_dict])  # Placeholder
+    st.dataframe(X, use_container_width=True)
+    X.to_csv("/home/muhammadridzuan/2025_GVHD/GVHD_App/saved_models/test_individual_input2.csv", index=False)
+
+    # Define features
+    train_features, cat_features = load_train_features()
+    X = preprocess(X)
+    X = X[train_features]
+    st.write("Processed Input Data:")
+    st.dataframe(X, use_container_width=True)
+
+    if st.session_state.SELF:
+        prob = 0.0
+    else:
+        prob = model.predict_proba(X)[0][1]
+
+    result_df = pd.DataFrame()
+    result_df = add_predictions(result_df, [prob])
+
+    st.write("Predictions:")
+    st.dataframe(result_df, use_container_width=False, width=300)

src/pages/2_Bulk_Predictions.py

@@ -0,0 +1,101 @@
+import streamlit as st
+import pandas as pd
+from model_utils import load_model, load_model_ensemble, ensemble_predict
+from preprocess_utils import load_train_features
+from preprocess_utils import preprocess_pipeline as preprocess
+from inference_utils import add_predictions, compute_metrics
+from sidebar import sidebar
+
+# Initialize sidebar
+sidebar()
+
+st.title("📊 Bulk Patient Predictions")
+
+uploaded_file = st.file_uploader("Upload CSV", type=["csv"])
+if uploaded_file:
+    df = pd.read_csv(uploaded_file, header=1)
+    st.write("Raw Data:")
+    st.dataframe(df)
+
+    if st.button("Preprocess"):
+        df_proc = preprocess(df)
+        edited_df = st.data_editor(df_proc, num_rows="dynamic")
+        st.session_state.bulk_input_df = edited_df
+
+    if st.button("Predict"):
+        if "bulk_input_df" not in st.session_state:
+            st.warning("Please preprocess data first.")
+        else:
+            if "ensemble" in st.session_state.selected_model:
+                # ensemble model
+                ensemble = True
+                try:
+                    ensemble_data = load_model_ensemble(st.session_state.selected_model)
+                    st.session_state.trained_models = ensemble_data["models"]
+                    models = ensemble_data["models"]
+                    st.session_state.best_iterations = ensemble_data.get("best_iterations", [])
+                    st.session_state.fold_scores = ensemble_data.get("fold_scores", [])
+
+                except Exception as e:
+                    st.error(f"Error loading ensemble: {str(e)}")
+            else:
+                # single model
+                ensemble = False
+                model_dict = load_model(st.session_state.selected_model)
+                model = model_dict["model"]
+
+            df = st.session_state.bulk_input_df
+
+            # Define the target column (customize this based on your use case)
+            target_col = "GVHD"  # or "Acute GVHD(<100 days)", etc.
+            
+            # Optional filtering depending on target choice
+            if target_col in ["Acute GVHD(<100 days)", "Chronic GVHD>100 days"]:
+                df = df[df[target_col] != 3]
+
+            y = df[target_col]
+
+            # Define features
+            train_features, cat_features = load_train_features()
+
+            X = df[train_features]
+
+            # Convert categorical columns to strings (CatBoost handles them)
+            for col in cat_features:
+                X[col] = X[col].astype(str)
+
+            # ensemble model prediction
+            if ensemble:
+                preds = ensemble_predict(models, X, cat_features)
+                metrics_result_ensemble = compute_metrics(y, preds)
+            else:
+                # single model prediction
+                preds = model.predict_proba(X)[:, 1]
+                metrics_result_single = compute_metrics(y, preds)
+            
+            st.session_state.targets_df = y
+            styled = add_predictions(X.copy(), preds)
+            st.write("Predictions:")
+            st.dataframe(styled, use_container_width=False, width=300)
+
+            if not ensemble:
+                st.write("Single Model Predictions:")
+                for metric, value in metrics_result_single.items():
+                    st.write(f"  **{metric}**: {value:.3f}")
+            else:
+                st.write("Ensemble Predictions:")
+                for metric, value in metrics_result_ensemble.items():
+                    st.write(f"  **{metric}**: {value:.3f}")
+
+            # Find difference in columns between uploaded data and training features
+            missing_features = set(st.session_state.orig_train_cols).union(train_features) - set(df.columns)
+            missing_features = set([i if i[-2:] != "_X" else '' for i in missing_features])
+            missing_features = sorted(list(missing_features))
+
+            new_features = set(df.columns) - set(st.session_state.orig_train_cols).union(train_features)
+            new_features = sorted(list(new_features))
+            if missing_features:
+                st.warning(f"**Missing features in uploaded data:** \n{'''  \n'''.join(missing_features)}")
+
+            if new_features:
+                st.warning(f"**New features in uploaded data not in training set:**  \n{'''  \n'''.join(new_features)}")

src/pages/3_Preprocessing_and_Training.py

@@ -0,0 +1,191 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+from catboost import CatBoostClassifier, cv, Pool
+from model_utils import get_model, save_model, save_model_ensemble, ensemble_predict
+from preprocess_utils import load_train_features
+from preprocess_utils import preprocess_pipeline as preprocess
+from inference_utils import compute_metrics
+from sidebar import sidebar
+from sklearn.model_selection import StratifiedKFold
+
+# Initialize sidebar
+sidebar()
+
+st.title("🧪 Preprocessing & Training")
+
+uploaded_file = st.file_uploader("Upload CSV", type=["csv"])
+if uploaded_file:
+    df = pd.read_csv(uploaded_file, header=1)
+    st.write("Raw Data:")
+    st.dataframe(df)
+
+    st.session_state.target_col = st.selectbox(
+        "Select target column to predict:",
+        options=[
+            "GVHD",
+            "Acute GVHD(<100 days)",
+            "Chronic GVHD>100 days",
+        ],
+        index=0
+    )
+
+if st.button("Preprocess"):
+    df_proc = preprocess(df)
+
+    # TODO: Remove. Temp
+    st.session_state.orig_train_cols = df_proc.columns.tolist()
+
+    edited_df = st.data_editor(df_proc, num_rows="dynamic")
+    st.session_state.edited_df = edited_df
+
+
+if st.button("Re-train"):
+    if "edited_df" not in st.session_state:
+        st.warning("Please preprocess and edit data first.")
+    else:
+        # Model selection
+        model_type = "CatBoost"  # Fixed to CatBoost for now
+
+        df = st.session_state.edited_df.copy()
+        target_col = st.session_state.target_col
+
+        if target_col in ["Acute GVHD(<100 days)", "Chronic GVHD>100 days"]:
+            df = df[df[target_col] != 3]
+
+        y = df[target_col]
+        st.write(df[target_col].value_counts())
+        train_features, cat_features = load_train_features()
+        X = df[train_features]
+
+        for col in cat_features:
+            X[col] = X[col].astype(str)
+
+        st.info("Running 5-Fold cross-validation with model saving...")
+        skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)
+
+        fold_models = []
+        fold_scores = []
+        best_iterations = []
+
+        for fold, (train_idx, val_idx) in enumerate(skf.split(X, y), start=1):
+            st.write(f"Training Fold {fold}...")
+
+            X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]
+            y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]
+
+            train_pool = Pool(X_train, y_train, cat_features=cat_features)
+            val_pool = Pool(X_val, y_val, cat_features=cat_features)
+
+            model = get_model(model_type, mode="ensemble")
+
+            if model_type == "CatBoost":
+                model.fit(
+                    X_train, y_train, 
+                    eval_set=(X_val, y_val), 
+                    cat_features=cat_features, 
+                    use_best_model=True, 
+                )
+            else:
+                model.fit(X_train, y_train)
+                
+            best_iter = model.get_best_iteration()
+            best_iterations.append(best_iter)
+
+            fold_models.append(model)
+            val_preds = model.predict_proba(X_val)[:, 1]
+            fold_scores.append(model.eval_metrics(val_pool, ["AUC", "F1", "Accuracy", "Precision", "Recall", "BrierScore", "Logloss"], best_iter))
+
+            st.success(f"Fold {fold} trained. Best iteration: {best_iter}")
+
+        st.session_state.trained_models = fold_models
+        st.session_state.fold_scores = fold_scores
+        st.session_state.best_iterations = best_iterations
+
+        ### TURN OFF SINGLE MODEL TRAINING ####
+        # Single model training
+        st.session_state.best_iteration = np.max(st.session_state.best_iterations) # if "best_iterations" in st.session_state else 5000
+       
+        final_model = get_model(model_type, mode="ensemble", best_iter=st.session_state.best_iteration)
+        if model_type == "CatBoost":
+            final_model.fit(
+                X, y, 
+                cat_features=cat_features, 
+            )
+        else:
+            final_model.fit(X, y)
+        st.session_state.trained_model = final_model
+
+        st.success("All folds completed. Models saved for ensembling.")
+
+# CV summary metrics
+if "fold_scores" in st.session_state:
+    st.subheader("Cross-Validation Metrics (5-Fold)")
+    metrics = ["AUC", "F1", "Accuracy", "Precision", "Recall", "BrierScore", "Logloss"]
+    scores = st.session_state.fold_scores
+
+    for metric in metrics:
+        values = [score[metric][-1] for score in scores]  # last = best_iteration
+        mean_val = sum(values) / len(values)
+        std_val = pd.Series(values).std()
+        st.write(f"**{metric}**: {mean_val:.3f} ± {std_val:.3f}")
+
+# Single & ensemble evaluation
+if "trained_model" in st.session_state or "trained_models" in st.session_state:
+    st.subheader("🔮 Ensemble Evaluation (on Training Data)")
+
+    models = st.session_state.trained_models
+    ### TURN OFF SINGLE MODEL EVALUATION ###
+    single_model = st.session_state.trained_model
+
+    df = st.session_state.edited_df.copy()
+    target_col = st.session_state.target_col
+    # st.session_state.targets_df = df[["GVHD", "Acute GVHD(<100 days)", "Chronic GVHD>100 days"]]
+
+    if target_col in ["Acute GVHD(<100 days)", "Chronic GVHD>100 days"]:
+        df = df[df[target_col] != 3]
+
+    y = df[target_col]
+    st.session_state.targets_df = y
+    train_features, cat_features = load_train_features()
+    X = df[train_features]
+
+    for col in cat_features:
+        X[col] = X[col].astype(str)
+
+    ### TURN OFF SINGLE MODEL EVALUATION ###
+    y_pred_prob_single = single_model.predict_proba(X)[:, 1]
+    metrics_result_single = compute_metrics(y, y_pred_prob_single)
+
+    y_pred_prob_ensemble = ensemble_predict(models, X, cat_features)
+    metrics_result_ensemble = compute_metrics(y, y_pred_prob_ensemble)
+
+    ### TURN OFF SINGLE MODEL EVALUATION ###
+    st.write("Single Model Predictions:")
+    for metric, value in metrics_result_single.items():
+        st.write(f"**{metric}**: {value:.3f}")
+
+    st.write("Ensemble Predictions:")
+    for metric, value in metrics_result_ensemble.items():
+        st.write(f"**{metric}**: {value:.3f}")
+        
+
+    user_model_name = st.text_input("Enter model name to be saved:")
+
+    if user_model_name:
+        ### TURN OFF SINGLE MODEL SAVING ###
+        filename = save_model(st.session_state.trained_model, user_model_name, metrics_result_single)
+
+        filename = save_model_ensemble(
+            st.session_state.trained_models,
+            user_model_name,
+            best_iterations=st.session_state.best_iterations,
+            fold_scores=st.session_state.fold_scores,
+            metrics_result_ensemble=metrics_result_ensemble
+        )
+
+        st.success(f"{filename} is successfully saved!")
+        st.success(f"Ensemble saved as {filename}_ensemble")
+
+else:
+    st.info("Train a model first before saving.")

src/params/model_params.yaml

@@ -0,0 +1,34 @@
+CatBoost:
+  ensemble:
+    learning_rate: 0.1
+    depth: 12
+    loss_function: Logloss
+    random_seed: 0
+    l2_leaf_reg: 7
+    subsample: 0.7
+    grow_policy: Lossguide  # SymmetricTree or Depthwise or Lossguide
+    bagging_temperature: 1
+    random_strength: 5
+    min_data_in_leaf: 5
+    iterations: 10000
+    early_stopping_rounds: 50
+    custom_loss: ['AUC', "F1", "Accuracy", "Precision", "Recall", "BrierScore", "Logloss"]
+    verbose: False
+
+# lr1e1_d12_l27_ss07_gpLg_bag1_rs5_m5
+
+  single_model:
+    # in this mode, the model is trained on the entire dataset using the best_iter obtained from cross-validation
+    learning_rate: 0.1
+    depth: 12
+    loss_function: Logloss
+    random_seed: 0
+    l2_leaf_reg: 7
+    subsample: 0.7
+    grow_policy: Lossguide  # SymmetricTree or Depthwise or Lossguide
+    bagging_temperature: 1
+    random_strength: 5
+    min_data_in_leaf: 5
+    custom_loss: ['AUC', "F1", "Accuracy", "Precision", "Recall", "BrierScore", "Logloss"]
+    verbose: False
+

src/preprocess_utils.py

@@ -0,0 +1,928 @@
+import numpy as np
+import pandas as pd
+import re
+from sklearn.preprocessing import MultiLabelBinarizer
+
+# Constants
+UNKNOWN_TOKEN = "X"
+DATE_FORMAT = '%d/%m/%Y'
+BLOOD_GROUP_COLS = ["D_Blood group", "Recepient_Blood group before HSCT"]
+NATIONALITY_CORRECTIONS = {
+    "AFGHANISTAN": "AFGHAN",
+    "ALGERIA": "ALGERIAN",
+    "EMARATI": "EMIRATI",
+    "UAE": "EMIRATI",
+    "PHILIPPINO": "FILIPINO",
+    "JORDAN": "JORDANIAN",
+    "JORDANI": "JORDANIAN",
+    "PAKISTAN": "PAKISTANI",
+    "PAKISTANII": "PAKISTANI",
+    "PALESTINE": "PALESTINIAN",
+    "PALESTENIAN": "PALESTINIAN",
+    "USA": "AMERICAN",
+}
+# 1. Regional Grouping (Geography-Based)
+regional_grouping = {
+    # Middle East
+    'EMIRATI': 'Middle East',
+    'OMANI': 'Middle East',
+    'SAUDI': 'Middle East',
+    'KUWAIT': 'Middle East',
+    'JORDANIAN': 'Middle East',
+    'LEBANESE': 'Middle East',
+    'IRAQI': 'Middle East',
+    'SYRIAN': 'Middle East',
+    'YEMENI': 'Middle East',
+    'PALESTINIAN': 'Middle East',
+
+    # North Africa
+    'EGYPTIAN': 'North Africa',
+    'SUDANESE': 'North Africa',
+    'ALGERIAN': 'North Africa',
+    'MOROCCAN': 'North Africa',
+    'MAURITANIA': 'North Africa',
+    'COMORAN': 'North Africa',
+
+    # South Asia
+    'INDIAN': 'South Asia',
+    'PAKISTANI': 'South Asia',
+    'BANGLADESHI': 'South Asia',
+    'SRI LANKAN': 'South Asia',
+    'AFGHAN': 'South Asia',
+
+    # Southeast Asia
+    'FILIPINO': 'Southeast Asia',
+    'INDONESIAN': 'Southeast Asia',
+
+    # East Africa
+    'ETHIOPIAN': 'East Africa',
+    'SOMALI': 'East Africa',
+    'ERITREAN': 'East Africa',
+
+    # Central Asia
+    'UZBEKISTANI': 'Central Asia',
+
+    # Western Nations / Oceania / Americas
+    'AMERICAN': 'Western',
+    'BRITISH': 'Western',
+    'NEW ZEALANDER': 'Oceania',
+    'FIJI': 'Oceania'
+}
+
+# 2. Cultural-Linguistic Grouping
+cultural_grouping = {
+    'EMIRATI': 'Arab',
+    'OMANI': 'Arab',
+    'SAUDI': 'Arab',
+    'KUWAIT': 'Arab',
+    'JORDANIAN': 'Arab',
+    'LEBANESE': 'Arab',
+    'IRAQI': 'Arab',
+    'SYRIAN': 'Arab',
+    'YEMENI': 'Arab',
+    'PALESTINIAN': 'Arab',
+    'EGYPTIAN': 'Arab',
+    'SUDANESE': 'Arab-African',
+    'ALGERIAN': 'Arab',
+    'MOROCCAN': 'Arab',
+    'MAURITANIA': 'Arab',
+    'COMORAN': 'Arab-African',
+    'INDIAN': 'South Asian',
+    'PAKISTANI': 'South Asian',
+    'BANGLADESHI': 'South Asian',
+    'SRI LANKAN': 'South Asian',
+    'AFGHAN': 'South Asian',
+    'FILIPINO': 'Southeast Asian',
+    'INDONESIAN': 'Southeast Asian',
+    'ETHIOPIAN': 'East African',
+    'SOMALI': 'East African',
+    'ERITREAN': 'East African',
+    'UZBEKISTANI': 'Central Asian',
+    'AMERICAN': 'Western/English-speaking',
+    'BRITISH': 'Western/English-speaking',
+    'NEW ZEALANDER': 'Western/English-speaking',
+    'FIJI': 'Pacific Islander'
+}
+
+# 3. World Bank Income Grouping
+income_grouping = {
+    'EMIRATI': 'High income',
+    'OMANI': 'High income',
+    'SAUDI': 'High income',
+    'KUWAIT': 'High income',
+    'JORDANIAN': 'Upper-middle income',
+    'LEBANESE': 'Upper-middle income',
+    'IRAQI': 'Upper-middle income',
+    'SYRIAN': 'Low income',
+    'YEMENI': 'Low income',
+    'PALESTINIAN': 'Lower-middle income',
+    'EGYPTIAN': 'Lower-middle income',
+    'SUDANESE': 'Low income',
+    'ALGERIAN': 'Lower-middle income',
+    'MOROCCAN': 'Lower-middle income',
+    'MAURITANIA': 'Low income',
+    'COMORAN': 'Low income',
+    'INDIAN': 'Lower-middle income',
+    'PAKISTANI': 'Lower-middle income',
+    'BANGLADESHI': 'Lower-middle income',
+    'SRI LANKAN': 'Lower-middle income',
+    'AFGHAN': 'Low income',
+    'FILIPINO': 'Lower-middle income',
+    'INDONESIAN': 'Lower-middle income',
+    'ETHIOPIAN': 'Low income',
+    'SOMALI': 'Low income',
+    'ERITREAN': 'Low income',
+    'UZBEKISTANI': 'Lower-middle income',
+    'AMERICAN': 'High income',
+    'BRITISH': 'High income',
+    'NEW ZEALANDER': 'High income',
+    'FIJI': 'Upper-middle income'
+}
+
+# 4. WHO Regional Office Grouping
+who_region_grouping = {
+    'EMIRATI': 'EMRO',
+    'OMANI': 'EMRO',
+    'SAUDI': 'EMRO',
+    'KUWAIT': 'EMRO',
+    'JORDANIAN': 'EMRO',
+    'LEBANESE': 'EMRO',
+    'IRAQI': 'EMRO',
+    'SYRIAN': 'EMRO',
+    'YEMENI': 'EMRO',
+    'PALESTINIAN': 'EMRO',
+    'EGYPTIAN': 'EMRO',
+    'SUDANESE': 'EMRO',
+    'ALGERIAN': 'AFRO',
+    'MOROCCAN': 'EMRO',
+    'MAURITANIA': 'AFRO',
+    'COMORAN': 'AFRO',
+    'INDIAN': 'SEARO',
+    'PAKISTANI': 'EMRO',
+    'BANGLADESHI': 'SEARO',
+    'SRI LANKAN': 'SEARO',
+    'AFGHAN': 'EMRO',
+    'FILIPINO': 'WPRO',
+    'INDONESIAN': 'SEARO',
+    'ETHIOPIAN': 'AFRO',
+    'SOMALI': 'EMRO',
+    'ERITREAN': 'AFRO',
+    'UZBEKISTANI': 'EURO',
+    'AMERICAN': 'AMRO',
+    'BRITISH': 'EURO',
+    'NEW ZEALANDER': 'WPRO',
+    'FIJI': 'WPRO'
+}
+groupings = {
+    'Recepient_Nationality_Geographical': regional_grouping,
+    'Recepient_Nationality_Cultural': cultural_grouping,
+    'Recepient_Nationality_Regional_Income': income_grouping,
+    'Recepient_Nationality_Regional_WHO': who_region_grouping
+}
+
+# FIRST_GVHD_PROPHYLAXIS_CORRECTIONS
+DRUG_SPELLING_CORRECTIONS = {
+    "CYCLOSPOPRIN": "CYCLOSPORIN",
+    "CYCLOSPRIN": "CYCLOSPORIN",
+    "CYCLOSPOROIN": "CYCLOSPORIN",
+    "CY": "CYCLOSPORIN",
+    "TAC": "TACROLIMUS", # no TACROLIMUS in new dataset, only TAC
+    "MTX": "METHOTREXATE", # one METHOTREXATE in new dataset (ID 118), replaced with MTX
+    "BUDESONIDE": "STEROID", # 3 BUDESONIDE in new dataset (ID 259, 263, 273), replaced with STEROID
+    "STEROIDS": "STEROID", # 6 STEROIDS in new dataset (ID 172, 175, 140, 146, 152, 166), replaced with STEROID
+    "ATG.": "ATG",
+    "FLUDARABINIE": "FLUDARABINE",
+    "FLUDRABINE":"FLUDARABINE",
+    "BUSULPHAN": "BUSULFAN",
+    "MEPHALAN": "MELPHALAN",
+}
+GENDER_MAP = {
+    0: "MALE", 1: "FEMALE", 2: UNKNOWN_TOKEN,
+    "0": "MALE", "1": "FEMALE", "2": UNKNOWN_TOKEN
+}
+RELATION_CORRECTIONS = {
+    r"(?i)BROTHER": "SIBLING", 
+    r"(?i)SISTER": "SIBLING",
+    r"(?i)FATHER": "FIRST DEGREE RELATIVE", 
+    r"(?i)MOTHER": "FIRST DEGREE RELATIVE",
+    r"(?i)SON": "FIRST DEGREE RELATIVE", 
+    r"(?i)DAUGHTER": "FIRST DEGREE RELATIVE",
+    r"(?i)COUSIN": "SECOND DEGREE RELATIVE", 
+    r"(?i)UNCLE": "SECOND DEGREE RELATIVE",
+    r"(?i)AUNT": "SECOND DEGREE RELATIVE",
+    r"(?i)other": UNKNOWN_TOKEN
+}
+STRING_NORMALIZATION_MAP = {
+    r"(?i)unknown": UNKNOWN_TOKEN, r"(?i)unkown": UNKNOWN_TOKEN,
+    r"(?i)Unknwon": UNKNOWN_TOKEN, np.nan: UNKNOWN_TOKEN,
+    r"(?i)\bMale\b": "MALE", r"(?i)\bFemale\b": "FEMALE",
+    "1o": "10", r"(?i)Umbilical Cord": "UMBILICAL CORD",
+    r"(?i)Umbilical Cord blood": "UMBILICAL CORD",
+    r"(?i)Bone Marrow": "BONE MARROW", "MDS": "MYELODYSPLASTIC SYNDROME"
+}
+diagnosis_group_map = {
+    "MYELOPROLIFERATIVE DISORDER": "MYELOPROLIFERATIVE NEOPLASMS",
+    "CML": "MYELOPROLIFERATIVE NEOPLASMS",
+    "MYELOFIBROSIS": "MYELOPROLIFERATIVE NEOPLASMS",
+    "NON-HODGKIN LYMPHOMA": "LYMPHOMA",
+    'NON HODGKIN LYMPHOMA': "LYMPHOMA",
+    "HODGKIN LYMPHOMA": "LYMPHOMA",
+    "BETA THALASSEMIA": "RED CELL DISORDERS",
+    'BETA THALESSEMIA': "RED CELL DISORDERS",
+    "ALPHA THALASSEMIA": "RED CELL DISORDERS",
+    "ALPHA THALESSEMIA": "RED CELL DISORDERS",
+    "ALPHA THALSSEMIA": "RED CELL DISORDERS",
+    "HEREDITARY SPHEROCYTOSIS": "RED CELL DISORDERS",
+    "SICKLE CELL DISEASE": "RED CELL DISORDERS",
+    "APLASTIC ANEMIA": "BMF SYNDROMES",
+    "FANCONI ANEMIA": "BMF SYNDROMES",
+    "DYSKERATOSIS CONGENITA": "BMF SYNDROMES",
+    'DYSKERATOSIS CONGENTIA': "BMF SYNDROMES",
+    "CHRONIC GRANULOMATOUS DISEASE": "IMMUNE DISORDERS",
+    "COMBINED VARIABLE IMMUNODEFICIENCY": "IMMUNE DISORDERS",
+    "SCID": "IMMUNE DISORDERS",
+
+    ## check this one
+    "X-LINKED HYPERGAMMAGLOBULINEMIA": "IMMUNE DISORDERS",
+    '-LINKED HYPERGAMMAGLOBULINEMIA': "IMMUNE DISORDERS",
+    '-LINKED HYPER IGM SYNDROME': "IMMUNE DISORDERS",
+    "HYPOGAMMAGLOBULINEMIA": "IMMUNE DISORDERS",
+    
+
+    ## check this one
+    "GLANZMANN": "OTHER",
+    'GLANZMANN THROMBASTHENIA': "OTHER",
+    
+    "CLL": "OTHER",
+    "PNH": "OTHER",
+    "HLH": "OTHER",
+    "LANGERHANS CELL HISTIOCYTOSIS": "OTHER",
+    "BLASTIC PLASMACYTOID DENDRITIC CELL NEOPLASM": "OTHER",
+    'BLASTIC PLASMACYTOID DENDRITRIC CELL NEOPLASM': "OTHER",
+    "B-ALL": "ALL",
+    "BALL": "ALL",
+    "TALL": "ALL",
+    "T-ALL": "ALL",
+    "AML": "AML",
+    "ACUTE MYELOID LEUKEMIA": "AML"
+}
+
+# # 0 nonmalignant; 1: malignant 
+malignant_map = {
+    'AML': 1, 
+    'RED CELL DISORDERS': 0, 
+    'AMYLOIDOSIS': 0, 
+    'BMF SYNDROMES': 0, 
+    'ALL': 1,
+    'OTHER': 0, 
+    'IMMUNE DISORDERS': 0, 
+    'CHRONIC LYMPHOCYTIC LEUKEMIA': 1,
+    'MYELOPROLIFERATIVE NEOPLASMS': 1, # note: CML is malignant; not sure about MYELOPROLIFERATIVE DISORDER & MYELOFIBROSIS
+    'HEMOPHAGOCYTIC LYMPHOHISTIOCYTOSIS (HLH)': 0, 
+    'LYMPHOMA': 1,
+    'MYELODYSPLASTIC SYNDROME': 1, 
+    'MEDULLOBLASTOMA': 0, 
+    'MULTIPLE MYELOMA': 0,
+    'NEUROBLASTOMA': 0, 
+    'PAROXYSMAL NOCTURNAL HEMOGLOBINURIA': 0,
+    'PLASMA CELL LEUKEMIA': 0
+}
+
+def load_train_features():
+    # Define features
+    HLA_sub12 = [
+
+    # Recipient - HLA-A
+    'R_HLA_A_1', 'R_HLA_A_2', 'R_HLA_A_3', 'R_HLA_A_4', 'R_HLA_A_7', 'R_HLA_A_8',
+    'R_HLA_A_11', 'R_HLA_A_12', 'R_HLA_A_20', 'R_HLA_A_23', 'R_HLA_A_24', 'R_HLA_A_25',
+    'R_HLA_A_26', 'R_HLA_A_29', 'R_HLA_A_30', 'R_HLA_A_31', 'R_HLA_A_32', 'R_HLA_A_33',
+    'R_HLA_A_34', 'R_HLA_A_66', 'R_HLA_A_68', 'R_HLA_A_69', 'R_HLA_A_74', 'R_HLA_A_X',
+
+    # Recipient - HLA-B
+    'R_HLA_B_7', 'R_HLA_B_8', 'R_HLA_B_13', 'R_HLA_B_14', 'R_HLA_B_15', 'R_HLA_B_18',
+    'R_HLA_B_23', 'R_HLA_B_24', 'R_HLA_B_27', 'R_HLA_B_35', 'R_HLA_B_37', 'R_HLA_B_38',
+    'R_HLA_B_39', 'R_HLA_B_40', 'R_HLA_B_41', 'R_HLA_B_42', 'R_HLA_B_44', 'R_HLA_B_45',
+    'R_HLA_B_46', 'R_HLA_B_49', 'R_HLA_B_50', 'R_HLA_B_51', 'R_HLA_B_52', 'R_HLA_B_53',
+    'R_HLA_B_55', 'R_HLA_B_56', 'R_HLA_B_57', 'R_HLA_B_58', 'R_HLA_B_73', 'R_HLA_B_81',
+    'R_HLA_B_X',
+
+    # Recipient - HLA-C
+    'R_HLA_C_1', 'R_HLA_C_2', 'R_HLA_C_3', 'R_HLA_C_4', 'R_HLA_C_5', 'R_HLA_C_6',
+    'R_HLA_C_7', 'R_HLA_C_8', 'R_HLA_C_12', 'R_HLA_C_14', 'R_HLA_C_15', 'R_HLA_C_16',
+    'R_HLA_C_17', 'R_HLA_C_18', 'R_HLA_C_38', 'R_HLA_C_49', 'R_HLA_C_50', 'R_HLA_C_X',
+
+    # Recipient - HLA-DR
+    'R_HLA_DR_1', 'R_HLA_DR_2', 'R_HLA_DR_3', 'R_HLA_DR_4', 'R_HLA_DR_5', 'R_HLA_DR_6',
+    'R_HLA_DR_7', 'R_HLA_DR_8', 'R_HLA_DR_9', 'R_HLA_DR_10', 'R_HLA_DR_11', 'R_HLA_DR_12',
+    'R_HLA_DR_13', 'R_HLA_DR_14', 'R_HLA_DR_15', 'R_HLA_DR_16', 'R_HLA_DR_17', 'R_HLA_DR_X',
+
+    # Recipient - HLA-DQ
+    'R_HLA_DQ_1', 'R_HLA_DQ_2', 'R_HLA_DQ_3', 'R_HLA_DQ_4', 'R_HLA_DQ_5', 'R_HLA_DQ_6',
+    'R_HLA_DQ_7', 'R_HLA_DQ_11', 'R_HLA_DQ_15', 'R_HLA_DQ_16', 'R_HLA_DQ_301', 'R_HLA_DQ_X',
+
+    # Donor - HLA-A
+    'D_HLA_A_1', 'D_HLA_A_2', 'D_HLA_A_3', 'D_HLA_A_8', 'D_HLA_A_11', 'D_HLA_A_12',
+    'D_HLA_A_23', 'D_HLA_A_24', 'D_HLA_A_25', 'D_HLA_A_26', 'D_HLA_A_29', 'D_HLA_A_30',
+    'D_HLA_A_31', 'D_HLA_A_32', 'D_HLA_A_33', 'D_HLA_A_34', 'D_HLA_A_66', 'D_HLA_A_68',
+    'D_HLA_A_69', 'D_HLA_A_7', 'D_HLA_A_74', 'D_HLA_A_X',
+
+    # Donor - HLA-B
+    'D_HLA_B_7', 'D_HLA_B_8', 'D_HLA_B_13', 'D_HLA_B_14', 'D_HLA_B_15', 'D_HLA_B_17',
+    'D_HLA_B_18', 'D_HLA_B_23', 'D_HLA_B_24', 'D_HLA_B_27', 'D_HLA_B_35', 'D_HLA_B_37',
+    'D_HLA_B_38', 'D_HLA_B_39', 'D_HLA_B_40', 'D_HLA_B_41', 'D_HLA_B_42', 'D_HLA_B_44',
+    'D_HLA_B_45', 'D_HLA_B_48', 'D_HLA_B_49', 'D_HLA_B_50', 'D_HLA_B_51', 'D_HLA_B_52',
+    'D_HLA_B_53', 'D_HLA_B_55', 'D_HLA_B_56', 'D_HLA_B_57', 'D_HLA_B_58', 'D_HLA_B_73',
+    'D_HLA_B_81', 'D_HLA_B_X',
+
+    # Donor - HLA-C
+    'D_HLA_C_1', 'D_HLA_C_2', 'D_HLA_C_3', 'D_HLA_C_4', 'D_HLA_C_5', 'D_HLA_C_6',
+    'D_HLA_C_7', 'D_HLA_C_8', 'D_HLA_C_12', 'D_HLA_C_14', 'D_HLA_C_15', 'D_HLA_C_16',
+    'D_HLA_C_17', 'D_HLA_C_18', 'D_HLA_C_38', 'D_HLA_C_49', 'D_HLA_C_50', 'D_HLA_C_X',
+
+    # Donor - HLA-DR
+    'D_HLA_DR_1', 'D_HLA_DR_2', 'D_HLA_DR_3', 'D_HLA_DR_4', 'D_HLA_DR_5', 'D_HLA_DR_6',
+    'D_HLA_DR_7', 'D_HLA_DR_8', 'D_HLA_DR_9', 'D_HLA_DR_10', 'D_HLA_DR_11', 'D_HLA_DR_12',
+    'D_HLA_DR_13', 'D_HLA_DR_14', 'D_HLA_DR_15', 'D_HLA_DR_16', 'D_HLA_DR_17', 'D_HLA_DR_X',
+
+    # Donor - HLA-DQ
+    'D_HLA_DQ_1', 'D_HLA_DQ_2', 'D_HLA_DQ_3', 'D_HLA_DQ_4', 'D_HLA_DQ_5', 'D_HLA_DQ_6',
+    'D_HLA_DQ_7', 'D_HLA_DQ_11', 'D_HLA_DQ_15', 'D_HLA_DQ_16', 'D_HLA_DQ_301', 'D_HLA_DQ_X'
+    ]
+
+
+    HLA_sub12_without_X = [i for i in HLA_sub12 if "_X" not in i]
+
+    prehsct_onehot = [
+        'PreHSCT_ALEMTUZUMAB',
+        'PreHSCT_ATG',
+        'PreHSCT_BEAM',
+        'PreHSCT_BUSULFAN',
+        'PreHSCT_CAMPATH',
+        'PreHSCT_CARMUSTINE',
+        'PreHSCT_CLOFARABINE',
+        'PreHSCT_CYCLOPHOSPHAMIDE',
+        'PreHSCT_CYCLOSPORIN',
+        'PreHSCT_CYTARABINE',
+        'PreHSCT_ETOPOSIDE',
+        'PreHSCT_FLUDARABINE',
+        'PreHSCT_GEMCITABINE',
+        'PreHSCT_MELPHALAN',
+        'PreHSCT_MTX',
+        'PreHSCT_OTHER',
+        'PreHSCT_RANIMUSTINE',
+        'PreHSCT_REDUCEDCONDITIONING',
+        'PreHSCT_RITUXIMAB',
+        'PreHSCT_SIROLIMUS',
+        'PreHSCT_TBI',
+        'PreHSCT_THIOTEPA',
+        'PreHSCT_TREOSULFAN',
+        'PreHSCT_UA',
+        'PreHSCT_VORNOSTAT',
+    ]
+
+    first_prophylaxis_onehot = [
+        'First_GVHD_prophylaxis_ABATACEPT',
+        'First_GVHD_prophylaxis_ALEMTUZUMAB',
+        'First_GVHD_prophylaxis_ATG',
+        'First_GVHD_prophylaxis_CYCLOPHOSPHAMIDE',
+        'First_GVHD_prophylaxis_CYCLOSPORIN',
+        'First_GVHD_prophylaxis_IMATINIB',
+        'First_GVHD_prophylaxis_LEFLUNOMIDE',
+        'First_GVHD_prophylaxis_MMF',
+        'First_GVHD_prophylaxis_MTX',
+        'First_GVHD_prophylaxis_NONE',
+        'First_GVHD_prophylaxis_RUXOLITINIB',
+        'First_GVHD_prophylaxis_SIROLIMUS',
+        'First_GVHD_prophylaxis_STEROID',
+        'First_GVHD_prophylaxis_TAC',
+    ]
+
+    train_features = [[ 
+        'Recipient_gender',
+        'R_Age_at_transplant_cutoff18',
+        'Recepient_Nationality_Cultural',
+        'Hematological Diagnosis_Grouped',
+        'Recepient_Blood group before HSCT_MergePlusMinus',
+        'D_Age_at_transplant_cutoff18',
+        'Age_Gap_R_D',
+        'Donor_gender',
+        'D_Blood group_MergePlusMinus',
+        'Number of lines of Rx before HSCT',
+        'Source of cells',
+        'Donor_relation to recipient',
+    ] + HLA_sub12_without_X + prehsct_onehot + first_prophylaxis_onehot][0]
+
+    # Categorical features
+    cat_features = [
+        'Recipient_gender',
+        'Recepient_Nationality_Cultural',
+        'Hematological Diagnosis_Grouped',
+        'Recepient_Blood group before HSCT_MergePlusMinus',
+        'Donor_gender',
+        'D_Blood group_MergePlusMinus',
+        'Source of cells',
+        'Donor_relation to recipient',
+    ]
+
+    return train_features, cat_features
+
+def load_dataset(file_path: str) -> pd.DataFrame:
+    """Load dataset from CSV file and drop columns with all missing values"""
+    df = pd.read_csv(file_path, header=1)
+    return df.dropna(axis=1, how="all")
+
+def normalize_strings(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Standardize string values across the dataset:
+    - Replace variations of unknown/NA with consistent token
+    - Correct common misspellings and abbreviations
+    - Capitalize all strings for consistency
+    - Strip leading/trailing whitespace
+    """
+    # Apply global string replacements
+    df = df.replace(STRING_NORMALIZATION_MAP, regex=True)
+    
+    # Handle nationality-specific replacements
+    non_nationality_cols = [col for col in df.columns if "Nationality" not in col]
+    df[non_nationality_cols] = df[non_nationality_cols].replace(
+        {r"(?i)\buk\b": UNKNOWN_TOKEN}, regex=True
+    )
+    
+    # Handle non-HLA specific replacements
+    non_hla_cols = [col for col in df.columns if "HLA" not in col]
+    df[non_hla_cols] = df[non_hla_cols].replace(
+        {r"(?i)\bna\b": UNKNOWN_TOKEN}, regex=True
+    )
+    
+    # Capitalize all string values
+    df = df.applymap(lambda x: x.upper() if isinstance(x, str) else x)
+    
+    # Strip whitespace
+    return df.applymap(lambda x: x.strip() if isinstance(x, str) else x)
+
+def clean_blood_group_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
+    """Remove spaces from specified blood group columns"""
+    for col in columns:
+        df[col] = df[col].str.replace(" ", "")
+    return df
+
+def process_hla_columns(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Clean and process HLA columns by:
+    1. Splitting combined HLA values into separate columns
+    2. Standardizing missing value representation
+    3. Sorting allele values numerically
+    4. Recombining cleaned values
+    """
+    # Padding function to ensure 2 elements, filling with 'NA'. Used for Individual_Predictions
+    def pad_list(val):
+        if not isinstance(val, list):
+            val = []
+        return (val + ['NA', 'NA'])[:2]
+
+    hla_columns = [col for col in df.columns if "R_HLA" in col or "D_HLA" in col]
+    # hla_columns = ['R_HLA_A', 'R_HLA_B', 'R_HLA_C', 'R_HLA_DR', 'R_HLA_DQ',
+                #    'D_HLA_A', 'D_HLA_B', 'D_HLA_C', 'D_HLA_DR', 'D_HLA_DQ']
+    
+    for col in hla_columns:
+        # Handle special NA representation
+        df[col] = df[col].replace({"NA": "NA&NA"})
+        
+        # Split into two separate columns
+        split_cols = [f"{col}1", f"{col}2"]
+
+        if type(df[col].iloc[0]) != list: # and "&" in df[col].iloc[0]:
+            df[split_cols] = df[col].str.split("&", expand=True)
+        elif type(df[col].iloc[0]) == list:
+            df[col] = df[col].apply(pad_list)
+            df[split_cols] = pd.DataFrame(df[col].tolist(), index=df.index)
+        
+        # Standardize missing values
+        missing_indicators = {" ", "NA", "N/A", UNKNOWN_TOKEN, "''", '""', "", "B1", None}
+        df[split_cols] = df[split_cols].replace(missing_indicators, np.nan)
+        
+        # Convert to numeric and handle zeros
+        df[split_cols] = df[split_cols].apply(pd.to_numeric, errors='coerce')
+        df[split_cols] = df[split_cols].replace(0, np.nan)
+        
+        # Sort values numerically
+        df[split_cols] = np.sort(df[split_cols], axis=1)
+        
+        # Convert numbers to integers, missing to 'X'
+        df[split_cols] = df[split_cols].applymap(lambda x: str(int(x)) if pd.notna(x) else UNKNOWN_TOKEN)
+
+        # Recombine cleaned values
+        df[col] = df[split_cols].astype(str).agg("&".join, axis=1)
+        
+    return df
+
+
+def cast_as_int_if_possible(x):
+    try:
+        i = int(x)
+        # Only return int if conversion is lossless (e.g., avoid converting '5.5' -> 5)
+        if float(x) == i:
+            return i
+    except:
+        pass
+    return x
+
+def HLA_unique_alleles(df, HLA_col1, HLA_col2):
+    HLA_col1_unique = df[HLA_col1].unique()
+    HLA_col2_unique = df[HLA_col2].unique()
+    
+    HLA_col1_unique = [cast_as_int_if_possible(val) for val in HLA_col1_unique]
+    HLA_col2_unique = [cast_as_int_if_possible(val) for val in HLA_col2_unique]
+    
+    unique_set = set(HLA_col1_unique).union(set(HLA_col2_unique))
+    
+    # Replace NaN with "X"
+    unique_set = {(UNKNOWN_TOKEN if pd.isna(item) else str(item)) for item in unique_set}
+    print('unique_set', unique_set)
+    return sorted(unique_set)
+
+def expand_HLA_cols_(df, HLA_col1, HLA_col2):
+    HLA_uniques = HLA_unique_alleles(df, HLA_col1, HLA_col2)
+
+    col_name = HLA_col1[:-1] # get "R_HLA_A" from "R_HLA_A1"
+    for i in HLA_uniques:
+        df[f"{col_name}_{i}"] = 0
+        df.loc[df[HLA_col1]==i, f"{col_name}_{i}"] = 1 # or = 1
+        df.loc[df[HLA_col2]==i, f"{col_name}_{i}"] = 1 # or = 1
+
+    return df
+
+def expand_HLA_cols(df):
+    df = expand_HLA_cols_(df, HLA_col1="R_HLA_A1", HLA_col2="R_HLA_A2")
+    df = expand_HLA_cols_(df, HLA_col1="R_HLA_B1", HLA_col2="R_HLA_B2")
+    df = expand_HLA_cols_(df, HLA_col1="R_HLA_C1", HLA_col2="R_HLA_C2")
+    df = expand_HLA_cols_(df, HLA_col1="R_HLA_DR1", HLA_col2="R_HLA_DR2")
+    df = expand_HLA_cols_(df, HLA_col1="R_HLA_DQ1", HLA_col2="R_HLA_DQ2")
+
+    df = expand_HLA_cols_(df, HLA_col1="D_HLA_A1", HLA_col2="D_HLA_A2")
+    df = expand_HLA_cols_(df, HLA_col1="D_HLA_B1", HLA_col2="D_HLA_B2")
+    df = expand_HLA_cols_(df, HLA_col1="D_HLA_C1", HLA_col2="D_HLA_C2")
+    df = expand_HLA_cols_(df, HLA_col1="D_HLA_DR1", HLA_col2="D_HLA_DR2")
+    df = expand_HLA_cols_(df, HLA_col1="D_HLA_DQ1", HLA_col2="D_HLA_DQ2")
+    return df
+
+def correct_nationalities(df: pd.DataFrame, column: str) -> pd.DataFrame:
+    """Standardize nationality names using predefined corrections"""
+    df[column] = df[column].replace(NATIONALITY_CORRECTIONS)
+    return df
+
+def correct_indiv_drug_name(drug_list):
+    # Find all the drug names and separators in the string
+    parts = re.split(r'([ /+])', drug_list)  # Split but keep the separators
+    
+    corrected_parts = []
+    
+    for part in parts:
+        # If the part is a drug name, apply the correction
+        if part.strip() and part.strip() not in {'', ' ', '/', '+'}:
+            corrected_part = DRUG_SPELLING_CORRECTIONS.get(part.strip(), part.strip())
+            corrected_parts.append(corrected_part)
+        else:
+            # If it's a separator (/, +, space), just keep it
+            corrected_parts.append(part)
+    
+    # Join the parts back together
+    return ''.join(corrected_parts)
+    
+def correct_drug_name_in_list(df: pd.DataFrame, column: str) -> pd.DataFrame:
+    """Standardize drug names in a list using predefined corrections, preserving separators."""
+    # Apply the correction function to each entry in the specified column
+    df[column] = df[column].apply(correct_indiv_drug_name)
+    
+    return df
+
+def standardize_compound_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
+    """
+    Process columns with compound values by:
+    1. Removing spaces
+    2. Standardizing separators
+    3. Sorting components alphabetically
+    """
+    for col in columns:
+        if col in df.columns and type(df[col].iloc[0]) != list:
+            # Clean string values
+            df[col] = df[col].str.replace(" ", "").str.replace("+", "/").str.replace(",", "/")
+
+            # Split, remove empty parts, sort, and join
+            df[col] = df[col].apply(
+                lambda x: "/".join(sorted([part for part in x.split("/") if part])) if isinstance(x, str) else x
+            )
+    return df
+
+def standardize_gender(df: pd.DataFrame) -> pd.DataFrame:
+    """Standardize donor gender values and infer from relationship where possible"""
+    # Apply gender mapping
+    df["Donor_gender"] = df["Donor_gender"].replace(GENDER_MAP)
+    df["Recipient_gender"] = df["Recipient_gender"].replace(GENDER_MAP)
+
+    # Infer gender from relationship
+    gender_map = {
+        "BROTHER": "MALE", "SISTER": "FEMALE",
+        "FATHER": "MALE", "MOTHER": "FEMALE",
+        "SON": "MALE", "DAUGHTER": "FEMALE",
+        "UNCLE": "MALE", "AUNT": "FEMALE"
+    }
+    for relationship, gender in gender_map.items():
+        mask = df["Donor_relation to recipient"] == relationship
+        df.loc[mask, "Donor_gender"] = gender
+        
+    return df
+
+def correct_donor_relationships(df: pd.DataFrame) -> pd.DataFrame:
+    """Standardize relationship categories using predefined corrections"""
+    return df.replace({"Donor_relation to recipient": RELATION_CORRECTIONS}, regex=True)
+
+def handle_self_donor_consistency(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Ensure data consistency for self-donors by:
+    1. Setting HLA values to 'SELF&SELF'
+    2. Verifying matching demographics
+    """
+    self_mask = df["Donor_relation to recipient"] == "SELF"
+    
+    # Set HLA values for self-donors
+    hla_cols = [col for col in df.columns if "R_HLA" in col or "D_HLA" in col]
+    df.loc[self_mask, hla_cols] = "SELF&SELF"
+    
+    # Verify demographic consistency
+    assert df.loc[self_mask, "Recipient_gender"].equals(
+        df.loc[self_mask, "Donor_gender"]
+    ), "Recipient/Donor gender mismatch for self-donors"
+    
+    assert df.loc[self_mask, "Recepient_Blood group before HSCT"].equals(
+        df.loc[self_mask, "D_Blood group"]
+    ), "Blood group mismatch for self-donors"
+    
+    assert df.loc[self_mask, "Recepient_DOB"].equals(
+        df.loc[self_mask, "Donor_DOB"]
+    ), "DOB mismatch for self-donors"
+    
+    return df
+
+def safe_extract_year(date_str: str) -> str:
+    """
+    Safely extract year from date string:
+    - Returns year as integer if valid
+    - Returns UNKNOWN_TOKEN for invalid/missing dates
+    """
+    if not isinstance(date_str, str) or date_str == UNKNOWN_TOKEN:
+        return UNKNOWN_TOKEN
+    
+    try:
+        # Handle special cases like "35 YEAR OLD"
+        if "YEAR" in date_str:
+            return UNKNOWN_TOKEN
+            
+        parts = date_str.split("/")
+        if len(parts) < 3:
+            return UNKNOWN_TOKEN
+            
+        year_part = parts[-1].strip()
+        return int(year_part) if year_part.isdigit() else UNKNOWN_TOKEN
+    except (ValueError, TypeError):
+        return UNKNOWN_TOKEN
+
+def calculate_ages(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Calculate:
+    1. Recipient age at transplant
+    2. Donor age at transplant
+    3. Age gap between recipient and donor
+    """
+    # Extract years safely
+    df["Recepient_DOB_Year"] = df["Recepient_DOB"].apply(safe_extract_year)
+    df["Donor_DOB_Year"] = df["Donor_DOB"].apply(safe_extract_year)
+    df["HSCT_date_Year"] = df["HSCT_date"].apply(safe_extract_year)
+    
+    # Calculate ages with safe conversion
+    def calculate_age_diff(row, dob_col, transplant_col):
+        try:
+            return int(row[transplant_col]) - int(row[dob_col])
+        except (TypeError, ValueError):
+            return UNKNOWN_TOKEN
+    
+    df["R_Age_at_transplant"] = df.apply(
+        lambda row: calculate_age_diff(row, "Recepient_DOB_Year", "HSCT_date_Year"), 
+        axis=1
+    )
+    
+    df["D_Age_at_transplant"] = df.apply(
+        lambda row: calculate_age_diff(row, "Donor_DOB_Year", "HSCT_date_Year"), 
+        axis=1
+    )
+    
+    df["Age_Gap_R_D"] = df.apply(
+        lambda row: calculate_age_diff(row, "Donor_DOB_Year", "Recepient_DOB_Year"), 
+        axis=1
+    )
+    
+    return df
+
+# Utility Function: Split and One-Hot Encode Drug Regimens
+def split_and_one_hot_encode(df, column_name, prefix):
+    """
+    Splits entries in a column by "/" and one-hot encodes the resulting tokens.
+    
+    Args:
+        df (pd.DataFrame): Input dataframe
+        column_name (str): Name of the column to process
+        prefix (str): Prefix for the resulting one-hot encoded columns
+    
+    Returns:
+        pd.DataFrame: DataFrame with one-hot encoded columns added
+    """
+    if type(df[column_name].iloc[0]) != list:
+        df[column_name] = df[column_name].fillna("").apply(lambda x: re.split(r'[/]', x) if x else [])
+    else:
+        pass
+    
+    mlb = MultiLabelBinarizer()
+    encoded_df = pd.DataFrame(
+        mlb.fit_transform(df[column_name]),
+        columns=[f"{prefix}_{drug.strip()}" for drug in mlb.classes_],
+        index=df.index
+    )
+    
+    df = pd.concat([df, encoded_df], axis=1)
+    return df
+
+# Normalize Blood Groups (Remove +/-)
+def merge_blood_groups(df, column, new_col):
+    """
+    Removes '+' and '-' from blood group values.
+
+    Args:
+        df (pd.DataFrame): Input dataframe
+        column (str): Column name to normalize
+        new_col (str): New column name for cleaned values
+
+    Returns:
+        pd.DataFrame: Updated dataframe
+    """
+    df[new_col] = df[column].apply(lambda x: re.sub(r'[+-]', '', x) if pd.notnull(x) else np.nan)
+    return df
+
+def binarize_age(df, age_col, cutoff, new_col):
+    """
+    Binarizes age column based on a cutoff. Non-numeric values are left as-is.
+
+    Args:
+        df (pd.DataFrame): Input dataframe
+        age_col (str): Column name containing age
+        cutoff (int): Age cutoff
+        new_col (str): New binary column name
+
+    Returns:
+        pd.DataFrame: Updated dataframe
+    """
+    def binarize_or_keep(val):
+        try:
+            return int(val >= cutoff)
+        except TypeError:
+            return val  # Leave strings or non-numeric values unchanged
+
+    df[new_col] = df[age_col].apply(binarize_or_keep)
+    return df
+
+# Create Composite Gender & Relation Columns
+def add_gender_relation_features(df):
+    """
+    Creates new columns combining donor relation with recipient and donor genders.
+    
+    Returns:
+        pd.DataFrame: Updated dataframe
+    """
+    df["Relation_and_Recipient_Gender"] = df["Donor_relation to recipient"] + " R_" + df["Recipient_gender"]
+    df["Relation_and_Donor_Gender"] = df["Donor_relation to recipient"] + " D_" + df["Donor_gender"]
+    df["Relation_and_Recipient_and_Donor_Gender"] = (
+        df["Donor_relation to recipient"] + " R_" + df["Recipient_gender"] + " D_" + df["Donor_gender"]
+    )
+    return df
+
+# Nationality-Based Groupings
+def apply_nationality_groupings(df, column, grouping_dicts):
+    """
+    Applies multiple groupings based on nationality.
+
+    Args:
+        df (pd.DataFrame): Input dataframe
+        column (str): Column to group by
+        grouping_dicts (dict): Dictionary of {new_col_name: mapping_dict}
+
+    Returns:
+        pd.DataFrame: Updated dataframe
+    """
+    for new_col, mapping in grouping_dicts.items():
+        df[new_col] = df[column].replace(mapping)
+    return df
+
+# Group and Binarize Diagnosis
+def group_and_binarize_diagnosis(df, original_col, group_map, malignant_map):
+    """
+    Groups diagnosis into categories and flags as malignant or not.
+
+    Args:
+        df (pd.DataFrame): Input dataframe
+        original_col (str): Original diagnosis column
+        group_map (dict): Mapping of diagnoses to groups
+        malignant_map (dict): Mapping of groups to binary malignancy label
+
+    Returns:
+        pd.DataFrame: Updated dataframe
+    """
+    grouped_col = f"{original_col}_Grouped"
+    malignant_col = f"{original_col}_Malignant"
+    
+    df[grouped_col] = df[original_col].replace(group_map)
+    df[malignant_col] = df[grouped_col].replace(malignant_map)
+    return df
+
+def preprocess_pipeline(df) -> pd.DataFrame:
+    """
+    Full preprocessing pipeline:
+    1. Load and initial cleaning
+    2. String normalization
+    3. Special column processing
+    4. Data corrections
+    5. Feature engineering
+    """
+    df = df.dropna(axis=1, how="all")
+
+    # Special column processing
+    # Strip leading/trailing spaces from column names
+    df.columns = df.columns.str.strip()
+    # Remove spaces from HLA columns
+    df.columns = [col.replace(" ", "") if "_HLA" in col else col for col in df.columns]
+
+    # String handling
+    df = normalize_strings(df)
+    df = clean_blood_group_columns(df, BLOOD_GROUP_COLS)
+    
+    # Data corrections
+    df = correct_nationalities(df, "Recepient_Nationality")
+    df = correct_drug_name_in_list(df, "PreHSCT conditioning regimen+/-ATG+/-TBI")
+    df = correct_drug_name_in_list(df, "First_GVHD prophylaxis")
+    # df = correct_drug_name_in_list(df, "Post HSCT regimen")
+    df = standardize_compound_columns(
+        df, 
+        ["PreHSCT conditioning regimen+/-ATG+/-TBI", "First_GVHD prophylaxis"]
+    )
+    df = standardize_gender(df)
+    df = correct_donor_relationships(df)    
+
+    if "SELF" in df["Donor_relation to recipient"].unique():
+        df = handle_self_donor_consistency(df)
+    
+    # HLA processing
+    df = process_hla_columns(df)
+    df = expand_HLA_cols(df)
+
+    # Feature engineering
+    df = calculate_ages(df)
+
+    # Final missing value handling
+    df = df.fillna(UNKNOWN_TOKEN)
+
+    # One-hot encode multi-drug regimen columns
+    df = split_and_one_hot_encode(df, 'PreHSCT conditioning regimen+/-ATG+/-TBI', 'PreHSCT')
+    df = split_and_one_hot_encode(df, 'First_GVHD prophylaxis', 'First_GVHD_prophylaxis')
+    # df = split_and_one_hot_encode(df, 'Post HSCT regimen', 'PostHSCT')
+
+    # Normalize blood groups
+    df = merge_blood_groups(df, "Recepient_Blood group before HSCT", "Recepient_Blood group before HSCT_MergePlusMinus")
+    df = merge_blood_groups(df, "D_Blood group", "D_Blood group_MergePlusMinus")
+
+    # Binarize ages
+    df = binarize_age(df, "R_Age_at_transplant", 16, "R_Age_at_transplant_cutoff16")
+    df = binarize_age(df, "R_Age_at_transplant", 18, "R_Age_at_transplant_cutoff18")
+    df = binarize_age(df, "D_Age_at_transplant", 16, "D_Age_at_transplant_cutoff16")
+    df = binarize_age(df, "D_Age_at_transplant", 18, "D_Age_at_transplant_cutoff18")
+
+    # Gender/Relation features
+    df = add_gender_relation_features(df)
+
+    # Group nationalities
+    df = apply_nationality_groupings(df, 'Recepient_Nationality', groupings)
+
+    # Group and binarize diagnosis
+    df = group_and_binarize_diagnosis(df, 'Hematological Diagnosis', diagnosis_group_map, malignant_map)
+
+    df = df.replace(UNKNOWN_TOKEN, np.nan)
+
+    # Add columns for new dfs for features that exist in the original dataset but not in the new one
+    for feature in load_train_features()[0]:
+        if ("_HLA" in feature or "First_GVHD_prophylaxis_" in feature or "PreHSCT_" in feature) and feature not in df.columns:
+            df[feature] = 0
+
+    return df
+
+if __name__ == "__main__":
+    processed_data = preprocess_pipeline(
+        "/home/muhammadridzuan/2025_GVHD/2024_GVHD_SSMC/GVHD_Intel_data_MBZUAI_1.2.csv"
+    )
+    processed_data.to_csv("preprocessed_gvhd_data.csv", index=False)

src/saved_models/250706_150941_corr_drug_names_single.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69baff3c0aaedf52175dfb01c7663031e988b668eb8c7b4fa03d920de43265ce
+size 149312

src/saved_models/250706_150942_corr_drug_names_ensemble.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71fe613ec10104d24e5d4623f053e46bf1abce9da257b8190ca6cea4a72ed7a5
+size 855627

src/sidebar.py

@@ -0,0 +1,54 @@
+import streamlit as st
+from pathlib import Path
+import glob
+from huggingface_hub import HfApi, hf_hub_download
+import pyarrow.parquet as pq
+
+# def sidebar():
+#     APP_DIR = Path(__file__).parent
+#     MODELS_DIR = APP_DIR / "saved_models"
+
+#     # Shared dropdown in the sidebar
+#     def get_model_options():
+#         models = ["Default"]
+#         model_files = glob.glob(str(MODELS_DIR / "*.pkl")) + glob.glob(str(MODELS_DIR / "*.cbm"))
+
+#         for m in model_files:
+#             models.append(Path(m).stem)
+#         return sorted(set(models))
+
+#     if 'selected_model' not in st.session_state:
+#         st.session_state.selected_model = "Default"
+
+#     st.sidebar.title("Model Selection")
+#     st.session_state.selected_model = st.sidebar.selectbox("Model", get_model_options())
+
+def sidebar():
+    def get_model_options():
+        models = ["Default"]
+        api = HfApi(token=st.secrets["HF_TOKEN"])
+        all_files = api.list_repo_files(repo_id=st.secrets["HF_REPO_ID"], repo_type="dataset")
+        parquet_files = [f for f in all_files if f.startswith("models/") and f.endswith(".parquet")]
+
+        for f in parquet_files:
+            try:
+                # Download and read Parquet file
+                downloaded = hf_hub_download(
+                    repo_id=st.secrets["HF_REPO_ID"],
+                    repo_type="dataset",
+                    filename=f,
+                    token=st.secrets["HF_TOKEN"]
+                )
+                table = pq.read_table(downloaded)
+                row = table.to_pylist()[0]
+                models.append(row["filename"])
+            except Exception as e:
+                st.warning(f"Skipping model file due to error: {f} ({e})")
+
+        return sorted(set(models))
+
+    if 'selected_model' not in st.session_state:
+        st.session_state.selected_model = "Default"
+
+    st.sidebar.title("Model Selection")
+    st.session_state.selected_model = st.sidebar.selectbox("Model", get_model_options())