{ "cells": [ { "cell_type": "code", "execution_count": 48, "id": "d0a81c59", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.preprocessing import LabelEncoder,StandardScaler\n", "from sklearn.metrics import classification_report,confusion_matrix\n", "from sklearn.linear_model import LogisticRegression\n", "from imblearn.over_sampling import SMOTE\n", "from xgboost import XGBClassifier" ] }, { "cell_type": "code", "execution_count": 2, "id": "efc8e955", "metadata": {}, "outputs": [], "source": [ "le=LabelEncoder()\n", "scaler=StandardScaler()\n", "scaler_new=StandardScaler()\n", "model_rf=RandomForestClassifier(random_state=42,n_estimators=100)\n", "model_rf_new=RandomForestClassifier(random_state=42,n_estimators=100)\n", "model_rf_new2=RandomForestClassifier(random_state=42)\n", "model_log_reg=LogisticRegression(random_state=42)\n", "le_new=LabelEncoder()\n", "smote=SMOTE(random_state=42)" ] }, { "cell_type": "code", "execution_count": 3, "id": "591fef9f", "metadata": {}, "outputs": [], "source": [ "df=pd.read_csv('../Data/cancer-risk-factors.csv')" ] }, { "cell_type": "code", "execution_count": 4, "id": "462e2140", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Patient_IDCancer_TypeAgeGenderSmokingAlcohol_UseObesityFamily_HistoryDiet_Red_MeatDiet_Salted_ProcessedFruit_Veg_IntakePhysical_ActivityAir_PollutionOccupational_HazardsBRCA_MutationH_Pylori_InfectionCalcium_IntakeOverall_Risk_ScoreBMIPhysical_Activity_LevelRisk_Level
0LU0000Breast68072805374631000.39869628.05Medium
1LU0001Prostate74189800371330050.42429925.49Medium
2LU0002Skin5517107033418100060.60508228.62Medium
3LU0003Colon61062206246480080.31844932.17Low
4LU0004Lung67110740631091090050.52435825.12Medium
\n", "
" ], "text/plain": [ " Patient_ID Cancer_Type Age ... BMI Physical_Activity_Level Risk_Level\n", "0 LU0000 Breast 68 ... 28.0 5 Medium\n", "1 LU0001 Prostate 74 ... 25.4 9 Medium\n", "2 LU0002 Skin 55 ... 28.6 2 Medium\n", "3 LU0003 Colon 61 ... 32.1 7 Low\n", "4 LU0004 Lung 67 ... 25.1 2 Medium\n", "\n", "[5 rows x 21 columns]" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": 5, "id": "c6ba8df2", "metadata": {}, "outputs": [], "source": [ "x=df.drop(columns=['Risk_Level','Patient_ID','Cancer_Type'])\n", "\n", "y=le.fit_transform(df['Risk_Level'])" ] }, { "cell_type": "code", "execution_count": 6, "id": "4beb6abe", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
AgeGenderSmokingAlcohol_UseObesityFamily_HistoryDiet_Red_MeatDiet_Salted_ProcessedFruit_Veg_IntakePhysical_ActivityAir_PollutionOccupational_HazardsBRCA_MutationH_Pylori_InfectionCalcium_IntakeOverall_Risk_ScoreBMIPhysical_Activity_Level
068072805374631000.39869628.05
174189800371330050.42429925.49
25517107033418100060.60508228.62
361062206246480080.31844932.17
467110740631091090050.52435825.12
.........................................................
1995601464010644531040.43753930.33
1996841578010012130020.45112825.94
1997650721004223600100.29576022.53
1998641102100210754200100.42220125.33
1999640341000510390000.51813723.03
\n", "

2000 rows × 18 columns

\n", "
" ], "text/plain": [ " Age Gender Smoking ... Overall_Risk_Score BMI Physical_Activity_Level\n", "0 68 0 7 ... 0.398696 28.0 5\n", "1 74 1 8 ... 0.424299 25.4 9\n", "2 55 1 7 ... 0.605082 28.6 2\n", "3 61 0 6 ... 0.318449 32.1 7\n", "4 67 1 10 ... 0.524358 25.1 2\n", "... ... ... ... ... ... ... ...\n", "1995 60 1 4 ... 0.437539 30.3 3\n", "1996 84 1 5 ... 0.451128 25.9 4\n", "1997 65 0 7 ... 0.295760 22.5 3\n", "1998 64 1 10 ... 0.422201 25.3 3\n", "1999 64 0 3 ... 0.518137 23.0 3\n", "\n", "[2000 rows x 18 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x" ] }, { "cell_type": "code", "execution_count": 7, "id": "e2d81eed", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([2, 2, 2, ..., 1, 2, 2], shape=(2000,))" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y" ] }, { "cell_type": "markdown", "id": "ad5c485e", "metadata": {}, "source": [ "## why cancer_type was removed from the features list\n", "\n", "- It's not a predictive input - it's an outcome label or categorical grouping that's already strongly correlated with Risk Level\n", "\n", "- If we include it, the model will cheat by learning the mapping like Prostate --> High risk, instead of learning from true risk factors (smoking, BMI etc)" ] }, { "cell_type": "code", "execution_count": 8, "id": "69038132", "metadata": {}, "outputs": [], "source": [ "x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.2,random_state=42,stratify=y)" ] }, { "cell_type": "code", "execution_count": 9, "id": "dd0c8622", "metadata": {}, "outputs": [], "source": [ "x_train=scaler.fit_transform(x_train)\n", "x_test=scaler.transform(x_test)" ] }, { "cell_type": "code", "execution_count": 10, "id": "c396aa63", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
RandomForestClassifier(random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], "text/plain": [ "RandomForestClassifier(random_state=42)" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model_rf.fit(x_train,y_train)" ] }, { "cell_type": "code", "execution_count": 11, "id": "ffcc47b3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Classification Report \n", " precision recall f1-score support\n", "\n", " High 1.00 0.95 0.97 20\n", " Low 1.00 1.00 1.00 65\n", " Medium 1.00 1.00 1.00 315\n", "\n", " accuracy 1.00 400\n", " macro avg 1.00 0.98 0.99 400\n", "weighted avg 1.00 1.00 1.00 400\n", "\n" ] } ], "source": [ "y_pred_model_rf=model_rf.predict(x_test)\n", "\n", "print(\"Classification Report \")\n", "print(classification_report(y_test,y_pred_model_rf,target_names=le.classes_))" ] }, { "cell_type": "markdown", "id": "5bf5f8cd", "metadata": {}, "source": [ "The model accurately distinguishes all risk levels, with only 1 mistake out of 400 with imbalanced dataset." ] }, { "cell_type": "markdown", "id": "d9f39919", "metadata": {}, "source": [ "## Logistic Regression" ] }, { "cell_type": "code", "execution_count": 12, "id": "cfc18e05", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
LogisticRegression(random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], "text/plain": [ "LogisticRegression(random_state=42)" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model_log_reg.fit(x_train,y_train)" ] }, { "cell_type": "code", "execution_count": 13, "id": "630dedaa", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Classification Report \n", "\n", " precision recall f1-score support\n", "\n", " High 1.00 0.80 0.89 20\n", " Low 0.97 0.95 0.96 65\n", " Medium 0.98 0.99 0.99 315\n", "\n", " accuracy 0.98 400\n", " macro avg 0.98 0.92 0.95 400\n", "weighted avg 0.98 0.98 0.98 400\n", "\n", "Confusion Matrix\n", "\n", "[[ 16 0 4]\n", " [ 0 62 3]\n", " [ 0 2 313]]\n" ] } ], "source": [ "y_pred_model_log_reg=model_log_reg.predict(x_test)\n", "\n", "print(\"Classification Report \\n\")\n", "print(classification_report(y_test,y_pred_model_log_reg,target_names=le.classes_))\n", "\n", "print(\"Confusion Matrix\\n\")\n", "print(confusion_matrix(y_test,y_pred_model_log_reg))" ] }, { "cell_type": "markdown", "id": "0140bb07", "metadata": {}, "source": [ "Accuracy = 0.98" ] }, { "cell_type": "markdown", "id": "4ab14bd8", "metadata": {}, "source": [ "High Risk: 16 correctly predicted, 4 misclassified as Medium → model struggles a bit to distinguish “High” vs “Medium”.\n", "\n", "Low & Medium: almost perfectly predicted (only 5 total misclassifications)." ] }, { "cell_type": "markdown", "id": "68bd60b1", "metadata": {}, "source": [ "## Removing Overall_Risk_Score and retrying" ] }, { "cell_type": "code", "execution_count": 14, "id": "a6fbacd9", "metadata": {}, "outputs": [], "source": [ "x_new=df.drop(columns=['Risk_Level','Patient_ID','Cancer_Type','Overall_Risk_Score'])\n", "\n", "y_new=le_new.fit_transform(df['Risk_Level'])" ] }, { "cell_type": "code", "execution_count": 15, "id": "ae0dafd4", "metadata": {}, "outputs": [], "source": [ "x_train_new,x_test_new,y_train_new,y_test_new=train_test_split(x_new,y_new,test_size=0.2,random_state=42,stratify=y)" ] }, { "cell_type": "code", "execution_count": 16, "id": "1c31c5be", "metadata": {}, "outputs": [], "source": [ "x_train_new=scaler_new.fit_transform(x_train_new)\n", "x_test_new=scaler_new.transform(x_test_new)" ] }, { "cell_type": "code", "execution_count": 17, "id": "93a00c2e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
RandomForestClassifier(random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], "text/plain": [ "RandomForestClassifier(random_state=42)" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model_rf_new.fit(x_train_new,y_train_new)" ] }, { "cell_type": "code", "execution_count": 18, "id": "35c79d1b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Classification Report\n", " precision recall f1-score support\n", "\n", " High 1.00 0.05 0.10 20\n", " Low 0.85 0.34 0.48 65\n", " Medium 0.83 0.99 0.90 315\n", "\n", " accuracy 0.83 400\n", " macro avg 0.89 0.46 0.49 400\n", "weighted avg 0.84 0.83 0.80 400\n", "\n", "Confusion Matrix\n", "[[ 1 0 19]\n", " [ 0 22 43]\n", " [ 0 4 311]]\n" ] } ], "source": [ "y_pred_model_rf_new=model_rf_new.predict(x_test_new)\n", "\n", "print(\"Classification Report\")\n", "print(classification_report(y_test_new,y_pred_model_rf_new,target_names=le.classes_))\n", "\n", "print(\"Confusion Matrix\")\n", "print(confusion_matrix(y_test_new,y_pred_model_rf_new))" ] }, { "cell_type": "code", "execution_count": 19, "id": "48e99065", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Risk_Level\n", "Medium 1574\n", "Low 324\n", "High 102\n", "Name: count, dtype: int64" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.Risk_Level.value_counts()" ] }, { "cell_type": "markdown", "id": "3433e6d7", "metadata": {}, "source": [ "## SMOTE - Synthetic Minority Over-Sampling Technique" ] }, { "cell_type": "code", "execution_count": 20, "id": "b172a680", "metadata": {}, "outputs": [], "source": [ "x=df.drop(columns=['Risk_Level','Patient_ID','Cancer_Type','Overall_Risk_Score'])\n", "y=df['Risk_Level']" ] }, { "cell_type": "code", "execution_count": 21, "id": "5f0906ce", "metadata": {}, "outputs": [], "source": [ "x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.2,random_state=42,stratify=y)" ] }, { "cell_type": "code", "execution_count": 22, "id": "b12b1a05", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Class Distribution\n", "Risk_Level\n", "Medium 1259\n", "Low 1259\n", "High 1259\n", "Name: count, dtype: int64\n" ] } ], "source": [ "x_train_res,y_train_res=smote.fit_resample(x_train,y_train)\n", "\n", "print(\"Class Distribution\")\n", "print(y_train_res.value_counts())" ] }, { "cell_type": "code", "execution_count": 23, "id": "bebf6f0a", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
RandomForestClassifier(random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ], "text/plain": [ "RandomForestClassifier(random_state=42)" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model_rf_new2.fit(x_train_res,y_train_res)" ] }, { "cell_type": "code", "execution_count": 24, "id": "5d9c3eaa", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Classification Report\n", " precision recall f1-score support\n", "\n", " High 0.40 0.30 0.34 20\n", " Low 0.73 0.57 0.64 65\n", " Medium 0.87 0.93 0.90 315\n", "\n", " accuracy 0.84 400\n", " macro avg 0.67 0.60 0.63 400\n", "weighted avg 0.83 0.84 0.83 400\n", "\n", "Confusion Matrix \n", "[[ 6 0 14]\n", " [ 0 37 28]\n", " [ 9 14 292]]\n" ] } ], "source": [ "y_pred_model_rf_new2=model_rf_new2.predict(x_test)\n", "\n", "print(\"Classification Report\")\n", "print(classification_report(y_test,y_pred_model_rf_new2))\n", "\n", "print(\"Confusion Matrix \")\n", "print(confusion_matrix(y_test,y_pred_model_rf_new2))" ] }, { "cell_type": "markdown", "id": "1a2a7294", "metadata": {}, "source": [ "- Accuracy: 84%\n", "- High Risk: 7 correctly classified, 13 mis-classified as Medium\n", "- Low Risk: 38 correctly classified, 27 confused with Medium\n", "- Medium Risk: 291 correctly classified, 24 confused as High/Low\n", "\n", "- Still struggles to detect High-risk patients even with SMOTE" ] }, { "cell_type": "markdown", "id": "8e633058", "metadata": {}, "source": [ "## Optuna Tuning" ] }, { "cell_type": "code", "execution_count": 25, "id": "120c1bb7", "metadata": {}, "outputs": [], "source": [ "import optuna\n", "from optuna.samplers import TPESampler\n", "from sklearn.metrics import classification_report,confusion_matrix,make_scorer,f1_score\n", "from sklearn.model_selection import StratifiedKFold,cross_val_score\n", "\n", "import numpy as np\n", "import joblib\n" ] }, { "cell_type": "code", "execution_count": 26, "id": "596486fd", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(3777, 17)" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x_train_res.shape" ] }, { "cell_type": "code", "execution_count": 27, "id": "21d8d71e", "metadata": {}, "outputs": [], "source": [ "def macro_f1(y_true,y_pred):\n", " return f1_score(y_true,y_pred,average='macro')\n", "\n", "scorer=make_scorer(macro_f1)" ] }, { "cell_type": "code", "execution_count": 37, "id": "3cf7e970", "metadata": {}, "outputs": [], "source": [ "def objective(trial):\n", " params={\n", " 'n_estimators':trial.suggest_int('n_estimators',50,500),\n", " 'max_depth':trial.suggest_int('max_depth',3,30),\n", " 'min_samples_split':trial.suggest_int('min_samples_split',2,10),\n", " 'min_samples_leaf':trial.suggest_int('min_samples_leaf',1,10),\n", " 'max_features':trial.suggest_categorical('max_features',['sqrt','log2']),\n", " 'bootstrap':trial.suggest_categorical('bootstrap',[True,False]),\n", " 'criterion':trial.suggest_categorical('criterion',['gini','entropy']),\n", " 'random_state':42,\n", " 'n_jobs':-1\n", " }\n", "\n", " model=RandomForestClassifier(**params)\n", " cv=StratifiedKFold(n_splits=3,shuffle=True,random_state=42)\n", " scores=[]\n", "\n", " for train_idx,val_idx in cv.split(x_train_res,y_train_res):\n", " x_t,x_v=x_train_res.iloc[train_idx],x_train_res.iloc[val_idx]\n", " y_t,y_v=y_train_res.iloc[train_idx],y_train_res.iloc[val_idx]\n", "\n", " model.fit(x_t,y_t)\n", " y_pred=model.predict(x_v)\n", "\n", " score=f1_score(y_v,y_pred,average='macro')\n", "\n", " scores.append(score)\n", "\n", " return np.mean(scores)" ] }, { "cell_type": "code", "execution_count": 38, "id": "c289c5dd", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:15,640]\u001b[0m A new study created in memory with name: f1_macro_f1\u001b[0m\n" ] } ], "source": [ "sampler=TPESampler(seed=42)\n", "study=optuna.create_study(direction='maximize',sampler=sampler,study_name='f1_macro_f1')\n" ] }, { "cell_type": "code", "execution_count": 39, "id": "04a7d79e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 0. Best value: 0.940225: 2%|▏ | 1/50 [00:01<01:28, 1.81s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:19,063]\u001b[0m Trial 0 finished with value: 0.9402245055803525 and parameters: {'n_estimators': 218, 'max_depth': 29, 'min_samples_split': 8, 'min_samples_leaf': 6, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 0 with value: 0.9402245055803525.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 1. Best value: 0.948605: 4%|▍ | 2/50 [00:02<00:53, 1.12s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:19,705]\u001b[0m Trial 1 finished with value: 0.9486052688620008 and parameters: {'n_estimators': 59, 'max_depth': 30, 'min_samples_split': 9, 'min_samples_leaf': 3, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 1 with value: 0.9486052688620008.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 1. Best value: 0.948605: 6%|▌ | 3/50 [00:04<01:18, 1.67s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:22,028]\u001b[0m Trial 2 finished with value: 0.9043278095227171 and parameters: {'n_estimators': 325, 'max_depth': 6, 'min_samples_split': 4, 'min_samples_leaf': 4, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 1 with value: 0.9486052688620008.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 1. Best value: 0.948605: 8%|▊ | 4/50 [00:07<01:41, 2.21s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:25,077]\u001b[0m Trial 3 finished with value: 0.9075875044390282 and parameters: {'n_estimators': 324, 'max_depth': 7, 'min_samples_split': 2, 'min_samples_leaf': 10, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 1 with value: 0.9486052688620008.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 1. Best value: 0.948605: 10%|█ | 5/50 [00:08<01:18, 1.74s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:25,976]\u001b[0m Trial 4 finished with value: 0.9256181230083148 and parameters: {'n_estimators': 105, 'max_depth': 16, 'min_samples_split': 2, 'min_samples_leaf': 10, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 1 with value: 0.9486052688620008.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 1. Best value: 0.948605: 12%|█▏ | 6/50 [00:12<01:43, 2.36s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:29,539]\u001b[0m Trial 5 finished with value: 0.9322147044821582 and parameters: {'n_estimators': 487, 'max_depth': 24, 'min_samples_split': 10, 'min_samples_leaf': 9, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 1 with value: 0.9486052688620008.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 1. Best value: 0.948605: 14%|█▍ | 7/50 [00:14<01:33, 2.17s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:31,305]\u001b[0m Trial 6 finished with value: 0.9430899140401174 and parameters: {'n_estimators': 225, 'max_depth': 10, 'min_samples_split': 9, 'min_samples_leaf': 4, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 1 with value: 0.9486052688620008.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 1. Best value: 0.948605: 16%|█▌ | 8/50 [00:17<01:51, 2.66s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:35,019]\u001b[0m Trial 7 finished with value: 0.91313982534829 and parameters: {'n_estimators': 398, 'max_depth': 8, 'min_samples_split': 2, 'min_samples_leaf': 9, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 1 with value: 0.9486052688620008.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 8. Best value: 0.954463: 18%|█▊ | 9/50 [00:21<02:07, 3.11s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:39,124]\u001b[0m Trial 8 finished with value: 0.9544632976825597 and parameters: {'n_estimators': 439, 'max_depth': 20, 'min_samples_split': 4, 'min_samples_leaf': 1, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 8 with value: 0.9544632976825597.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 8. Best value: 0.954463: 20%|██ | 10/50 [00:23<01:40, 2.51s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:40,284]\u001b[0m Trial 9 finished with value: 0.9285765516979954 and parameters: {'n_estimators': 103, 'max_depth': 22, 'min_samples_split': 8, 'min_samples_leaf': 6, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 8 with value: 0.9544632976825597.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 8. Best value: 0.954463: 22%|██▏ | 11/50 [00:27<02:03, 3.16s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:44,908]\u001b[0m Trial 10 finished with value: 0.953107098740316 and parameters: {'n_estimators': 487, 'max_depth': 16, 'min_samples_split': 5, 'min_samples_leaf': 1, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 8 with value: 0.9544632976825597.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 8. Best value: 0.954463: 24%|██▍ | 12/50 [00:32<02:15, 3.56s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:49,385]\u001b[0m Trial 11 finished with value: 0.9531109399921487 and parameters: {'n_estimators': 492, 'max_depth': 16, 'min_samples_split': 5, 'min_samples_leaf': 1, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 8 with value: 0.9544632976825597.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 8. Best value: 0.954463: 26%|██▌ | 13/50 [00:35<02:13, 3.62s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:53,147]\u001b[0m Trial 12 finished with value: 0.9525673166202905 and parameters: {'n_estimators': 417, 'max_depth': 17, 'min_samples_split': 5, 'min_samples_leaf': 1, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 8 with value: 0.9544632976825597.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 8. Best value: 0.954463: 28%|██▊ | 14/50 [00:39<02:12, 3.69s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:17:57,007]\u001b[0m Trial 13 finished with value: 0.9493478379094 and parameters: {'n_estimators': 421, 'max_depth': 20, 'min_samples_split': 6, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 8 with value: 0.9544632976825597.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 8. Best value: 0.954463: 30%|███ | 15/50 [00:44<02:18, 3.94s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:01,540]\u001b[0m Trial 14 finished with value: 0.9519922293229085 and parameters: {'n_estimators': 496, 'max_depth': 12, 'min_samples_split': 4, 'min_samples_leaf': 1, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 8 with value: 0.9544632976825597.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 8. Best value: 0.954463: 32%|███▏ | 16/50 [00:47<02:06, 3.71s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:04,716]\u001b[0m Trial 15 finished with value: 0.8020546065458228 and parameters: {'n_estimators': 359, 'max_depth': 3, 'min_samples_split': 6, 'min_samples_leaf': 3, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 8 with value: 0.9544632976825597.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 8. Best value: 0.954463: 34%|███▍ | 17/50 [00:51<02:06, 3.82s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:08,785]\u001b[0m Trial 16 finished with value: 0.950137110636568 and parameters: {'n_estimators': 439, 'max_depth': 25, 'min_samples_split': 4, 'min_samples_leaf': 2, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 8 with value: 0.9544632976825597.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 8. Best value: 0.954463: 36%|███▌ | 18/50 [00:53<01:48, 3.40s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:11,212]\u001b[0m Trial 17 finished with value: 0.9496028656158345 and parameters: {'n_estimators': 252, 'max_depth': 14, 'min_samples_split': 3, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 8 with value: 0.9544632976825597.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 8. Best value: 0.954463: 38%|███▊ | 19/50 [00:58<01:54, 3.68s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:15,547]\u001b[0m Trial 18 finished with value: 0.9330966850779684 and parameters: {'n_estimators': 452, 'max_depth': 19, 'min_samples_split': 6, 'min_samples_leaf': 5, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 8 with value: 0.9544632976825597.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 8. Best value: 0.954463: 40%|████ | 20/50 [01:01<01:48, 3.61s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:18,987]\u001b[0m Trial 19 finished with value: 0.945331168136364 and parameters: {'n_estimators': 370, 'max_depth': 27, 'min_samples_split': 7, 'min_samples_leaf': 3, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 8 with value: 0.9544632976825597.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 42%|████▏ | 21/50 [01:04<01:38, 3.40s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:21,913]\u001b[0m Trial 20 finished with value: 0.9549380893493572 and parameters: {'n_estimators': 308, 'max_depth': 21, 'min_samples_split': 3, 'min_samples_leaf': 1, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 44%|████▍ | 22/50 [01:07<01:28, 3.16s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:24,510]\u001b[0m Trial 21 finished with value: 0.9541063663071631 and parameters: {'n_estimators': 276, 'max_depth': 21, 'min_samples_split': 3, 'min_samples_leaf': 1, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 46%|████▌ | 23/50 [01:10<01:22, 3.04s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:27,271]\u001b[0m Trial 22 finished with value: 0.9504203023568696 and parameters: {'n_estimators': 293, 'max_depth': 21, 'min_samples_split': 3, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 48%|████▊ | 24/50 [01:11<01:09, 2.67s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:29,080]\u001b[0m Trial 23 finished with value: 0.9535825379109207 and parameters: {'n_estimators': 181, 'max_depth': 23, 'min_samples_split': 3, 'min_samples_leaf': 1, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 50%|█████ | 25/50 [01:14<01:06, 2.65s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:31,685]\u001b[0m Trial 24 finished with value: 0.9506722889924931 and parameters: {'n_estimators': 279, 'max_depth': 18, 'min_samples_split': 3, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 52%|█████▏ | 26/50 [01:15<00:55, 2.31s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:33,188]\u001b[0m Trial 25 finished with value: 0.944441129652556 and parameters: {'n_estimators': 155, 'max_depth': 25, 'min_samples_split': 4, 'min_samples_leaf': 3, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 54%|█████▍ | 27/50 [01:19<00:58, 2.56s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:36,328]\u001b[0m Trial 26 finished with value: 0.9530063385194881 and parameters: {'n_estimators': 332, 'max_depth': 21, 'min_samples_split': 3, 'min_samples_leaf': 1, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 56%|█████▌ | 28/50 [01:21<00:56, 2.58s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:38,960]\u001b[0m Trial 27 finished with value: 0.9369888364058075 and parameters: {'n_estimators': 280, 'max_depth': 13, 'min_samples_split': 2, 'min_samples_leaf': 4, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 58%|█████▊ | 29/50 [01:25<01:00, 2.89s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:42,571]\u001b[0m Trial 28 finished with value: 0.9506829151551216 and parameters: {'n_estimators': 375, 'max_depth': 26, 'min_samples_split': 4, 'min_samples_leaf': 2, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 60%|██████ | 30/50 [01:27<00:51, 2.55s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:44,347]\u001b[0m Trial 29 finished with value: 0.9354010959947366 and parameters: {'n_estimators': 225, 'max_depth': 28, 'min_samples_split': 5, 'min_samples_leaf': 7, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 62%|██████▏ | 31/50 [01:28<00:43, 2.31s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:46,085]\u001b[0m Trial 30 finished with value: 0.9530400375710419 and parameters: {'n_estimators': 181, 'max_depth': 19, 'min_samples_split': 3, 'min_samples_leaf': 1, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 64%|██████▍ | 32/50 [01:30<00:39, 2.22s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:48,081]\u001b[0m Trial 31 finished with value: 0.9527759527018582 and parameters: {'n_estimators': 195, 'max_depth': 23, 'min_samples_split': 3, 'min_samples_leaf': 1, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 66%|██████▌ | 33/50 [01:32<00:34, 2.02s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:49,636]\u001b[0m Trial 32 finished with value: 0.9495756049913657 and parameters: {'n_estimators': 143, 'max_depth': 23, 'min_samples_split': 2, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 68%|██████▊ | 34/50 [01:34<00:34, 2.18s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:52,200]\u001b[0m Trial 33 finished with value: 0.9541932447278181 and parameters: {'n_estimators': 258, 'max_depth': 30, 'min_samples_split': 4, 'min_samples_leaf': 1, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 70%|███████ | 35/50 [01:36<00:31, 2.11s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:54,143]\u001b[0m Trial 34 finished with value: 0.953145556956542 and parameters: {'n_estimators': 249, 'max_depth': 30, 'min_samples_split': 4, 'min_samples_leaf': 3, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 72%|███████▏ | 36/50 [01:39<00:33, 2.37s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:57,129]\u001b[0m Trial 35 finished with value: 0.9506925838564144 and parameters: {'n_estimators': 320, 'max_depth': 28, 'min_samples_split': 4, 'min_samples_leaf': 2, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 20. Best value: 0.954938: 74%|███████▍ | 37/50 [01:41<00:29, 2.25s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:18:59,097]\u001b[0m Trial 36 finished with value: 0.9534119413984014 and parameters: {'n_estimators': 257, 'max_depth': 21, 'min_samples_split': 5, 'min_samples_leaf': 3, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 20 with value: 0.9549380893493572.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 37. Best value: 0.95548: 76%|███████▌ | 38/50 [01:44<00:29, 2.46s/it] " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:02,032]\u001b[0m Trial 37 finished with value: 0.9554797869857187 and parameters: {'n_estimators': 312, 'max_depth': 14, 'min_samples_split': 2, 'min_samples_leaf': 1, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 37 with value: 0.9554797869857187.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 37. Best value: 0.95548: 78%|███████▊ | 39/50 [01:47<00:26, 2.42s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:04,385]\u001b[0m Trial 38 finished with value: 0.9468216488082293 and parameters: {'n_estimators': 312, 'max_depth': 10, 'min_samples_split': 2, 'min_samples_leaf': 2, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 37 with value: 0.9554797869857187.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 39. Best value: 0.956278: 80%|████████ | 40/50 [01:50<00:27, 2.79s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:08,035]\u001b[0m Trial 39 finished with value: 0.9562781984861224 and parameters: {'n_estimators': 385, 'max_depth': 14, 'min_samples_split': 2, 'min_samples_leaf': 1, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 39 with value: 0.9562781984861224.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 39. Best value: 0.956278: 82%|████████▏ | 41/50 [01:54<00:26, 2.92s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:11,272]\u001b[0m Trial 40 finished with value: 0.9394173495088581 and parameters: {'n_estimators': 351, 'max_depth': 14, 'min_samples_split': 2, 'min_samples_leaf': 4, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 39 with value: 0.9562781984861224.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 39. Best value: 0.956278: 84%|████████▍ | 42/50 [01:58<00:26, 3.33s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:15,554]\u001b[0m Trial 41 finished with value: 0.9524341432874323 and parameters: {'n_estimators': 460, 'max_depth': 11, 'min_samples_split': 2, 'min_samples_leaf': 1, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 39 with value: 0.9562781984861224.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 39. Best value: 0.956278: 86%|████████▌ | 43/50 [02:01<00:23, 3.38s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:19,059]\u001b[0m Trial 42 finished with value: 0.9366957549814917 and parameters: {'n_estimators': 380, 'max_depth': 8, 'min_samples_split': 2, 'min_samples_leaf': 1, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 39 with value: 0.9562781984861224.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 39. Best value: 0.956278: 88%|████████▊ | 44/50 [02:05<00:20, 3.37s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:22,395]\u001b[0m Trial 43 finished with value: 0.9535863206069665 and parameters: {'n_estimators': 345, 'max_depth': 15, 'min_samples_split': 3, 'min_samples_leaf': 1, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 39 with value: 0.9562781984861224.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 39. Best value: 0.956278: 90%|█████████ | 45/50 [02:08<00:17, 3.51s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:26,241]\u001b[0m Trial 44 finished with value: 0.950670196352161 and parameters: {'n_estimators': 401, 'max_depth': 17, 'min_samples_split': 4, 'min_samples_leaf': 2, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 39 with value: 0.9562781984861224.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 39. Best value: 0.956278: 92%|█████████▏| 46/50 [02:11<00:13, 3.29s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:29,001]\u001b[0m Trial 45 finished with value: 0.8856118566735183 and parameters: {'n_estimators': 304, 'max_depth': 5, 'min_samples_split': 2, 'min_samples_leaf': 1, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 39 with value: 0.9562781984861224.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 39. Best value: 0.956278: 94%|█████████▍| 47/50 [02:14<00:09, 3.17s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:31,900]\u001b[0m Trial 46 finished with value: 0.9322110157504361 and parameters: {'n_estimators': 396, 'max_depth': 13, 'min_samples_split': 10, 'min_samples_leaf': 8, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 39 with value: 0.9562781984861224.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 39. Best value: 0.956278: 96%|█████████▌| 48/50 [02:18<00:06, 3.40s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:35,836]\u001b[0m Trial 47 finished with value: 0.940832576437288 and parameters: {'n_estimators': 432, 'max_depth': 9, 'min_samples_split': 4, 'min_samples_leaf': 2, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 39 with value: 0.9562781984861224.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 39. Best value: 0.956278: 98%|█████████▊| 49/50 [02:20<00:03, 3.02s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:37,979]\u001b[0m Trial 48 finished with value: 0.9431310986931073 and parameters: {'n_estimators': 226, 'max_depth': 18, 'min_samples_split': 5, 'min_samples_leaf': 3, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 39 with value: 0.9562781984861224.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 39. Best value: 0.956278: 100%|██████████| 50/50 [02:24<00:00, 2.90s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:19:42,170]\u001b[0m Trial 49 finished with value: 0.9552110413774174 and parameters: {'n_estimators': 461, 'max_depth': 15, 'min_samples_split': 3, 'min_samples_leaf': 1, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 39 with value: 0.9562781984861224.\u001b[0m\n", "Best Trail:\n", " Value (macro F1): 0.9562781984861224\n", " Params: \n", " n_estimators:385 \n", " max_depth:14 \n", " min_samples_split:2 \n", " min_samples_leaf:1 \n", " max_features:log2 \n", " bootstrap:True \n", " criterion:gini \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "n_trials=50\n", "study.optimize(objective,n_trials=n_trials,show_progress_bar=True)\n", "\n", "print(\"Best Trail:\")\n", "print(\" Value (macro F1):\",study.best_value)\n", "print(\" Params: \")\n", "for k , v in study.best_params.items():\n", " print(f\" {k}:{v} \")" ] }, { "cell_type": "code", "execution_count": 41, "id": "eb1945d1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " precision recall f1-score support\n", "\n", " High 0.38 0.30 0.33 20\n", " Low 0.71 0.57 0.63 65\n", " Medium 0.87 0.92 0.90 315\n", "\n", " accuracy 0.83 400\n", " macro avg 0.65 0.60 0.62 400\n", "weighted avg 0.82 0.83 0.83 400\n", "\n", "[[ 6 0 14]\n", " [ 0 37 28]\n", " [ 10 15 290]]\n" ] } ], "source": [ "best_params={\n", " 'n_estimators':395,\n", " 'max_depth':14,\n", " 'min_samples_split':2,\n", " 'min_samples_leaf':1,\n", " 'max_features':'log2',\n", " 'bootstrap':True,\n", " 'criterion':'gini',\n", " 'random_state':42,\n", " 'n_jobs':-1\n", "}\n", "\n", "final_rf=RandomForestClassifier(**best_params)\n", "final_rf.fit(x_train_res,y_train_res)\n", "y_pred_test=final_rf.predict(x_test)\n", "\n", "\n", "print(classification_report(y_test,y_pred_test))\n", "print(confusion_matrix(y_test,y_pred_test))" ] }, { "cell_type": "code", "execution_count": 44, "id": "bd77ec4b", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 13:40:39,625]\u001b[0m A new study created in memory with name: no-name-5fb595de-671b-40e6-88ef-08841bb8d048\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:40:41,523]\u001b[0m Trial 0 finished with value: 0.6781927193559647 and parameters: {'n_estimators': 212, 'max_depth': 20, 'min_samples_split': 7, 'min_samples_leaf': 4, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 0 with value: 0.6781927193559647.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:40:42,522]\u001b[0m Trial 1 finished with value: 0.6629638614896765 and parameters: {'n_estimators': 106, 'max_depth': 20, 'min_samples_split': 7, 'min_samples_leaf': 2, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 0 with value: 0.6781927193559647.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:40:44,740]\u001b[0m Trial 2 finished with value: 0.6619005300738027 and parameters: {'n_estimators': 284, 'max_depth': 8, 'min_samples_split': 4, 'min_samples_leaf': 3, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 0 with value: 0.6781927193559647.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:40:47,590]\u001b[0m Trial 3 finished with value: 0.6826791849574412 and parameters: {'n_estimators': 282, 'max_depth': 8, 'min_samples_split': 2, 'min_samples_leaf': 6, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 3 with value: 0.6826791849574412.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:40:48,922]\u001b[0m Trial 4 finished with value: 0.6866802084189415 and parameters: {'n_estimators': 136, 'max_depth': 13, 'min_samples_split': 2, 'min_samples_leaf': 6, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 4 with value: 0.6866802084189415.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:40:51,951]\u001b[0m Trial 5 finished with value: 0.6768940216351282 and parameters: {'n_estimators': 391, 'max_depth': 17, 'min_samples_split': 8, 'min_samples_leaf': 6, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 4 with value: 0.6866802084189415.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:40:53,774]\u001b[0m Trial 6 finished with value: 0.6727455325157291 and parameters: {'n_estimators': 216, 'max_depth': 10, 'min_samples_split': 7, 'min_samples_leaf': 3, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 4 with value: 0.6866802084189415.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:40:57,106]\u001b[0m Trial 7 finished with value: 0.674193138367836 and parameters: {'n_estimators': 332, 'max_depth': 8, 'min_samples_split': 2, 'min_samples_leaf': 5, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 4 with value: 0.6866802084189415.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:00,704]\u001b[0m Trial 8 finished with value: 0.6736621635699978 and parameters: {'n_estimators': 359, 'max_depth': 15, 'min_samples_split': 4, 'min_samples_leaf': 1, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 4 with value: 0.6866802084189415.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:02,331]\u001b[0m Trial 9 finished with value: 0.6868708615453808 and parameters: {'n_estimators': 135, 'max_depth': 16, 'min_samples_split': 7, 'min_samples_leaf': 4, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 9 with value: 0.6868708615453808.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:04,260]\u001b[0m Trial 10 finished with value: 0.66550679803404 and parameters: {'n_estimators': 180, 'max_depth': 13, 'min_samples_split': 5, 'min_samples_leaf': 1, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 9 with value: 0.6868708615453808.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:05,470]\u001b[0m Trial 11 finished with value: 0.6874234681111303 and parameters: {'n_estimators': 102, 'max_depth': 13, 'min_samples_split': 5, 'min_samples_leaf': 5, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 11 with value: 0.6874234681111303.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:07,085]\u001b[0m Trial 12 finished with value: 0.687353954078862 and parameters: {'n_estimators': 146, 'max_depth': 16, 'min_samples_split': 5, 'min_samples_leaf': 4, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 11 with value: 0.6874234681111303.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:08,300]\u001b[0m Trial 13 finished with value: 0.6839464367419095 and parameters: {'n_estimators': 103, 'max_depth': 12, 'min_samples_split': 5, 'min_samples_leaf': 4, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 11 with value: 0.6874234681111303.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:10,120]\u001b[0m Trial 14 finished with value: 0.6876073035910736 and parameters: {'n_estimators': 173, 'max_depth': 17, 'min_samples_split': 5, 'min_samples_leaf': 5, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 14 with value: 0.6876073035910736.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:12,121]\u001b[0m Trial 15 finished with value: 0.6842322294573115 and parameters: {'n_estimators': 193, 'max_depth': 18, 'min_samples_split': 4, 'min_samples_leaf': 5, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 14 with value: 0.6876073035910736.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:14,608]\u001b[0m Trial 16 finished with value: 0.6853742601491238 and parameters: {'n_estimators': 246, 'max_depth': 11, 'min_samples_split': 6, 'min_samples_leaf': 5, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 14 with value: 0.6876073035910736.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:16,388]\u001b[0m Trial 17 finished with value: 0.6875895278333811 and parameters: {'n_estimators': 169, 'max_depth': 14, 'min_samples_split': 3, 'min_samples_leaf': 5, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 14 with value: 0.6876073035910736.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:18,171]\u001b[0m Trial 18 finished with value: 0.688355799951779 and parameters: {'n_estimators': 166, 'max_depth': 18, 'min_samples_split': 3, 'min_samples_leaf': 3, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 18 with value: 0.688355799951779.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:20,397]\u001b[0m Trial 19 finished with value: 0.6838160448709697 and parameters: {'n_estimators': 219, 'max_depth': 18, 'min_samples_split': 3, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 18 with value: 0.688355799951779.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:23,004]\u001b[0m Trial 20 finished with value: 0.6884278429946876 and parameters: {'n_estimators': 252, 'max_depth': 19, 'min_samples_split': 3, 'min_samples_leaf': 3, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 20 with value: 0.6884278429946876.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:25,605]\u001b[0m Trial 21 finished with value: 0.6853869176285746 and parameters: {'n_estimators': 254, 'max_depth': 19, 'min_samples_split': 3, 'min_samples_leaf': 3, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 20 with value: 0.6884278429946876.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:27,447]\u001b[0m Trial 22 finished with value: 0.6876296664740956 and parameters: {'n_estimators': 165, 'max_depth': 18, 'min_samples_split': 3, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 20 with value: 0.6884278429946876.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:30,554]\u001b[0m Trial 23 finished with value: 0.6901941372868317 and parameters: {'n_estimators': 312, 'max_depth': 19, 'min_samples_split': 3, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 23 with value: 0.6901941372868317.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:33,678]\u001b[0m Trial 24 finished with value: 0.6804354531999861 and parameters: {'n_estimators': 315, 'max_depth': 20, 'min_samples_split': 3, 'min_samples_leaf': 3, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 23 with value: 0.6901941372868317.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:36,613]\u001b[0m Trial 25 finished with value: 0.6944101706169946 and parameters: {'n_estimators': 297, 'max_depth': 19, 'min_samples_split': 4, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 25 with value: 0.6944101706169946.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:39,685]\u001b[0m Trial 26 finished with value: 0.6944101706169946 and parameters: {'n_estimators': 298, 'max_depth': 19, 'min_samples_split': 4, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 25 with value: 0.6944101706169946.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:42,719]\u001b[0m Trial 27 finished with value: 0.6456514383884826 and parameters: {'n_estimators': 313, 'max_depth': 6, 'min_samples_split': 4, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 25 with value: 0.6944101706169946.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:45,585]\u001b[0m Trial 28 finished with value: 0.6702473269057848 and parameters: {'n_estimators': 286, 'max_depth': 19, 'min_samples_split': 4, 'min_samples_leaf': 1, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 25 with value: 0.6944101706169946.\u001b[0m\n", "\u001b[32m[I 2026-07-11 13:41:48,388]\u001b[0m Trial 29 finished with value: 0.6743933030469619 and parameters: {'n_estimators': 352, 'max_depth': 20, 'min_samples_split': 4, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 25 with value: 0.6944101706169946.\u001b[0m\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "0.6944101706169946 {'n_estimators': 297, 'max_depth': 19, 'min_samples_split': 4, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}\n" ] } ], "source": [ "from imblearn.pipeline import Pipeline as ImbPipeline\n", "from imblearn.over_sampling import SMOTE\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.model_selection import StratifiedKFold,cross_val_score\n", "from sklearn.metrics import f1_score,make_scorer,recall_score\n", "import optuna\n", "import numpy as np\n", "\n", "\n", "def macro_f1(y_true,y_pred):\n", " return f1_score(y_true,y_pred,average='macro')\n", "\n", "macro_f1_scorer=make_scorer(macro_f1)\n", "\n", "def objective(trial):\n", " params={\n", " 'n_estimators':trial.suggest_int('n_estimators',100,400),\n", " 'max_depth':trial.suggest_int('max_depth',6,20),\n", " 'min_samples_split':trial.suggest_int('min_samples_split',2,8),\n", " 'min_samples_leaf':trial.suggest_int('min_samples_leaf',1,6),\n", " 'max_features':trial.suggest_categorical('max_features',['sqrt','log2']),\n", " 'bootstrap':trial.suggest_categorical('bootstrap',[True,False]),\n", " 'criterion':trial.suggest_categorical('criterion',['gini','entropy']),\n", " 'random_state':42,\n", " 'n_jobs':-1\n", " \n", " }\n", "\n", "\n", " sm=SMOTE(random_state=42)\n", " rf=RandomForestClassifier(**params)\n", " pipe=ImbPipeline([('smote',sm),('rf',rf)])\n", "\n", " cv=StratifiedKFold(n_splits=3,shuffle=True,random_state=42)\n", " scores=cross_val_score(pipe,x_train,y_train,cv=cv,scoring=macro_f1_scorer,n_jobs=1)\n", " return float(np.mean(scores))\n", "\n", "study=optuna.create_study(direction='maximize',sampler=optuna.samplers.TPESampler(seed=42))\n", "study.optimize(objective,n_trials=30)\n", "print(study.best_value,study.best_params)" ] }, { "cell_type": "code", "execution_count": 46, "id": "2da851fd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " precision recall f1-score support\n", "\n", " High 0.33 0.30 0.32 20\n", " Low 0.70 0.62 0.66 65\n", " Medium 0.88 0.91 0.89 315\n", "\n", " accuracy 0.83 400\n", " macro avg 0.64 0.61 0.62 400\n", "weighted avg 0.82 0.83 0.83 400\n", "\n", "[[ 6 0 14]\n", " [ 0 40 25]\n", " [ 12 17 286]]\n" ] } ], "source": [ "best=study.best_params\n", "rf=RandomForestClassifier(**best,random_state=42,n_jobs=-1)\n", "\n", "pipe_final=ImbPipeline([('smote',SMOTE(random_state=42)),('rf',rf)])\n", "pipe_final.fit(x_train,y_train)\n", "\n", "y_test_pred=pipe_final.predict(x_test)\n", "print(classification_report(y_test,y_test_pred))\n", "print(confusion_matrix(y_test,y_test_pred))" ] }, { "cell_type": "markdown", "id": "8fdfd09e", "metadata": {}, "source": [ "## **XGBoost**" ] }, { "cell_type": "code", "execution_count": 49, "id": "ac83c271", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Baseline XGB Classifier\n", " precision recall f1-score support\n", "\n", " High 0.35 0.40 0.37 20\n", " Low 0.76 0.63 0.69 65\n", " Medium 0.89 0.91 0.90 315\n", "\n", " accuracy 0.84 400\n", " macro avg 0.67 0.65 0.65 400\n", "weighted avg 0.84 0.84 0.84 400\n", "\n", "Confusion Matrix\n", "[[ 8 0 12]\n", " [ 0 41 24]\n", " [ 15 13 287]]\n" ] } ], "source": [ "le=LabelEncoder()\n", "le.fit(y_train)\n", "\n", "y_train_enc=le.transform(y_train)\n", "y_test_enc=le.transform(y_test)\n", "\n", "xgb=XGBClassifier(use_label_encoder=False,eval_metrics='mlogloss',random_state=42,n_jobs=-1)\n", "pipe=ImbPipeline([\n", " ('smote',SMOTE(random_state=42)),\n", " ('xgb',xgb)\n", "])\n", "\n", "\n", "pipe.fit(x_train,y_train_enc)\n", "y_pred_enc=pipe.predict(x_test)\n", "\n", "y_pred=le.inverse_transform(y_pred_enc)\n", "y_test_orig=le.inverse_transform(y_test_enc)\n", "\n", "print(\"Baseline XGB Classifier\")\n", "print(classification_report(y_test_orig,y_pred))\n", "print(\"Confusion Matrix\")\n", "print(confusion_matrix(y_test_orig,y_pred))" ] }, { "cell_type": "code", "execution_count": null, "id": "cdbb0ed3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 55, "id": "184e344d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:28,459]\u001b[0m A new study created in memory with name: xgb_high_recall\u001b[0m\n", "Best trial: 0. Best value: 0.477513: 2%|▎ | 1/40 [00:01<01:00, 1.54s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:29,996]\u001b[0m Trial 0 finished with value: 0.4775132275132275 and parameters: {'n_estimators': 181, 'max_depth': 13, 'learning_rate': 0.1205712628744377, 'subsample': 0.8394633936788146, 'colsample_bytree': 0.4936111842654619, 'gamma': 0.7799726016810132, 'reg_alpha': 0.2904180608409973, 'reg_lambda': 4.330880728874676}. Best is trial 0 with value: 0.4775132275132275.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 0. Best value: 0.477513: 5%|▌ | 2/40 [00:04<01:33, 2.46s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:33,094]\u001b[0m Trial 1 finished with value: 0.4656084656084656 and parameters: {'n_estimators': 260, 'max_depth': 10, 'learning_rate': 0.010725209743171996, 'subsample': 0.9879639408647978, 'colsample_bytree': 0.899465584480253, 'gamma': 1.0616955533913808, 'reg_alpha': 0.9091248360355031, 'reg_lambda': 0.9170225492671691}. Best is trial 0 with value: 0.4775132275132275.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 2. Best value: 0.502646: 8%|▊ | 3/40 [00:06<01:17, 2.11s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:34,783]\u001b[0m Trial 2 finished with value: 0.5026455026455027 and parameters: {'n_estimators': 156, 'max_depth': 8, 'learning_rate': 0.04345454109729477, 'subsample': 0.7164916560792167, 'colsample_bytree': 0.7671117368334277, 'gamma': 0.6974693032602092, 'reg_alpha': 1.4607232426760908, 'reg_lambda': 1.8318092164684585}. Best is trial 2 with value: 0.5026455026455027.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 2. Best value: 0.502646: 10%|█ | 4/40 [00:09<01:25, 2.38s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:37,585]\u001b[0m Trial 3 finished with value: 0.5026455026455027 and parameters: {'n_estimators': 210, 'max_depth': 11, 'learning_rate': 0.019721610970574007, 'subsample': 0.8056937753654446, 'colsample_bytree': 0.7554487413172255, 'gamma': 0.23225206359998862, 'reg_alpha': 3.0377242595071916, 'reg_lambda': 0.8526206184364576}. Best is trial 2 with value: 0.5026455026455027.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 2. Best value: 0.502646: 12%|█▎ | 5/40 [00:09<01:01, 1.75s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:38,209]\u001b[0m Trial 4 finished with value: 0.4902998236331569 and parameters: {'n_estimators': 72, 'max_depth': 13, 'learning_rate': 0.26690431824362526, 'subsample': 0.9233589392465844, 'colsample_bytree': 0.5827682615040224, 'gamma': 0.48836057003191935, 'reg_alpha': 3.4211651325607844, 'reg_lambda': 2.2007624686980067}. Best is trial 2 with value: 0.5026455026455027.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 2. Best value: 0.502646: 15%|█▌ | 6/40 [00:10<00:50, 1.50s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:39,222]\u001b[0m Trial 5 finished with value: 0.4775132275132275 and parameters: {'n_estimators': 92, 'max_depth': 8, 'learning_rate': 0.011240768803005551, 'subsample': 0.9637281608315128, 'colsample_bytree': 0.5552679889600102, 'gamma': 3.31261142176991, 'reg_alpha': 1.5585553804470549, 'reg_lambda': 2.600340105889054}. Best is trial 2 with value: 0.5026455026455027.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 6. Best value: 0.611111: 18%|█▊ | 7/40 [00:11<00:45, 1.38s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:40,344]\u001b[0m Trial 6 finished with value: 0.6111111111111112 and parameters: {'n_estimators': 241, 'max_depth': 5, 'learning_rate': 0.27051668818999286, 'subsample': 0.9100531293444458, 'colsample_bytree': 0.9636993649385135, 'gamma': 4.474136752138244, 'reg_alpha': 2.9894998940554256, 'reg_lambda': 4.609371175115584}. Best is trial 6 with value: 0.6111111111111112.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 6. Best value: 0.611111: 20%|██ | 8/40 [00:13<00:41, 1.30s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:41,496]\u001b[0m Trial 7 finished with value: 0.587742504409171 and parameters: {'n_estimators': 81, 'max_depth': 5, 'learning_rate': 0.011662890273931383, 'subsample': 0.7301321323053057, 'colsample_bytree': 0.6332063738136893, 'gamma': 1.3567451588694794, 'reg_alpha': 4.143687545759647, 'reg_lambda': 1.7837666334679465}. Best is trial 6 with value: 0.6111111111111112.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 6. Best value: 0.611111: 22%|██▎ | 9/40 [00:14<00:42, 1.38s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:43,041]\u001b[0m Trial 8 finished with value: 0.5996472663139331 and parameters: {'n_estimators': 148, 'max_depth': 8, 'learning_rate': 0.016149614799999188, 'subsample': 0.9208787923016158, 'colsample_bytree': 0.44473038620786254, 'gamma': 4.9344346830025865, 'reg_alpha': 3.861223846483287, 'reg_lambda': 0.993578407670862}. Best is trial 6 with value: 0.6111111111111112.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 6. Best value: 0.611111: 25%|██▌ | 10/40 [00:15<00:37, 1.26s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:44,022]\u001b[0m Trial 9 finished with value: 0.4779541446208113 and parameters: {'n_estimators': 51, 'max_depth': 11, 'learning_rate': 0.11069143219393454, 'subsample': 0.8916028672163949, 'colsample_bytree': 0.8627622080115674, 'gamma': 0.3702232586704518, 'reg_alpha': 1.7923286427213632, 'reg_lambda': 0.5793452976256486}. Best is trial 6 with value: 0.6111111111111112.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 10. Best value: 0.623457: 28%|██▊ | 11/40 [00:17<00:45, 1.59s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:46,355]\u001b[0m Trial 10 finished with value: 0.6234567901234568 and parameters: {'n_estimators': 386, 'max_depth': 3, 'learning_rate': 0.038855487710266354, 'subsample': 0.6071847502459279, 'colsample_bytree': 0.9859842597031323, 'gamma': 2.620977036231838, 'reg_alpha': 4.7988537297270994, 'reg_lambda': 4.83141665027002}. Best is trial 10 with value: 0.6234567901234568.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 10. Best value: 0.623457: 30%|███ | 12/40 [00:20<00:49, 1.77s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:48,533]\u001b[0m Trial 11 finished with value: 0.6234567901234568 and parameters: {'n_estimators': 382, 'max_depth': 3, 'learning_rate': 0.04471816347733262, 'subsample': 0.6085253890492004, 'colsample_bytree': 0.9839977856238663, 'gamma': 2.842955664505752, 'reg_alpha': 4.93239217948423, 'reg_lambda': 4.837262261607134}. Best is trial 10 with value: 0.6234567901234568.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 10. Best value: 0.623457: 32%|███▎ | 13/40 [00:22<00:52, 1.93s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:50,848]\u001b[0m Trial 12 finished with value: 0.6234567901234568 and parameters: {'n_estimators': 392, 'max_depth': 3, 'learning_rate': 0.03913347433912368, 'subsample': 0.6014761105717582, 'colsample_bytree': 0.9899292817313973, 'gamma': 2.33881157458022, 'reg_alpha': 4.985551785544109, 'reg_lambda': 3.947385331597324}. Best is trial 10 with value: 0.6234567901234568.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 13. Best value: 0.635362: 35%|███▌ | 14/40 [00:24<00:52, 2.03s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:53,103]\u001b[0m Trial 13 finished with value: 0.6353615520282186 and parameters: {'n_estimators': 397, 'max_depth': 3, 'learning_rate': 0.03729616587885411, 'subsample': 0.6041112651819361, 'colsample_bytree': 0.8641676912511723, 'gamma': 2.7616775480277633, 'reg_alpha': 4.996812193207756, 'reg_lambda': 3.4965257693084375}. Best is trial 13 with value: 0.6353615520282186.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 13. Best value: 0.635362: 38%|███▊ | 15/40 [00:27<00:53, 2.15s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:55,547]\u001b[0m Trial 14 finished with value: 0.6234567901234568 and parameters: {'n_estimators': 333, 'max_depth': 5, 'learning_rate': 0.02739965716925839, 'subsample': 0.6638123593227387, 'colsample_bytree': 0.8258871687212778, 'gamma': 2.085139014428166, 'reg_alpha': 4.424651701191843, 'reg_lambda': 3.443405771251516}. Best is trial 13 with value: 0.6353615520282186.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 15. Best value: 0.659612: 40%|████ | 16/40 [00:28<00:46, 1.95s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:57,028]\u001b[0m Trial 15 finished with value: 0.6596119929453262 and parameters: {'n_estimators': 326, 'max_depth': 3, 'learning_rate': 0.08210947295749188, 'subsample': 0.6662225125541511, 'colsample_bytree': 0.7112497603982444, 'gamma': 3.6542473093839565, 'reg_alpha': 4.408445775547389, 'reg_lambda': 3.322562986675382}. Best is trial 15 with value: 0.6596119929453262.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 15. Best value: 0.659612: 42%|████▎ | 17/40 [00:30<00:41, 1.80s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:58,469]\u001b[0m Trial 16 finished with value: 0.6234567901234568 and parameters: {'n_estimators': 319, 'max_depth': 6, 'learning_rate': 0.08386521513949594, 'subsample': 0.6768058996875773, 'colsample_bytree': 0.6570645472111242, 'gamma': 3.7148705256210013, 'reg_alpha': 3.7661903543164272, 'reg_lambda': 3.197342331354388}. Best is trial 15 with value: 0.6596119929453262.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 15. Best value: 0.659612: 45%|████▌ | 18/40 [00:31<00:37, 1.71s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:21:59,958]\u001b[0m Trial 17 finished with value: 0.6472663139329806 and parameters: {'n_estimators': 321, 'max_depth': 4, 'learning_rate': 0.06774850204374368, 'subsample': 0.665260582832281, 'colsample_bytree': 0.7260683543798567, 'gamma': 3.9400000103510395, 'reg_alpha': 2.720110432378378, 'reg_lambda': 3.1392096722072846}. Best is trial 15 with value: 0.6596119929453262.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 15. Best value: 0.659612: 48%|████▊ | 19/40 [00:33<00:36, 1.72s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:01,705]\u001b[0m Trial 18 finished with value: 0.6234567901234568 and parameters: {'n_estimators': 326, 'max_depth': 6, 'learning_rate': 0.07326255805357453, 'subsample': 0.7419439234137122, 'colsample_bytree': 0.7240570120681563, 'gamma': 3.966543120628947, 'reg_alpha': 2.868447337527959, 'reg_lambda': 2.8633632148130945}. Best is trial 15 with value: 0.6596119929453262.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 15. Best value: 0.659612: 50%|█████ | 20/40 [00:34<00:31, 1.60s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:03,028]\u001b[0m Trial 19 finished with value: 0.6115520282186949 and parameters: {'n_estimators': 273, 'max_depth': 4, 'learning_rate': 0.15100456835238596, 'subsample': 0.7839944835498073, 'colsample_bytree': 0.67052673004047, 'gamma': 4.134336554829186, 'reg_alpha': 2.324049717777458, 'reg_lambda': 3.9995133564836687}. Best is trial 15 with value: 0.6596119929453262.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 15. Best value: 0.659612: 52%|█████▎ | 21/40 [00:36<00:30, 1.61s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:04,666]\u001b[0m Trial 20 finished with value: 0.6234567901234568 and parameters: {'n_estimators': 301, 'max_depth': 7, 'learning_rate': 0.06280774734188155, 'subsample': 0.6596846581153344, 'colsample_bytree': 0.7872639185096457, 'gamma': 3.5044011770131576, 'reg_alpha': 2.3938982551155354, 'reg_lambda': 2.6045735587848986}. Best is trial 15 with value: 0.6596119929453262.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 15. Best value: 0.659612: 55%|█████▌ | 22/40 [00:38<00:30, 1.72s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:06,634]\u001b[0m Trial 21 finished with value: 0.6353615520282186 and parameters: {'n_estimators': 357, 'max_depth': 4, 'learning_rate': 0.0644385659157019, 'subsample': 0.6445338180981789, 'colsample_bytree': 0.7112952324919727, 'gamma': 3.0294233014065366, 'reg_alpha': 4.422899565707845, 'reg_lambda': 3.579696989490311}. Best is trial 15 with value: 0.6596119929453262.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 15. Best value: 0.659612: 57%|█████▊ | 23/40 [00:39<00:28, 1.69s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:08,265]\u001b[0m Trial 22 finished with value: 0.6234567901234568 and parameters: {'n_estimators': 353, 'max_depth': 4, 'learning_rate': 0.1727900246355595, 'subsample': 0.6965645209738722, 'colsample_bytree': 0.8982162919137702, 'gamma': 1.933569335886307, 'reg_alpha': 4.429353924472363, 'reg_lambda': 3.040370585178839}. Best is trial 15 with value: 0.6596119929453262.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 60%|██████ | 24/40 [00:41<00:26, 1.66s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:09,847]\u001b[0m Trial 23 finished with value: 0.6600529100529101 and parameters: {'n_estimators': 290, 'max_depth': 3, 'learning_rate': 0.027779075143754845, 'subsample': 0.6369120198064522, 'colsample_bytree': 0.8282367128208368, 'gamma': 4.484606161412573, 'reg_alpha': 3.648944292538882, 'reg_lambda': 3.821346518465741}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 62%|██████▎ | 25/40 [00:43<00:25, 1.68s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:11,581]\u001b[0m Trial 24 finished with value: 0.6600529100529101 and parameters: {'n_estimators': 289, 'max_depth': 4, 'learning_rate': 0.024048392407115295, 'subsample': 0.6370875829833148, 'colsample_bytree': 0.7041754351247903, 'gamma': 4.813729577909058, 'reg_alpha': 3.5022435828227803, 'reg_lambda': 3.9607773323363813}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 65%|██████▌ | 26/40 [00:44<00:23, 1.65s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:13,172]\u001b[0m Trial 25 finished with value: 0.6477072310405644 and parameters: {'n_estimators': 285, 'max_depth': 6, 'learning_rate': 0.02559450975807772, 'subsample': 0.633805121886806, 'colsample_bytree': 0.8126786864582728, 'gamma': 4.817212876841958, 'reg_alpha': 3.440388801084951, 'reg_lambda': 3.975878281476729}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 68%|██████▊ | 27/40 [00:46<00:21, 1.67s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:14,885]\u001b[0m Trial 26 finished with value: 0.6243386243386243 and parameters: {'n_estimators': 236, 'max_depth': 4, 'learning_rate': 0.01598915657662053, 'subsample': 0.7649011048051568, 'colsample_bytree': 0.6065377995817691, 'gamma': 4.4915015925207635, 'reg_alpha': 3.5075590754557218, 'reg_lambda': 4.315763581020662}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 70%|███████ | 28/40 [00:48<00:19, 1.64s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:16,462]\u001b[0m Trial 27 finished with value: 0.6485890652557319 and parameters: {'n_estimators': 294, 'max_depth': 3, 'learning_rate': 0.023917660812065228, 'subsample': 0.6962724076714358, 'colsample_bytree': 0.6868162482977465, 'gamma': 4.623009888484231, 'reg_alpha': 3.719278953806693, 'reg_lambda': 3.7545727995862874}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 72%|███████▎ | 29/40 [00:49<00:17, 1.64s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:18,080]\u001b[0m Trial 28 finished with value: 0.6485890652557319 and parameters: {'n_estimators': 247, 'max_depth': 5, 'learning_rate': 0.0305788650439054, 'subsample': 0.6334830901784149, 'colsample_bytree': 0.536190864258645, 'gamma': 4.2546479074078, 'reg_alpha': 4.002145238530504, 'reg_lambda': 4.340493465153248}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 75%|███████▌ | 30/40 [00:51<00:15, 1.57s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:19,487]\u001b[0m Trial 29 finished with value: 0.6238977072310407 and parameters: {'n_estimators': 216, 'max_depth': 4, 'learning_rate': 0.05101402839891347, 'subsample': 0.6972361513351812, 'colsample_bytree': 0.4746879921037327, 'gamma': 3.2969941507735667, 'reg_alpha': 4.1785311802727865, 'reg_lambda': 4.359130206493319}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 78%|███████▊ | 31/40 [00:51<00:12, 1.38s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:20,424]\u001b[0m Trial 30 finished with value: 0.5873015873015873 and parameters: {'n_estimators': 195, 'max_depth': 3, 'learning_rate': 0.11273322653216135, 'subsample': 0.8489817519564029, 'colsample_bytree': 0.8194554716987092, 'gamma': 3.6225838713236196, 'reg_alpha': 0.2137259003982086, 'reg_lambda': 2.6571533424652394}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 80%|████████ | 32/40 [00:53<00:11, 1.42s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:21,957]\u001b[0m Trial 31 finished with value: 0.648148148148148 and parameters: {'n_estimators': 292, 'max_depth': 3, 'learning_rate': 0.02160820505018446, 'subsample': 0.6930064742807758, 'colsample_bytree': 0.672237465291253, 'gamma': 4.639498695978463, 'reg_alpha': 3.6236796671114826, 'reg_lambda': 3.6418042581999432}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 82%|████████▎ | 33/40 [00:55<00:10, 1.54s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:23,765]\u001b[0m Trial 32 finished with value: 0.648148148148148 and parameters: {'n_estimators': 353, 'max_depth': 3, 'learning_rate': 0.015616026078862817, 'subsample': 0.6361876830291481, 'colsample_bytree': 0.69843057697909, 'gamma': 4.967218967794195, 'reg_alpha': 3.256770071489237, 'reg_lambda': 3.74447647101747}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 85%|████████▌ | 34/40 [00:56<00:09, 1.53s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:25,286]\u001b[0m Trial 33 finished with value: 0.6477072310405644 and parameters: {'n_estimators': 267, 'max_depth': 5, 'learning_rate': 0.03090667440693108, 'subsample': 0.7150326231495643, 'colsample_bytree': 0.7622494623130056, 'gamma': 4.274628776972463, 'reg_alpha': 4.59187265715653, 'reg_lambda': 4.087313563208951}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 88%|████████▊ | 35/40 [00:58<00:07, 1.57s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:26,935]\u001b[0m Trial 34 finished with value: 0.5992063492063493 and parameters: {'n_estimators': 294, 'max_depth': 4, 'learning_rate': 0.02039902162716406, 'subsample': 0.7568763725495224, 'colsample_bytree': 0.6253028131743433, 'gamma': 4.631859686427428, 'reg_alpha': 0.6115771361104569, 'reg_lambda': 3.2949687606016207}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 90%|█████████ | 36/40 [01:00<00:06, 1.58s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:28,549]\u001b[0m Trial 35 finished with value: 0.6358024691358025 and parameters: {'n_estimators': 303, 'max_depth': 3, 'learning_rate': 0.023770382737560936, 'subsample': 0.6876417350588806, 'colsample_bytree': 0.7503436851749404, 'gamma': 3.8489745335386716, 'reg_alpha': 3.2016741061940577, 'reg_lambda': 3.781409709265974}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 92%|█████████▎| 37/40 [01:01<00:04, 1.42s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:29,590]\u001b[0m Trial 36 finished with value: 0.6366843033509699 and parameters: {'n_estimators': 161, 'max_depth': 4, 'learning_rate': 0.01335241576469445, 'subsample': 0.7210008993336884, 'colsample_bytree': 0.5745420116275524, 'gamma': 4.321085452497504, 'reg_alpha': 3.8078659675186763, 'reg_lambda': 2.2459193035880585}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 95%|█████████▌| 38/40 [01:02<00:02, 1.44s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:31,085]\u001b[0m Trial 37 finished with value: 0.648148148148148 and parameters: {'n_estimators': 256, 'max_depth': 6, 'learning_rate': 0.01808876165651747, 'subsample': 0.6537291656175717, 'colsample_bytree': 0.5209777116821823, 'gamma': 4.698289079945772, 'reg_alpha': 4.118013946410196, 'reg_lambda': 4.554348219408152}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 98%|█████████▊| 39/40 [01:06<00:02, 2.03s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:34,501]\u001b[0m Trial 38 finished with value: 0.4656084656084656 and parameters: {'n_estimators': 222, 'max_depth': 13, 'learning_rate': 0.0524783595517451, 'subsample': 0.6273082628990618, 'colsample_bytree': 0.9178093144705662, 'gamma': 0.01333980638640675, 'reg_alpha': 2.5780528218243477, 'reg_lambda': 2.9557778621550854}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 23. Best value: 0.660053: 100%|██████████| 40/40 [01:07<00:00, 1.70s/it]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 17:22:36,283]\u001b[0m Trial 39 finished with value: 0.6115520282186949 and parameters: {'n_estimators': 342, 'max_depth': 7, 'learning_rate': 0.20539910138006007, 'subsample': 0.8094244480684174, 'colsample_bytree': 0.789062391904634, 'gamma': 4.069859896585812, 'reg_alpha': 3.1116916273861417, 'reg_lambda': 4.98982669776982}. Best is trial 23 with value: 0.6600529100529101.\u001b[0m\n", "\n", " Best CV Mean High-Recall : 0.6600529100529101\n", "best Params\n", " n_estimators: 290\n", " max_depth: 3\n", " learning_rate: 0.027779075143754845\n", " subsample: 0.6369120198064522\n", " colsample_bytree: 0.8282367128208368\n", " gamma: 4.484606161412573\n", " reg_alpha: 3.648944292538882\n", " reg_lambda: 3.821346518465741\n", "\n", " Final Test Classification Report (labels in orignal names):\n", " precision recall f1-score support\n", "\n", " High 0.21 0.60 0.31 20\n", " Low 0.68 0.72 0.70 65\n", " Medium 0.91 0.79 0.84 315\n", "\n", " accuracy 0.77 400\n", " macro avg 0.60 0.70 0.62 400\n", "weighted avg 0.83 0.77 0.79 400\n", "\n", "Confusion Matrix (rows=true) , cols=predicted order=le.classes_\n", "[[ 12 0 8]\n", " [ 0 47 18]\n", " [ 45 22 248]]\n" ] } ], "source": [ "TARGET_LABEL='High'\n", "LABEL_ORDER=['High','Low','Medium']\n", "N_TRIALS=40\n", "CV_FOLDS=3\n", "\n", "le=LabelEncoder()\n", "le.fit(y_train)\n", "y_train_enc=le.transform(y_train)\n", "y_test_enc=le.transform(y_test)\n", "\n", "target_index=int(np.where(le.classes_==TARGET_LABEL)[0][0])\n", "\n", "\n", "def objective(trial):\n", " params={\n", " 'n_estimators':trial.suggest_int('n_estimators',50,400),\n", " 'max_depth':trial.suggest_int('max_depth',3,13),\n", " 'learning_rate':trial.suggest_float('learning_rate',0.01,0.3,log=True),\n", " 'subsample':trial.suggest_float('subsample',0.6,1.0),\n", " 'colsample_bytree':trial.suggest_float('colsample_bytree',0.4,1.0),\n", " 'gamma':trial.suggest_float('gamma',0.0,5.0),\n", " 'reg_alpha':trial.suggest_float('reg_alpha',0.0,5.0),\n", " 'reg_lambda':trial.suggest_float('reg_lambda',0.0,5.0),\n", " 'random_state':42,\n", " 'use_label_encoder':False,\n", " 'eval_metric':'mlogloss'\n", " }\n", "\n", " xgb=XGBClassifier(**params,n_jobs=-1)\n", " pipe=ImbPipeline([('smote',SMOTE(random_state=42)),('xgb',xgb)])\n", "\n", " cv=StratifiedKFold(n_splits=CV_FOLDS,shuffle=True,random_state=42)\n", "\n", " recalls=[]\n", "\n", " for train_idx,val_idx in cv.split(x_train,y_train_enc):\n", " x_t,x_v=x_train.iloc[train_idx],x_train.iloc[val_idx]\n", " y_t,y_v=y_train_enc[train_idx],y_train_enc[val_idx]\n", "\n", " pipe.fit(x_t,y_t)\n", " y_pred=pipe.predict(x_v)\n", "\n", " recs=recall_score(y_v,y_pred,labels=list(range(len(le.classes_))),average=None,zero_division=0)\n", " recall_high=recs[target_index]\n", " recalls.append(recall_high)\n", " return float(np.mean(recalls))\n", "\n", "sampler=TPESampler(seed=42)\n", "study=optuna.create_study(direction='maximize',sampler=sampler,study_name='xgb_high_recall')\n", "study.optimize(objective,n_trials=N_TRIALS,show_progress_bar=True)\n", "\n", "print(\"\\n Best CV Mean High-Recall : \",study.best_value)\n", "print(\"best Params\")\n", "for k,v in study.best_params.items():\n", " print(f\" {k}: {v}\")\n", "\n", "\n", "best_params=study.best_params.copy()\n", "best_params.update({'use_label_encoder':False,'eval_metric':'mlogloss','random_state':42})\n", "final_xgb=XGBClassifier(**best_params,n_jobs=-1)\n", "\n", "final_pipe=ImbPipeline([('smote',SMOTE(random_state=42)),('xgb',final_xgb)])\n", "final_pipe.fit(x_train,y_train_enc)\n", "\n", "\n", "y_test_pred_enc=final_pipe.predict(x_test)\n", "y_test_pred=le.inverse_transform(y_test_pred_enc)\n", "y_test_orig=y_test\n", "\n", "print(\"\\n Final Test Classification Report (labels in orignal names):\")\n", "print(classification_report(y_test_orig,y_test_pred,labels=le.classes_))\n", "\n", "print(\"Confusion Matrix (rows=true) , cols=predicted order=le.classes_\")\n", "print(confusion_matrix(y_test_orig,y_test_pred,labels=le.classes_))\n" ] }, { "cell_type": "code", "execution_count": 57, "id": "0ca93983", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Class Weights: {np.int64(0): np.float64(15.75), np.int64(1): np.float64(4.846153846153846), np.int64(2): np.float64(1.0)}\n", "class-Weighted -Classification Report \n", " precision recall f1-score support\n", "\n", " High 0.21 0.80 0.33 20\n", " Low 0.42 0.80 0.55 65\n", " Medium 0.91 0.58 0.71 315\n", "\n", " accuracy 0.62 400\n", " macro avg 0.51 0.73 0.53 400\n", "weighted avg 0.80 0.62 0.66 400\n", "\n", "confusion_matrix\n", "[[ 16 0 4]\n", " [ 0 52 13]\n", " [ 60 73 182]]\n" ] } ], "source": [ "y_train_enc=le.fit_transform(y_train)\n", "y_test_enc=le.transform(y_test)\n", "\n", "classes,counts=np.unique(y_test_enc,return_counts=True)\n", "class_weights={cls : max(counts)/count for cls,count in zip (classes , counts)}\n", "print(\"Class Weights: \",class_weights)\n", "\n", "xgb_weighted=XGBClassifier(\n", " objective='multi:softmax',\n", " num_class=len(classes),\n", " eval_metric='mlogloss',\n", " random_state=42,\n", " n_estimators=105,\n", " max_depth=3,\n", " learning_rate=0.014843806162322944,\n", " subsample=0.7730982444721965,\n", " colsample_bytree=0.8509237341976973,\n", " gamma=4.134336554829186,\n", " reg_alpha=4.453508368364819,\n", " reg_lambda=2.616923339470738,\n", " n_jobs=-1\n", ")\n", "\n", "sample_weights=np.array([class_weights[y] for y in y_train_enc])\n", "xgb_weighted.fit(x_train,y_train_enc,sample_weight=sample_weights)\n", "\n", "y_pred_enc=xgb_weighted.predict(x_test)\n", "print(\"class-Weighted -Classification Report \")\n", "print(classification_report(y_test_enc,y_pred_enc,target_names=le.classes_))\n", "print(\"confusion_matrix\")\n", "print(confusion_matrix(y_test_enc,y_pred_enc))" ] }, { "cell_type": "code", "execution_count": 60, "id": "f3dc54eb", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:30,914]\u001b[0m A new study created in memory with name: xgb_class_weighted_macroF1\u001b[0m\n", "Best trial: 0. Best value: 0.698032: 2%|▎ | 1/40 [00:00<00:37, 1.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:31,866]\u001b[0m Trial 0 finished with value: 0.6980319612469358 and parameters: {'n_estimators': 181, 'max_depth': 12, 'learning_rate': 0.06504856968981275, 'subsample': 0.8394633936788146, 'colsample_bytree': 0.4936111842654619, 'gamma': 0.7799726016810132, 'reg_alpha': 0.2904180608409973, 'reg_lambda': 4.330880728874676}. Best is trial 0 with value: 0.6980319612469358.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 0. Best value: 0.698032: 5%|▌ | 2/40 [00:03<01:20, 2.11s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:34,785]\u001b[0m Trial 1 finished with value: 0.6514549839777004 and parameters: {'n_estimators': 260, 'max_depth': 9, 'learning_rate': 0.001124579825911934, 'subsample': 0.9879639408647978, 'colsample_bytree': 0.899465584480253, 'gamma': 1.0616955533913808, 'reg_alpha': 0.9091248360355031, 'reg_lambda': 0.9170225492671691}. Best is trial 0 with value: 0.6980319612469358.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 0. Best value: 0.698032: 8%|▊ | 3/40 [00:05<01:02, 1.69s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:35,968]\u001b[0m Trial 2 finished with value: 0.6967044982252054 and parameters: {'n_estimators': 156, 'max_depth': 7, 'learning_rate': 0.01174843954800703, 'subsample': 0.7164916560792167, 'colsample_bytree': 0.7671117368334277, 'gamma': 0.6974693032602092, 'reg_alpha': 1.4607232426760908, 'reg_lambda': 1.8318092164684585}. Best is trial 0 with value: 0.6980319612469358.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 0. Best value: 0.698032: 10%|█ | 4/40 [00:07<01:05, 1.83s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:38,023]\u001b[0m Trial 3 finished with value: 0.6828770196102784 and parameters: {'n_estimators': 210, 'max_depth': 10, 'learning_rate': 0.003123317753376431, 'subsample': 0.8056937753654446, 'colsample_bytree': 0.7554487413172255, 'gamma': 0.23225206359998862, 'reg_alpha': 3.0377242595071916, 'reg_lambda': 0.8526206184364576}. Best is trial 0 with value: 0.6980319612469358.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 0. Best value: 0.698032: 12%|█▎ | 5/40 [00:07<00:45, 1.31s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:38,409]\u001b[0m Trial 4 finished with value: 0.6949217604887568 and parameters: {'n_estimators': 72, 'max_depth': 12, 'learning_rate': 0.24659691172104828, 'subsample': 0.9233589392465844, 'colsample_bytree': 0.5827682615040224, 'gamma': 0.48836057003191935, 'reg_alpha': 3.4211651325607844, 'reg_lambda': 2.2007624686980067}. Best is trial 0 with value: 0.6980319612469358.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 0. Best value: 0.698032: 15%|█▌ | 6/40 [00:08<00:36, 1.08s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:39,046]\u001b[0m Trial 5 finished with value: 0.6511004756285076 and parameters: {'n_estimators': 92, 'max_depth': 7, 'learning_rate': 0.0012167028814593455, 'subsample': 0.9637281608315128, 'colsample_bytree': 0.5552679889600102, 'gamma': 3.31261142176991, 'reg_alpha': 1.5585553804470549, 'reg_lambda': 2.600340105889054}. Best is trial 0 with value: 0.6980319612469358.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 0. Best value: 0.698032: 18%|█▊ | 7/40 [00:08<00:30, 1.07it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:39,667]\u001b[0m Trial 6 finished with value: 0.6602231021586435 and parameters: {'n_estimators': 241, 'max_depth': 4, 'learning_rate': 0.25221951700214285, 'subsample': 0.9100531293444458, 'colsample_bytree': 0.9636993649385135, 'gamma': 4.474136752138244, 'reg_alpha': 2.9894998940554256, 'reg_lambda': 4.609371175115584}. Best is trial 0 with value: 0.6980319612469358.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 0. Best value: 0.698032: 20%|██ | 8/40 [00:09<00:24, 1.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:40,059]\u001b[0m Trial 7 finished with value: 0.5839741278036462 and parameters: {'n_estimators': 81, 'max_depth': 4, 'learning_rate': 0.001294295611551122, 'subsample': 0.7301321323053057, 'colsample_bytree': 0.6332063738136893, 'gamma': 1.3567451588694794, 'reg_alpha': 4.143687545759647, 'reg_lambda': 1.7837666334679465}. Best is trial 0 with value: 0.6980319612469358.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 0. Best value: 0.698032: 22%|██▎ | 9/40 [00:09<00:23, 1.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:40,785]\u001b[0m Trial 8 finished with value: 0.6129143229367422 and parameters: {'n_estimators': 148, 'max_depth': 7, 'learning_rate': 0.0022340165853190056, 'subsample': 0.9208787923016158, 'colsample_bytree': 0.44473038620786254, 'gamma': 4.9344346830025865, 'reg_alpha': 3.861223846483287, 'reg_lambda': 0.993578407670862}. Best is trial 0 with value: 0.6980319612469358.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 0. Best value: 0.698032: 25%|██▌ | 10/40 [00:10<00:20, 1.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:41,353]\u001b[0m Trial 9 finished with value: 0.6763057841365363 and parameters: {'n_estimators': 51, 'max_depth': 10, 'learning_rate': 0.0563600475052774, 'subsample': 0.8916028672163949, 'colsample_bytree': 0.8627622080115674, 'gamma': 0.3702232586704518, 'reg_alpha': 1.7923286427213632, 'reg_lambda': 0.5793452976256486}. Best is trial 0 with value: 0.6980319612469358.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 0. Best value: 0.698032: 28%|██▊ | 11/40 [00:11<00:24, 1.20it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:42,509]\u001b[0m Trial 10 finished with value: 0.6369172588635298 and parameters: {'n_estimators': 383, 'max_depth': 2, 'learning_rate': 0.01919733178920577, 'subsample': 0.6071847502459279, 'colsample_bytree': 0.4129900873785599, 'gamma': 2.2782144378149125, 'reg_alpha': 0.0779739982922818, 'reg_lambda': 4.828764294174368}. Best is trial 0 with value: 0.6980319612469358.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 11. Best value: 0.698676: 30%|███ | 12/40 [00:12<00:27, 1.02it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:43,810]\u001b[0m Trial 11 finished with value: 0.6986762534393138 and parameters: {'n_estimators': 167, 'max_depth': 12, 'learning_rate': 0.011492603695411418, 'subsample': 0.7414922757859675, 'colsample_bytree': 0.7318392529417277, 'gamma': 1.8910445420052762, 'reg_alpha': 0.23304486130435842, 'reg_lambda': 3.454869464124512}. Best is trial 11 with value: 0.6986762534393138.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 12. Best value: 0.703468: 32%|███▎ | 13/40 [00:14<00:28, 1.05s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:45,018]\u001b[0m Trial 12 finished with value: 0.7034679420755018 and parameters: {'n_estimators': 311, 'max_depth': 12, 'learning_rate': 0.05866160424897018, 'subsample': 0.8072636662657275, 'colsample_bytree': 0.703055408788255, 'gamma': 1.8222115161922807, 'reg_alpha': 0.10512877265791049, 'reg_lambda': 3.547283387544222}. Best is trial 12 with value: 0.7034679420755018.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 12. Best value: 0.703468: 35%|███▌ | 14/40 [00:16<00:34, 1.32s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:46,964]\u001b[0m Trial 13 finished with value: 0.6584668400540913 and parameters: {'n_estimators': 332, 'max_depth': 12, 'learning_rate': 0.008438029472007186, 'subsample': 0.7444369379790783, 'colsample_bytree': 0.7085337539039133, 'gamma': 2.0042417683381455, 'reg_alpha': 4.996812193207756, 'reg_lambda': 3.4661248350485216}. Best is trial 12 with value: 0.7034679420755018.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 14. Best value: 0.708629: 38%|███▊ | 15/40 [00:17<00:31, 1.26s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:48,098]\u001b[0m Trial 14 finished with value: 0.7086294817312174 and parameters: {'n_estimators': 311, 'max_depth': 10, 'learning_rate': 0.04188007640384457, 'subsample': 0.6536309038943456, 'colsample_bytree': 0.6862340416485463, 'gamma': 2.6750443231727212, 'reg_alpha': 0.6940641877629383, 'reg_lambda': 3.5074852325945285}. Best is trial 14 with value: 0.7086294817312174.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 14. Best value: 0.708629: 40%|████ | 16/40 [00:18<00:28, 1.19s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:49,125]\u001b[0m Trial 15 finished with value: 0.7012715493987175 and parameters: {'n_estimators': 308, 'max_depth': 10, 'learning_rate': 0.059672036341475033, 'subsample': 0.6496456186041755, 'colsample_bytree': 0.671395787726565, 'gamma': 3.117223333224823, 'reg_alpha': 0.9660956277259602, 'reg_lambda': 3.635424858575993}. Best is trial 14 with value: 0.7086294817312174.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 14. Best value: 0.708629: 42%|████▎ | 17/40 [00:19<00:27, 1.21s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:50,376]\u001b[0m Trial 16 finished with value: 0.6942749359564063 and parameters: {'n_estimators': 314, 'max_depth': 9, 'learning_rate': 0.032786070711853216, 'subsample': 0.6698186937738351, 'colsample_bytree': 0.8225134422491962, 'gamma': 2.912554411220168, 'reg_alpha': 2.1847380129789054, 'reg_lambda': 2.8512497916385695}. Best is trial 14 with value: 0.7086294817312174.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 14. Best value: 0.708629: 45%|████▌ | 18/40 [00:20<00:25, 1.17s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:51,448]\u001b[0m Trial 17 finished with value: 0.7022350379089782 and parameters: {'n_estimators': 393, 'max_depth': 11, 'learning_rate': 0.1208126868376585, 'subsample': 0.8064114554738081, 'colsample_bytree': 0.6237003702668842, 'gamma': 3.901945686956894, 'reg_alpha': 0.7511370295512978, 'reg_lambda': 3.906540631154805}. Best is trial 14 with value: 0.7086294817312174.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 18. Best value: 0.711102: 48%|████▊ | 19/40 [00:21<00:24, 1.15s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:52,556]\u001b[0m Trial 18 finished with value: 0.7111021280184385 and parameters: {'n_estimators': 355, 'max_depth': 8, 'learning_rate': 0.12862098053948898, 'subsample': 0.60460836501618, 'colsample_bytree': 0.7906052995533104, 'gamma': 2.533881284358335, 'reg_alpha': 2.2240260252820163, 'reg_lambda': 2.9621613503115176}. Best is trial 18 with value: 0.7111021280184385.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 18. Best value: 0.711102: 50%|█████ | 20/40 [00:22<00:22, 1.13s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:53,631]\u001b[0m Trial 19 finished with value: 0.7099263297758601 and parameters: {'n_estimators': 356, 'max_depth': 8, 'learning_rate': 0.11930470227392652, 'subsample': 0.6001330615376621, 'colsample_bytree': 0.8084670052606318, 'gamma': 2.507285077111833, 'reg_alpha': 2.372668649847154, 'reg_lambda': 2.9298986778838154}. Best is trial 18 with value: 0.7111021280184385.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 18. Best value: 0.711102: 52%|█████▎ | 21/40 [00:23<00:21, 1.11s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:54,713]\u001b[0m Trial 20 finished with value: 0.7054092460152243 and parameters: {'n_estimators': 358, 'max_depth': 6, 'learning_rate': 0.1349959075171375, 'subsample': 0.6045066034757586, 'colsample_bytree': 0.9947663176861599, 'gamma': 3.46654577274796, 'reg_alpha': 2.459197753390793, 'reg_lambda': 2.9111046405876935}. Best is trial 18 with value: 0.7111021280184385.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 21. Best value: 0.712258: 55%|█████▌ | 22/40 [00:24<00:18, 1.03s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:55,555]\u001b[0m Trial 21 finished with value: 0.7122577004390723 and parameters: {'n_estimators': 275, 'max_depth': 8, 'learning_rate': 0.1174807309023837, 'subsample': 0.6527114042832046, 'colsample_bytree': 0.8008868570924177, 'gamma': 2.5663288761930296, 'reg_alpha': 2.5867628260703337, 'reg_lambda': 3.1226657899513626}. Best is trial 21 with value: 0.7122577004390723.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 22. Best value: 0.712991: 57%|█████▊ | 23/40 [00:25<00:16, 1.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:56,385]\u001b[0m Trial 22 finished with value: 0.7129905942958111 and parameters: {'n_estimators': 267, 'max_depth': 8, 'learning_rate': 0.11633884836838314, 'subsample': 0.6780094632538717, 'colsample_bytree': 0.8107668947462211, 'gamma': 2.351396865390427, 'reg_alpha': 2.4474395438686747, 'reg_lambda': 2.4024783021857155}. Best is trial 22 with value: 0.7129905942958111.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 22. Best value: 0.712991: 60%|██████ | 24/40 [00:26<00:14, 1.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:57,209]\u001b[0m Trial 23 finished with value: 0.7121701736636493 and parameters: {'n_estimators': 275, 'max_depth': 5, 'learning_rate': 0.16398473826009327, 'subsample': 0.6820502472520884, 'colsample_bytree': 0.9174885395083423, 'gamma': 1.5471045326450694, 'reg_alpha': 2.6709809244432785, 'reg_lambda': 2.3676286800169097}. Best is trial 22 with value: 0.7129905942958111.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 24. Best value: 0.71754: 62%|██████▎ | 25/40 [00:27<00:13, 1.12it/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:58,017]\u001b[0m Trial 24 finished with value: 0.7175403820880574 and parameters: {'n_estimators': 270, 'max_depth': 5, 'learning_rate': 0.1987912719438194, 'subsample': 0.7020087917511156, 'colsample_bytree': 0.9173148640389713, 'gamma': 1.4077658162168274, 'reg_alpha': 2.8235595204082857, 'reg_lambda': 2.2190469086305615}. Best is trial 24 with value: 0.7175403820880574.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 24. Best value: 0.71754: 65%|██████▌ | 26/40 [00:27<00:11, 1.22it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:58,675]\u001b[0m Trial 25 finished with value: 0.709779594342259 and parameters: {'n_estimators': 231, 'max_depth': 3, 'learning_rate': 0.2974592138837979, 'subsample': 0.697635939241076, 'colsample_bytree': 0.8543888596860163, 'gamma': 1.3401344918897475, 'reg_alpha': 3.197294966146854, 'reg_lambda': 1.5345048123626595}. Best is trial 24 with value: 0.7175403820880574.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 24. Best value: 0.71754: 68%|██████▊ | 27/40 [00:28<00:10, 1.22it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:32:59,496]\u001b[0m Trial 26 finished with value: 0.6866963819913902 and parameters: {'n_estimators': 273, 'max_depth': 6, 'learning_rate': 0.08745221651454381, 'subsample': 0.7649011048051568, 'colsample_bytree': 0.9300232834666233, 'gamma': 3.770634003396035, 'reg_alpha': 3.534164643549822, 'reg_lambda': 2.0936734682102482}. Best is trial 24 with value: 0.7175403820880574.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 24. Best value: 0.71754: 70%|███████ | 28/40 [00:29<00:10, 1.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:00,508]\u001b[0m Trial 27 finished with value: 0.6965634767647796 and parameters: {'n_estimators': 203, 'max_depth': 5, 'learning_rate': 0.027640690649905424, 'subsample': 0.6490305376309605, 'colsample_bytree': 0.883514099031861, 'gamma': 2.1923095167250284, 'reg_alpha': 1.84366618236838, 'reg_lambda': 0.13732298398813603}. Best is trial 24 with value: 0.7175403820880574.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 24. Best value: 0.71754: 72%|███████▎ | 29/40 [00:30<00:09, 1.19it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:01,251]\u001b[0m Trial 28 finished with value: 0.7144807342331146 and parameters: {'n_estimators': 247, 'max_depth': 8, 'learning_rate': 0.2120761095841181, 'subsample': 0.7741686384095295, 'colsample_bytree': 0.8357517390406574, 'gamma': 1.6483921938031032, 'reg_alpha': 2.733736545530231, 'reg_lambda': 1.4355600332986134}. Best is trial 24 with value: 0.7175403820880574.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 24. Best value: 0.71754: 75%|███████▌ | 30/40 [00:31<00:08, 1.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:02,160]\u001b[0m Trial 29 finished with value: 0.7101340119304016 and parameters: {'n_estimators': 128, 'max_depth': 6, 'learning_rate': 0.08060902510029831, 'subsample': 0.7695997589714981, 'colsample_bytree': 0.9537236408116762, 'gamma': 0.011784792689336632, 'reg_alpha': 4.275557977959188, 'reg_lambda': 1.4767441028661485}. Best is trial 24 with value: 0.7175403820880574.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 24. Best value: 0.71754: 78%|███████▊ | 31/40 [00:31<00:07, 1.25it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:02,816]\u001b[0m Trial 30 finished with value: 0.702413451487052 and parameters: {'n_estimators': 200, 'max_depth': 9, 'learning_rate': 0.19895831661905272, 'subsample': 0.8489817519564029, 'colsample_bytree': 0.8440968568462979, 'gamma': 0.9849865538712952, 'reg_alpha': 2.679852057349216, 'reg_lambda': 1.3235859990233962}. Best is trial 24 with value: 0.7175403820880574.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 31. Best value: 0.717799: 80%|████████ | 32/40 [00:32<00:06, 1.22it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:03,678]\u001b[0m Trial 31 finished with value: 0.7177991222453585 and parameters: {'n_estimators': 284, 'max_depth': 8, 'learning_rate': 0.17728690044659165, 'subsample': 0.7012072530384043, 'colsample_bytree': 0.7869681016552836, 'gamma': 1.6456030999994784, 'reg_alpha': 2.743974785549257, 'reg_lambda': 2.5397343711774245}. Best is trial 31 with value: 0.7177991222453585.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 31. Best value: 0.717799: 82%|████████▎ | 33/40 [00:33<00:05, 1.23it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:04,490]\u001b[0m Trial 32 finished with value: 0.7080985937019547 and parameters: {'n_estimators': 258, 'max_depth': 8, 'learning_rate': 0.17972145737835551, 'subsample': 0.7046240146887897, 'colsample_bytree': 0.8983417058692577, 'gamma': 1.5988888588247945, 'reg_alpha': 2.816744808278578, 'reg_lambda': 2.5845226893326303}. Best is trial 31 with value: 0.7177991222453585.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 31. Best value: 0.717799: 85%|████████▌ | 34/40 [00:34<00:05, 1.17it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:05,434]\u001b[0m Trial 33 finished with value: 0.7168651845115117 and parameters: {'n_estimators': 237, 'max_depth': 9, 'learning_rate': 0.08433657840538367, 'subsample': 0.6859312020832035, 'colsample_bytree': 0.7547814708680873, 'gamma': 0.8666984074118537, 'reg_alpha': 2.023913136197992, 'reg_lambda': 2.0307637224346937}. Best is trial 31 with value: 0.7177991222453585.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 31. Best value: 0.717799: 88%|████████▊ | 35/40 [00:35<00:04, 1.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:06,362]\u001b[0m Trial 34 finished with value: 0.7093238106389458 and parameters: {'n_estimators': 227, 'max_depth': 9, 'learning_rate': 0.0852314492333849, 'subsample': 0.7680083062749024, 'colsample_bytree': 0.7547448707632682, 'gamma': 0.9435872338804697, 'reg_alpha': 2.001593910652774, 'reg_lambda': 1.8878323910181254}. Best is trial 31 with value: 0.7177991222453585.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 31. Best value: 0.717799: 90%|█████████ | 36/40 [00:36<00:03, 1.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:07,222]\u001b[0m Trial 35 finished with value: 0.7159605039836507 and parameters: {'n_estimators': 292, 'max_depth': 11, 'learning_rate': 0.2026207600339525, 'subsample': 0.712515652473763, 'colsample_bytree': 0.7630385230264625, 'gamma': 1.2079127080174275, 'reg_alpha': 1.3746630082378646, 'reg_lambda': 1.2539644622318245}. Best is trial 31 with value: 0.7177991222453585.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 31. Best value: 0.717799: 92%|█████████▎| 37/40 [00:37<00:02, 1.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:08,143]\u001b[0m Trial 36 finished with value: 0.7144616238199001 and parameters: {'n_estimators': 283, 'max_depth': 11, 'learning_rate': 0.18639362869986956, 'subsample': 0.7160427533769607, 'colsample_bytree': 0.7672244316052241, 'gamma': 0.7897281818126173, 'reg_alpha': 1.2175727723938592, 'reg_lambda': 1.0818601677435098}. Best is trial 31 with value: 0.7177991222453585.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 37. Best value: 0.723265: 95%|█████████▌| 38/40 [00:38<00:01, 1.02it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:09,341]\u001b[0m Trial 37 finished with value: 0.72326451208224 and parameters: {'n_estimators': 291, 'max_depth': 11, 'learning_rate': 0.08667123678568317, 'subsample': 0.6918719574908302, 'colsample_bytree': 0.6542877699770573, 'gamma': 0.6013772946946787, 'reg_alpha': 1.46970667377079, 'reg_lambda': 1.8770068118685597}. Best is trial 37 with value: 0.72326451208224.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 37. Best value: 0.723265: 98%|█████████▊| 39/40 [00:39<00:01, 1.03s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:10,504]\u001b[0m Trial 38 finished with value: 0.7113531747165202 and parameters: {'n_estimators': 223, 'max_depth': 9, 'learning_rate': 0.043104738464802804, 'subsample': 0.686735172473941, 'colsample_bytree': 0.5620956828068722, 'gamma': 0.7127674800107269, 'reg_alpha': 1.6782452164249357, 'reg_lambda': 2.020706824532498}. Best is trial 37 with value: 0.72326451208224.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Best trial: 37. Best value: 0.723265: 100%|██████████| 40/40 [00:40<00:00, 1.02s/it]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[I 2026-07-11 18:33:11,888]\u001b[0m Trial 39 finished with value: 0.7190124483307517 and parameters: {'n_estimators': 334, 'max_depth': 11, 'learning_rate': 0.07355763202954871, 'subsample': 0.6258472760415172, 'colsample_bytree': 0.6614303737986453, 'gamma': 0.4632518529281504, 'reg_alpha': 3.111691627386141, 'reg_lambda': 1.7335263513565262}. Best is trial 37 with value: 0.72326451208224.\u001b[0m\n", "Best CV Macro-F1: 0.72326451208224\n", "Best Params:\n", " n_estimators : 291\n", " max_depth : 11\n", " learning_rate : 0.08667123678568317\n", " subsample : 0.6918719574908302\n", " colsample_bytree : 0.6542877699770573\n", " gamma : 0.6013772946946787\n", " reg_alpha : 1.46970667377079\n", " reg_lambda : 1.8770068118685597\n", "\n", " Final Classification Report (class-weighted XGB):\n", " precision recall f1-score support\n", "\n", " High 0.59 0.50 0.54 20\n", " Low 0.75 0.74 0.74 65\n", " Medium 0.92 0.93 0.92 315\n", "\n", " accuracy 0.88 400\n", " macro avg 0.75 0.72 0.74 400\n", "weighted avg 0.87 0.88 0.87 400\n", "\n", "Confusion Matrix (rows=true , cols=pred)\n", "[[ 10 0 10]\n", " [ 0 48 17]\n", " [ 7 16 292]]\n" ] } ], "source": [ "le=LabelEncoder()\n", "le.fit(y_train)\n", "y_train_enc=le.transform(y_train)\n", "y_test_enc=le.transform(y_test)\n", "n_classes=len(le.classes_)\n", "\n", "def objective(trial):\n", " params={\n", " 'n_estimators':trial.suggest_int('n_estimators',50,400),\n", " 'max_depth':trial.suggest_int('max_depth',2,12),\n", " 'learning_rate':trial.suggest_float('learning_rate',1e-3,0.3,log=True),\n", " 'subsample':trial.suggest_float('subsample',0.6,1.0),\n", " 'colsample_bytree':trial.suggest_float('colsample_bytree',0.4,1.0),\n", " 'gamma':trial.suggest_float('gamma',0.0,5.0),\n", " 'reg_alpha':trial.suggest_float('reg_alpha',0.0,5.0),\n", " 'reg_lambda':trial.suggest_float('reg_lambda',0.0,5.0),\n", "\n", " 'use_label_encoder':False,\n", " 'eval_metric':'mlogloss',\n", " }\n", "\n", " cv=StratifiedKFold(n_splits=3,shuffle=True,random_state=42)\n", " scores=[]\n", "\n", " for train_idx,val_idx in cv.split(x_train,y_train_enc):\n", " x_t,x_v=x_train.iloc[train_idx],x_train.iloc[val_idx]\n", " y_t,y_v=y_train_enc[train_idx],y_train_enc[val_idx]\n", "\n", " classes , counts =np.unique(y_t,return_counts=True)\n", "\n", " class_weights_map={cls:float(max(counts)/cnt) for cls , cnt in zip(classes,counts)}\n", " sample_weights=np.array([class_weights_map[y] for y in y_t])\n", "\n", " model=XGBClassifier(**params,n_jobs=1,random_state=42)\n", "\n", " model.fit(x_t,y_t,sample_weight=sample_weights,verbose=True)\n", "\n", " y_pred=model.predict(x_v)\n", "\n", " scores.append(f1_score(y_v,y_pred,average='macro'))\n", " return float(np.mean(scores))\n", "\n", "\n", "sampler=TPESampler(seed=42)\n", "study=optuna.create_study(direction='maximize',sampler=sampler,study_name='xgb_class_weighted_macroF1')\n", "\n", "study.optimize(objective,n_trials=40,show_progress_bar=True)\n", "\n", "print(\"Best CV Macro-F1:\",study.best_value)\n", "print(\"Best Params:\")\n", "for k,v in study.best_params.items():\n", " print(f\" {k} : {v}\")\n", "\n", "\n", "best=study.best_params.copy()\n", "best.update({\"use_label_encoder\":False,'eval_metric':'mlogloss','random_state':42})\n", "\n", "classes,counts=np.unique(y_train_enc,return_counts=True)\n", "final_class_weight_map={ cls :float(max(counts)/cnt) for cls, cnt in zip(classes,counts)}\n", "final_sample_weights=np.array([final_class_weight_map[y] for y in y_train_enc])\n", "\n", "final_xgb=XGBClassifier(**best,n_jobs=-1)\n", "final_xgb.fit(x_train,y_train_enc,sample_weight=final_sample_weights,verbose=True)\n", "\n", "\n", "y_test_pred_enc=final_xgb.predict(x_test)\n", "y_test_pred=le.inverse_transform(y_test_pred_enc)\n", "y_test_orig=y_test\n", "\n", "print(\"\\n Final Classification Report (class-weighted XGB):\")\n", "print(classification_report(y_test_orig,y_test_pred,labels=le.classes_))\n", "print(\"Confusion Matrix (rows=true , cols=pred)\")\n", "print(confusion_matrix(y_test_orig,y_test_pred,labels=le.classes_))" ] }, { "cell_type": "markdown", "id": "68aeeed9", "metadata": {}, "source": [ "## Save the model as a .pkl file" ] }, { "cell_type": "code", "execution_count": 68, "id": "5087d550", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Model Saved Succesfully as 'model_xgb_new.pkl\n" ] } ], "source": [ "joblib.dump(final_xgb,'../Models/model_xgb_new.pkl')\n", "print(\"Model Saved Succesfully as 'model_xgb_new.pkl\")" ] }, { "cell_type": "code", "execution_count": 69, "id": "3152663a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Label encoder pickle file saved as Label_encoder.pkl\n" ] } ], "source": [ "joblib.dump(le,'../Models/Label_encoder.pkl')\n", "print(\"Label encoder pickle file saved as Label_encoder.pkl\")" ] }, { "cell_type": "code", "execution_count": 70, "id": "a5438c22", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['Age', 'Gender', 'Smoking', 'Alcohol_Use', 'Obesity', 'Family_History', 'Diet_Red_Meat', 'Diet_Salted_Processed', 'Fruit_Veg_Intake', 'Physical_Activity', 'Air_Pollution', 'Occupational_Hazards', 'BRCA_Mutation', 'H_Pylori_Infection', 'Calcium_Intake', 'BMI', 'Physical_Activity_Level']\n" ] } ], "source": [ "feature_names=list(x_train.columns)\n", "print(feature_names)" ] }, { "cell_type": "code", "execution_count": 71, "id": "1d7f3d66", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "features names saved as Feature_names.pkl\n" ] } ], "source": [ "joblib.dump(feature_names,'../Models/Feature_names.pkl')\n", "print(\"features names saved as Feature_names.pkl\")" ] } ], "metadata": { "kernelspec": { "display_name": "ai-ml", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.15" } }, "nbformat": 4, "nbformat_minor": 5 }