{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "54c4cb63",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:14.529709Z",
     "iopub.status.busy": "2023-08-07T01:10:14.529349Z",
     "iopub.status.idle": "2023-08-07T01:10:17.073736Z",
     "shell.execute_reply": "2023-08-07T01:10:17.072654Z"
    },
    "papermill": {
     "duration": 2.562251,
     "end_time": "2023-08-07T01:10:17.076306",
     "exception": false,
     "start_time": "2023-08-07T01:10:14.514055",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os, sys, time, copy, datetime\n",
    "from pathlib import Path\n",
    "from tqdm.auto import tqdm\n",
    "tqdm.pandas()\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn import feature_selection\n",
    "from sklearn import model_selection\n",
    "from sklearn import linear_model\n",
    "from sklearn import ensemble\n",
    "from sklearn import metrics\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn import decomposition\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "from sklearn import feature_selection\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn import impute\n",
    "# from sklearn.impute import IterativeImputer, SimpleImputer, KNNImputer\n",
    "\n",
    "from imblearn.over_sampling import SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eba79b03",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.100909Z",
     "iopub.status.busy": "2023-08-07T01:10:17.100495Z",
     "iopub.status.idle": "2023-08-07T01:10:17.108685Z",
     "shell.execute_reply": "2023-08-07T01:10:17.107598Z"
    },
    "papermill": {
     "duration": 0.023091,
     "end_time": "2023-08-07T01:10:17.111042",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.087951",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'seed': 2023}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = dict(\n",
    "    seed = 2023\n",
    ")\n",
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3f2ef74d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.136586Z",
     "iopub.status.busy": "2023-08-07T01:10:17.135799Z",
     "iopub.status.idle": "2023-08-07T01:10:17.143260Z",
     "shell.execute_reply": "2023-08-07T01:10:17.142268Z"
    },
    "papermill": {
     "duration": 0.022965,
     "end_time": "2023-08-07T01:10:17.145523",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.122558",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sample_submission.csv', 'greeks.csv', 'train.csv', 'test.csv']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATA_DIR = Path(\"../input/icr-identify-age-related-conditions/\")\n",
    "os.listdir(DATA_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e13d8906",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.170828Z",
     "iopub.status.busy": "2023-08-07T01:10:17.170410Z",
     "iopub.status.idle": "2023-08-07T01:10:17.231981Z",
     "shell.execute_reply": "2023-08-07T01:10:17.230845Z"
    },
    "papermill": {
     "duration": 0.077258,
     "end_time": "2023-08-07T01:10:17.234558",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.157300",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train shape: (617, 58)\n",
      "Test shape: (5, 57)\n",
      "Greek shape: (617, 6)\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv(DATA_DIR/'train.csv')\n",
    "greek_df = pd.read_csv(DATA_DIR/'greeks.csv')\n",
    "test_df = pd.read_csv(DATA_DIR/'test.csv')\n",
    "sample_df = pd.read_csv(DATA_DIR/'sample_submission.csv')\n",
    "\n",
    "train_df.columns = train_df.columns.str.replace(' ', '')\n",
    "test_df.columns = test_df.columns.str.replace(' ', '')\n",
    "greek_df.columns = greek_df.columns.str.replace(' ', '')\n",
    "\n",
    "print(f\"Train shape: {train_df.shape}\")\n",
    "print(f\"Test shape: {test_df.shape}\")\n",
    "print(f\"Greek shape: {greek_df.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4c065a2b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.260585Z",
     "iopub.status.busy": "2023-08-07T01:10:17.259520Z",
     "iopub.status.idle": "2023-08-07T01:10:17.306156Z",
     "shell.execute_reply": "2023-08-07T01:10:17.305052Z"
    },
    "papermill": {
     "duration": 0.062386,
     "end_time": "2023-08-07T01:10:17.308653",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.246267",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>AB</th>\n",
       "      <th>AF</th>\n",
       "      <th>AH</th>\n",
       "      <th>AM</th>\n",
       "      <th>AR</th>\n",
       "      <th>AX</th>\n",
       "      <th>AY</th>\n",
       "      <th>AZ</th>\n",
       "      <th>BC</th>\n",
       "      <th>...</th>\n",
       "      <th>FL</th>\n",
       "      <th>FR</th>\n",
       "      <th>FS</th>\n",
       "      <th>GB</th>\n",
       "      <th>GE</th>\n",
       "      <th>GF</th>\n",
       "      <th>GH</th>\n",
       "      <th>GI</th>\n",
       "      <th>GL</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000ff2bfdfe9</td>\n",
       "      <td>0.209377</td>\n",
       "      <td>3109.03329</td>\n",
       "      <td>85.200147</td>\n",
       "      <td>22.394407</td>\n",
       "      <td>8.138688</td>\n",
       "      <td>0.699861</td>\n",
       "      <td>0.025578</td>\n",
       "      <td>9.812214</td>\n",
       "      <td>5.555634</td>\n",
       "      <td>...</td>\n",
       "      <td>7.298162</td>\n",
       "      <td>1.73855</td>\n",
       "      <td>0.094822</td>\n",
       "      <td>11.339138</td>\n",
       "      <td>72.611063</td>\n",
       "      <td>2003.810319</td>\n",
       "      <td>22.136229</td>\n",
       "      <td>69.834944</td>\n",
       "      <td>0.120343</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>007255e47698</td>\n",
       "      <td>0.145282</td>\n",
       "      <td>978.76416</td>\n",
       "      <td>85.200147</td>\n",
       "      <td>36.968889</td>\n",
       "      <td>8.138688</td>\n",
       "      <td>3.632190</td>\n",
       "      <td>0.025578</td>\n",
       "      <td>13.517790</td>\n",
       "      <td>1.229900</td>\n",
       "      <td>...</td>\n",
       "      <td>0.173229</td>\n",
       "      <td>0.49706</td>\n",
       "      <td>0.568932</td>\n",
       "      <td>9.292698</td>\n",
       "      <td>72.611063</td>\n",
       "      <td>27981.562750</td>\n",
       "      <td>29.135430</td>\n",
       "      <td>32.131996</td>\n",
       "      <td>21.978000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>013f2bd269f5</td>\n",
       "      <td>0.470030</td>\n",
       "      <td>2635.10654</td>\n",
       "      <td>85.200147</td>\n",
       "      <td>32.360553</td>\n",
       "      <td>8.138688</td>\n",
       "      <td>6.732840</td>\n",
       "      <td>0.025578</td>\n",
       "      <td>12.824570</td>\n",
       "      <td>1.229900</td>\n",
       "      <td>...</td>\n",
       "      <td>7.709560</td>\n",
       "      <td>0.97556</td>\n",
       "      <td>1.198821</td>\n",
       "      <td>37.077772</td>\n",
       "      <td>88.609437</td>\n",
       "      <td>13676.957810</td>\n",
       "      <td>28.022851</td>\n",
       "      <td>35.192676</td>\n",
       "      <td>0.196941</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>043ac50845d5</td>\n",
       "      <td>0.252107</td>\n",
       "      <td>3819.65177</td>\n",
       "      <td>120.201618</td>\n",
       "      <td>77.112203</td>\n",
       "      <td>8.138688</td>\n",
       "      <td>3.685344</td>\n",
       "      <td>0.025578</td>\n",
       "      <td>11.053708</td>\n",
       "      <td>1.229900</td>\n",
       "      <td>...</td>\n",
       "      <td>6.122162</td>\n",
       "      <td>0.49706</td>\n",
       "      <td>0.284466</td>\n",
       "      <td>18.529584</td>\n",
       "      <td>82.416803</td>\n",
       "      <td>2094.262452</td>\n",
       "      <td>39.948656</td>\n",
       "      <td>90.493248</td>\n",
       "      <td>0.155829</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>044fb8a146ec</td>\n",
       "      <td>0.380297</td>\n",
       "      <td>3733.04844</td>\n",
       "      <td>85.200147</td>\n",
       "      <td>14.103738</td>\n",
       "      <td>8.138688</td>\n",
       "      <td>3.942255</td>\n",
       "      <td>0.054810</td>\n",
       "      <td>3.396778</td>\n",
       "      <td>102.151980</td>\n",
       "      <td>...</td>\n",
       "      <td>8.153058</td>\n",
       "      <td>48.50134</td>\n",
       "      <td>0.121914</td>\n",
       "      <td>16.408728</td>\n",
       "      <td>146.109943</td>\n",
       "      <td>8524.370502</td>\n",
       "      <td>45.381316</td>\n",
       "      <td>36.262628</td>\n",
       "      <td>0.096614</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Id        AB          AF          AH         AM        AR  \\\n",
       "0  000ff2bfdfe9  0.209377  3109.03329   85.200147  22.394407  8.138688   \n",
       "1  007255e47698  0.145282   978.76416   85.200147  36.968889  8.138688   \n",
       "2  013f2bd269f5  0.470030  2635.10654   85.200147  32.360553  8.138688   \n",
       "3  043ac50845d5  0.252107  3819.65177  120.201618  77.112203  8.138688   \n",
       "4  044fb8a146ec  0.380297  3733.04844   85.200147  14.103738  8.138688   \n",
       "\n",
       "         AX        AY         AZ          BC  ...        FL        FR  \\\n",
       "0  0.699861  0.025578   9.812214    5.555634  ...  7.298162   1.73855   \n",
       "1  3.632190  0.025578  13.517790    1.229900  ...  0.173229   0.49706   \n",
       "2  6.732840  0.025578  12.824570    1.229900  ...  7.709560   0.97556   \n",
       "3  3.685344  0.025578  11.053708    1.229900  ...  6.122162   0.49706   \n",
       "4  3.942255  0.054810   3.396778  102.151980  ...  8.153058  48.50134   \n",
       "\n",
       "         FS         GB          GE            GF         GH         GI  \\\n",
       "0  0.094822  11.339138   72.611063   2003.810319  22.136229  69.834944   \n",
       "1  0.568932   9.292698   72.611063  27981.562750  29.135430  32.131996   \n",
       "2  1.198821  37.077772   88.609437  13676.957810  28.022851  35.192676   \n",
       "3  0.284466  18.529584   82.416803   2094.262452  39.948656  90.493248   \n",
       "4  0.121914  16.408728  146.109943   8524.370502  45.381316  36.262628   \n",
       "\n",
       "          GL  Class  \n",
       "0   0.120343      1  \n",
       "1  21.978000      0  \n",
       "2   0.196941      0  \n",
       "3   0.155829      0  \n",
       "4   0.096614      1  \n",
       "\n",
       "[5 rows x 58 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d8a07ffd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.334793Z",
     "iopub.status.busy": "2023-08-07T01:10:17.333965Z",
     "iopub.status.idle": "2023-08-07T01:10:17.348855Z",
     "shell.execute_reply": "2023-08-07T01:10:17.347224Z"
    },
    "papermill": {
     "duration": 0.030249,
     "end_time": "2023-08-07T01:10:17.351229",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.320980",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing values in train set:  131\n",
      "Missing values in test set: 0\n",
      "Missing values in greek set: 0\n"
     ]
    }
   ],
   "source": [
    "# check for missiargsortng values in all the datasets\n",
    "print(\"Missing values in train set: \", train_df.isna().sum().sum())\n",
    "print(f\"Missing values in test set: {test_df.isna().sum().sum()}\")\n",
    "print(f\"Missing values in greek set: {greek_df.isna().sum().sum()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1694f9f9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.377440Z",
     "iopub.status.busy": "2023-08-07T01:10:17.376592Z",
     "iopub.status.idle": "2023-08-07T01:10:17.383823Z",
     "shell.execute_reply": "2023-08-07T01:10:17.382686Z"
    },
    "papermill": {
     "duration": 0.022748,
     "end_time": "2023-08-07T01:10:17.386164",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.363416",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# FEATURES = test_df.columns.tolist()[1:]\n",
    "cat_cols = train_df.select_dtypes(include=['object', 'category']).columns.tolist()[1:]\n",
    "num_cols = train_df.select_dtypes(include=np.number).columns.tolist()[:-1]\n",
    "target_df = pd.DataFrame(train_df['Class'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "52e87bd7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.412178Z",
     "iopub.status.busy": "2023-08-07T01:10:17.411767Z",
     "iopub.status.idle": "2023-08-07T01:10:17.416313Z",
     "shell.execute_reply": "2023-08-07T01:10:17.415273Z"
    },
    "papermill": {
     "duration": 0.020244,
     "end_time": "2023-08-07T01:10:17.418604",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.398360",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# def log_transform(df, num_cols=num_cols):\n",
    "#     for f in num_cols:\n",
    "#         if df[f].var() > 1.0:\n",
    "#             print(f\"{f} --> var: {df[f].var()}\")\n",
    "#             df[f] = np.log(df[f])\n",
    "            \n",
    "#     return df\n",
    "\n",
    "# train_df = log_transform(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "65cce2fa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.444868Z",
     "iopub.status.busy": "2023-08-07T01:10:17.444476Z",
     "iopub.status.idle": "2023-08-07T01:10:17.454813Z",
     "shell.execute_reply": "2023-08-07T01:10:17.453747Z"
    },
    "papermill": {
     "duration": 0.026136,
     "end_time": "2023-08-07T01:10:17.457141",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.431005",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df['EJ'] = train_df['EJ'].factorize()[0]\n",
    "test_df['EJ'] = test_df['EJ'].factorize()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5f3fe770",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.484502Z",
     "iopub.status.busy": "2023-08-07T01:10:17.484126Z",
     "iopub.status.idle": "2023-08-07T01:10:17.575684Z",
     "shell.execute_reply": "2023-08-07T01:10:17.574477Z"
    },
    "papermill": {
     "duration": 0.108478,
     "end_time": "2023-08-07T01:10:17.578016",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.469538",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "460d70ae51f6409ba6d7fd151a2d9b3b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/56 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "FEATURES = []\n",
    "means = {}; stds = {}; nans = {};\n",
    "\n",
    "for c in tqdm(train_df.columns[1:-1]):\n",
    "    m = train_df[c].mean()\n",
    "    means[c] = m\n",
    "    \n",
    "    s = train_df[c].std()\n",
    "    stds[c] = s\n",
    "    \n",
    "    train_df[c] = (train_df[c] - m)/s\n",
    "    n = train_df[c].min() - 0.5\n",
    "    nans[c] = n\n",
    "    train_df[c].fillna(n)\n",
    "    FEATURES.append(c)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "011d1e87",
   "metadata": {
    "papermill": {
     "duration": 0.011901,
     "end_time": "2023-08-07T01:10:17.603007",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.591106",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "86a3e723",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.629212Z",
     "iopub.status.busy": "2023-08-07T01:10:17.628782Z",
     "iopub.status.idle": "2023-08-07T01:10:17.698964Z",
     "shell.execute_reply": "2023-08-07T01:10:17.697955Z"
    },
    "papermill": {
     "duration": 0.086105,
     "end_time": "2023-08-07T01:10:17.701370",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.615265",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>AB</th>\n",
       "      <th>AF</th>\n",
       "      <th>AH</th>\n",
       "      <th>AM</th>\n",
       "      <th>AR</th>\n",
       "      <th>AX</th>\n",
       "      <th>AY</th>\n",
       "      <th>AZ</th>\n",
       "      <th>BC</th>\n",
       "      <th>...</th>\n",
       "      <th>FI</th>\n",
       "      <th>FL</th>\n",
       "      <th>FR</th>\n",
       "      <th>FS</th>\n",
       "      <th>GB</th>\n",
       "      <th>GE</th>\n",
       "      <th>GF</th>\n",
       "      <th>GH</th>\n",
       "      <th>GI</th>\n",
       "      <th>GL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00eed32682bb</td>\n",
       "      <td>-1.018705</td>\n",
       "      <td>-1.522401</td>\n",
       "      <td>-0.927922</td>\n",
       "      <td>-0.558863</td>\n",
       "      <td>-0.962863</td>\n",
       "      <td>-2.173291</td>\n",
       "      <td>-0.144715</td>\n",
       "      <td>-2.428708</td>\n",
       "      <td>-0.123575</td>\n",
       "      <td>...</td>\n",
       "      <td>-3.446146</td>\n",
       "      <td>-0.472606</td>\n",
       "      <td>-0.070422</td>\n",
       "      <td>-0.322899</td>\n",
       "      <td>-2.074164</td>\n",
       "      <td>-0.913536</td>\n",
       "      <td>-0.758519</td>\n",
       "      <td>-3.192311</td>\n",
       "      <td>-1.394807</td>\n",
       "      <td>-0.826082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>010ebe33f668</td>\n",
       "      <td>-1.018705</td>\n",
       "      <td>-1.522401</td>\n",
       "      <td>-0.927922</td>\n",
       "      <td>-0.558863</td>\n",
       "      <td>-0.962863</td>\n",
       "      <td>-2.173291</td>\n",
       "      <td>-0.144715</td>\n",
       "      <td>-2.428708</td>\n",
       "      <td>-0.123575</td>\n",
       "      <td>...</td>\n",
       "      <td>-3.446146</td>\n",
       "      <td>-0.472606</td>\n",
       "      <td>-0.070422</td>\n",
       "      <td>-0.322899</td>\n",
       "      <td>-2.074164</td>\n",
       "      <td>-0.913536</td>\n",
       "      <td>-0.758519</td>\n",
       "      <td>-3.192311</td>\n",
       "      <td>-1.394807</td>\n",
       "      <td>-0.826082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02fa521e1838</td>\n",
       "      <td>-1.018705</td>\n",
       "      <td>-1.522401</td>\n",
       "      <td>-0.927922</td>\n",
       "      <td>-0.558863</td>\n",
       "      <td>-0.962863</td>\n",
       "      <td>-2.173291</td>\n",
       "      <td>-0.144715</td>\n",
       "      <td>-2.428708</td>\n",
       "      <td>-0.123575</td>\n",
       "      <td>...</td>\n",
       "      <td>-3.446146</td>\n",
       "      <td>-0.472606</td>\n",
       "      <td>-0.070422</td>\n",
       "      <td>-0.322899</td>\n",
       "      <td>-2.074164</td>\n",
       "      <td>-0.913536</td>\n",
       "      <td>-0.758519</td>\n",
       "      <td>-3.192311</td>\n",
       "      <td>-1.394807</td>\n",
       "      <td>-0.826082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>040e15f562a2</td>\n",
       "      <td>-1.018705</td>\n",
       "      <td>-1.522401</td>\n",
       "      <td>-0.927922</td>\n",
       "      <td>-0.558863</td>\n",
       "      <td>-0.962863</td>\n",
       "      <td>-2.173291</td>\n",
       "      <td>-0.144715</td>\n",
       "      <td>-2.428708</td>\n",
       "      <td>-0.123575</td>\n",
       "      <td>...</td>\n",
       "      <td>-3.446146</td>\n",
       "      <td>-0.472606</td>\n",
       "      <td>-0.070422</td>\n",
       "      <td>-0.322899</td>\n",
       "      <td>-2.074164</td>\n",
       "      <td>-0.913536</td>\n",
       "      <td>-0.758519</td>\n",
       "      <td>-3.192311</td>\n",
       "      <td>-1.394807</td>\n",
       "      <td>-0.826082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>046e85c7cc7f</td>\n",
       "      <td>-1.018705</td>\n",
       "      <td>-1.522401</td>\n",
       "      <td>-0.927922</td>\n",
       "      <td>-0.558863</td>\n",
       "      <td>-0.962863</td>\n",
       "      <td>-2.173291</td>\n",
       "      <td>-0.144715</td>\n",
       "      <td>-2.428708</td>\n",
       "      <td>-0.123575</td>\n",
       "      <td>...</td>\n",
       "      <td>-3.446146</td>\n",
       "      <td>-0.472606</td>\n",
       "      <td>-0.070422</td>\n",
       "      <td>-0.322899</td>\n",
       "      <td>-2.074164</td>\n",
       "      <td>-0.913536</td>\n",
       "      <td>-0.758519</td>\n",
       "      <td>-3.192311</td>\n",
       "      <td>-1.394807</td>\n",
       "      <td>-0.826082</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 57 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Id        AB        AF        AH        AM        AR        AX  \\\n",
       "0  00eed32682bb -1.018705 -1.522401 -0.927922 -0.558863 -0.962863 -2.173291   \n",
       "1  010ebe33f668 -1.018705 -1.522401 -0.927922 -0.558863 -0.962863 -2.173291   \n",
       "2  02fa521e1838 -1.018705 -1.522401 -0.927922 -0.558863 -0.962863 -2.173291   \n",
       "3  040e15f562a2 -1.018705 -1.522401 -0.927922 -0.558863 -0.962863 -2.173291   \n",
       "4  046e85c7cc7f -1.018705 -1.522401 -0.927922 -0.558863 -0.962863 -2.173291   \n",
       "\n",
       "         AY        AZ        BC  ...        FI        FL        FR        FS  \\\n",
       "0 -0.144715 -2.428708 -0.123575  ... -3.446146 -0.472606 -0.070422 -0.322899   \n",
       "1 -0.144715 -2.428708 -0.123575  ... -3.446146 -0.472606 -0.070422 -0.322899   \n",
       "2 -0.144715 -2.428708 -0.123575  ... -3.446146 -0.472606 -0.070422 -0.322899   \n",
       "3 -0.144715 -2.428708 -0.123575  ... -3.446146 -0.472606 -0.070422 -0.322899   \n",
       "4 -0.144715 -2.428708 -0.123575  ... -3.446146 -0.472606 -0.070422 -0.322899   \n",
       "\n",
       "         GB        GE        GF        GH        GI        GL  \n",
       "0 -2.074164 -0.913536 -0.758519 -3.192311 -1.394807 -0.826082  \n",
       "1 -2.074164 -0.913536 -0.758519 -3.192311 -1.394807 -0.826082  \n",
       "2 -2.074164 -0.913536 -0.758519 -3.192311 -1.394807 -0.826082  \n",
       "3 -2.074164 -0.913536 -0.758519 -3.192311 -1.394807 -0.826082  \n",
       "4 -2.074164 -0.913536 -0.758519 -3.192311 -1.394807 -0.826082  \n",
       "\n",
       "[5 rows x 57 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for c in test_df.columns[1:]:\n",
    "    m = means[c]\n",
    "    s = stds[c]\n",
    "    n = nans[c]\n",
    "    \n",
    "    test_df[c] = (test_df[c] - m)/s\n",
    "    test_df[c] = test_df[c].fillna(n)\n",
    "    \n",
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6ca01f25",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.728563Z",
     "iopub.status.busy": "2023-08-07T01:10:17.728183Z",
     "iopub.status.idle": "2023-08-07T01:10:17.769904Z",
     "shell.execute_reply": "2023-08-07T01:10:17.768822Z"
    },
    "papermill": {
     "duration": 0.058403,
     "end_time": "2023-08-07T01:10:17.772610",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.714207",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# initialize the kNN imputer with the desired number of neighbors\n",
    "knn_imp = impute.SimpleImputer(strategy='mean')\n",
    "\n",
    "# perform knn imputation\n",
    "knn_imp.fit(train_df[FEATURES])\n",
    "train_df[FEATURES] = pd.DataFrame(knn_imp.transform(train_df[FEATURES]), \n",
    "                                columns=FEATURES)\n",
    "test_df[FEATURES] = pd.DataFrame(knn_imp.transform(test_df[FEATURES]), \n",
    "                               columns=FEATURES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e9b40a98",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.800041Z",
     "iopub.status.busy": "2023-08-07T01:10:17.799639Z",
     "iopub.status.idle": "2023-08-07T01:10:17.806388Z",
     "shell.execute_reply": "2023-08-07T01:10:17.805414Z"
    },
    "papermill": {
     "duration": 0.022944,
     "end_time": "2023-08-07T01:10:17.808507",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.785563",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def ZScoreOutlierRemoval(data):\n",
    "    z_score = (data - data.mean()) / data.std()\n",
    "    \n",
    "    # remove outliers\n",
    "    clean_data = data[z_score.abs() <= 3]\n",
    "    \n",
    "    return clean_data\n",
    "\n",
    "\n",
    "def IQR_outlier_removal(data):\n",
    "    # calculate the IQR\n",
    "    q1 = data.quantile(0.25)\n",
    "    q3 = data.quantile(0.75)\n",
    "    iqr = q3 - q1\n",
    "    \n",
    "    # remove outliers\n",
    "    clean_data = data[(data >= q1 - 1.5 * iqr) & (data <= q3 + 1.5 * iqr)]\n",
    "    return clean_data\n",
    "\n",
    "# train_df = train_df.progress_apply(ZScoreOutlierRemoval)\n",
    "# test_df = test_df.progress_apply(ZScoreOutlierRemoval)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb710766",
   "metadata": {
    "papermill": {
     "duration": 0.011626,
     "end_time": "2023-08-07T01:10:17.832122",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.820496",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Features with missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "62ff0b38",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.858379Z",
     "iopub.status.busy": "2023-08-07T01:10:17.857976Z",
     "iopub.status.idle": "2023-08-07T01:10:17.863317Z",
     "shell.execute_reply": "2023-08-07T01:10:17.862438Z"
    },
    "papermill": {
     "duration": 0.020968,
     "end_time": "2023-08-07T01:10:17.865433",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.844465",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# plt.figure(figsize=(10, 5))\n",
    "\n",
    "# df_missing = train_df[FEATURES].isna().sum().reset_index()\n",
    "\n",
    "# # rename columns\n",
    "# df_missing.columns = ['feature', 'missing_count']\n",
    "\n",
    "# # filter features with missing values\n",
    "# df_missing = df_missing.loc[df_missing['missing_count'] > 0]\n",
    "\n",
    "# # create a bar chart\n",
    "# df_missing.plot(x='feature', y='missing_count', kind='bar')\n",
    "\n",
    "# # Set the chart title and axis labels\n",
    "# plt.title('Missing Values Count', fontsize=16)\n",
    "# plt.xlabel('Columns', fontsize=14)\n",
    "# plt.ylabel('Count', fontsize=14)\n",
    "\n",
    "# plt.tick_params(axis='x', which='major', labelsize=14)\n",
    "# plt.tick_params(axis='y', which='major', labelsize=14)\n",
    "\n",
    "# # Display the chart\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47e385df",
   "metadata": {
    "papermill": {
     "duration": 0.011905,
     "end_time": "2023-08-07T01:10:17.889412",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.877507",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "`BQ` and `EL` have a lot of missing values. Need to carefully impute the missing values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "368eb56b",
   "metadata": {
    "papermill": {
     "duration": 0.012566,
     "end_time": "2023-08-07T01:10:17.914651",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.902085",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Idea gotten from this wonderful notebook []https://www.kaggle.com/code/bjoernholzhauer/icr-fastai-transfer-learning/notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f5e2a26d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:17.941827Z",
     "iopub.status.busy": "2023-08-07T01:10:17.941412Z",
     "iopub.status.idle": "2023-08-07T01:10:17.946467Z",
     "shell.execute_reply": "2023-08-07T01:10:17.945401Z"
    },
    "papermill": {
     "duration": 0.021647,
     "end_time": "2023-08-07T01:10:17.948780",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.927133",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# # https://www.kaggle.com/code/bjoernholzhauer/icr-fastai-transfer-learning/notebook\n",
    "# # specify target columns\n",
    "# binary_target = ['Class']\n",
    "# cat_targets = ['Alpha', 'Beta', 'Gamma', 'Delta']\n",
    "\n",
    "# # convert the targets to integers\n",
    "# ordinal_encoder = preprocessing.OrdinalEncoder(dtype=np.int32)\n",
    "# train_greek_df[binary_target + cat_targets] = (ordinal_encoder\n",
    "#                                                .fit_transform(\n",
    "#                                                    train_greek_df[binary_target + cat_targets]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "025235b3",
   "metadata": {
    "papermill": {
     "duration": 0.012439,
     "end_time": "2023-08-07T01:10:17.974409",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.961970",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Checking the distribution of the target label\n",
    "\n",
    "There's class imbalance, and this needs to be handled well"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "354dcfff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:18.001818Z",
     "iopub.status.busy": "2023-08-07T01:10:18.001411Z",
     "iopub.status.idle": "2023-08-07T01:10:18.006997Z",
     "shell.execute_reply": "2023-08-07T01:10:18.005845Z"
    },
    "papermill": {
     "duration": 0.022024,
     "end_time": "2023-08-07T01:10:18.009244",
     "exception": false,
     "start_time": "2023-08-07T01:10:17.987220",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# # set the size of the chart\n",
    "# plt.figure(figsize=(5, 4))\n",
    "# plt.legend(fontsize=13)\n",
    "\n",
    "# # create the count plot\n",
    "# sns.countplot(data=train_df, x='EJ', hue='Class')\n",
    "\n",
    "# # set the labels and title\n",
    "# plt.xlabel('EJ', fontsize=14)\n",
    "# plt.ylabel('Count', fontsize=14)\n",
    "# plt.title('Count plot of EJ with class', fontsize=16)\n",
    "\n",
    "# # adjust the tick label size\n",
    "# plt.tick_params(axis='x', which='major', labelsize=14)\n",
    "# plt.tick_params(axis='y', which='major', labelsize=14)\n",
    "\n",
    "# # Add a legend\n",
    "# plt.legend(title='Class')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "18aa350d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:18.036943Z",
     "iopub.status.busy": "2023-08-07T01:10:18.036230Z",
     "iopub.status.idle": "2023-08-07T01:10:18.040895Z",
     "shell.execute_reply": "2023-08-07T01:10:18.039832Z"
    },
    "papermill": {
     "duration": 0.021129,
     "end_time": "2023-08-07T01:10:18.043208",
     "exception": false,
     "start_time": "2023-08-07T01:10:18.022079",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# df = pd.DataFrame(train_df['Class'].value_counts())\n",
    "# df['pct'] = np.round((train_df['Class'].value_counts()/train_df.shape[0]).values * 100, 4)\n",
    "# df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "539d9e78",
   "metadata": {
    "papermill": {
     "duration": 0.012095,
     "end_time": "2023-08-07T01:10:18.067924",
     "exception": false,
     "start_time": "2023-08-07T01:10:18.055829",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Split the data\n",
    "Using Stratified KFold and computing a local cv. Use bagging and downsampling to balance the classes. By bagging, the model will see all the data even though we are downsampling negatives. By balancing the classes, we improve the competition metric which is `balanced log loss`\n",
    "\n",
    "[]https://www.kaggle.com/code/cdeotte/rapids-cuml-svc-baseline-lb-0-27-cv-0-35\n",
    "\n",
    "## Class weights\n",
    "\n",
    "Since the dataset is imbalanced, we would like the classifier to heavily weight the few examples that are available. This causes the model to pay attention to examples from under represented classes\n",
    "\n",
    "Using class weights changes the range of the loss. Which in turn affects the stability of the training depending on the optimizer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "cecb47e4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:18.094866Z",
     "iopub.status.busy": "2023-08-07T01:10:18.094281Z",
     "iopub.status.idle": "2023-08-07T01:10:18.098876Z",
     "shell.execute_reply": "2023-08-07T01:10:18.098210Z"
    },
    "papermill": {
     "duration": 0.020662,
     "end_time": "2023-08-07T01:10:18.100987",
     "exception": false,
     "start_time": "2023-08-07T01:10:18.080325",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# neg, pos = np.bincount(train_df['Class'])\n",
    "# total = neg + pos\n",
    "# weight_for_0 = (1/neg) * (total / 2.0)\n",
    "# weight_for_1 = (1/pos) * (total / 2.0)\n",
    "\n",
    "# class_weight = {0: weight_for_0, 1: weight_for_1}\n",
    "\n",
    "# print(f\"Weight for class 0: {weight_for_0:.2f}\")\n",
    "# print(f\"Weight for class 1: {weight_for_1:.2f}\")\n",
    "\n",
    "# # compute the class weights\n",
    "# weights = compute_class_weight('balanced', classes=[0, 1], y=train_df['Class'])\n",
    "\n",
    "# # create a dictionary mapping class labels to weights\n",
    "# weight_dict = dict(zip([0, 1], weights))\n",
    "\n",
    "# # assign sample weights based on class labels\n",
    "# sample_weights = [weight_dict[yi] for yi in train_df['Class']]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04b51a1c",
   "metadata": {
    "papermill": {
     "duration": 0.011975,
     "end_time": "2023-08-07T01:10:18.125668",
     "exception": false,
     "start_time": "2023-08-07T01:10:18.113693",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f458284e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:18.153071Z",
     "iopub.status.busy": "2023-08-07T01:10:18.152429Z",
     "iopub.status.idle": "2023-08-07T01:10:18.157162Z",
     "shell.execute_reply": "2023-08-07T01:10:18.156468Z"
    },
    "papermill": {
     "duration": 0.020827,
     "end_time": "2023-08-07T01:10:18.159141",
     "exception": false,
     "start_time": "2023-08-07T01:10:18.138314",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# # Define an estimator for RFE\n",
    "# estimator = ensemble.RandomForestClassifier(random_state=config['seed'])\n",
    "\n",
    "# # Create an RFE object with the estimator and 8 features to select\n",
    "# selector = feature_selection.SelectFromModel(estimator=estimator)\n",
    "# # selector = feature_selection.RFE(estimator=estimator, n_features_to_select=32)\n",
    "# selector.fit(train_df[FEATURES], train_df['Class'])\n",
    "# # Print the selected features for each model\n",
    "# SEL_FEATURES = train_df[FEATURES].columns[selector.get_support()].tolist()\n",
    "# print(f\"Selected features:\\n{SEL_FEATURES}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e4e3e33",
   "metadata": {
    "papermill": {
     "duration": 0.012103,
     "end_time": "2023-08-07T01:10:18.183662",
     "exception": false,
     "start_time": "2023-08-07T01:10:18.171559",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c22a8a9e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:18.211363Z",
     "iopub.status.busy": "2023-08-07T01:10:18.210992Z",
     "iopub.status.idle": "2023-08-07T01:10:18.215633Z",
     "shell.execute_reply": "2023-08-07T01:10:18.214503Z"
    },
    "papermill": {
     "duration": 0.02026,
     "end_time": "2023-08-07T01:10:18.217728",
     "exception": false,
     "start_time": "2023-08-07T01:10:18.197468",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# train_df['Class'] = target_df['Class'].values\n",
    "# train_df\n",
    "# train_df.drop('EJ', axis=1, inplace=True)\n",
    "FEATURES.remove('EJ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1fca9c47",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:10:18.245074Z",
     "iopub.status.busy": "2023-08-07T01:10:18.244429Z",
     "iopub.status.idle": "2023-08-07T01:11:06.011435Z",
     "shell.execute_reply": "2023-08-07T01:11:06.010473Z"
    },
    "papermill": {
     "duration": 47.783817,
     "end_time": "2023-08-07T01:11:06.014098",
     "exception": false,
     "start_time": "2023-08-07T01:10:18.230281",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################\n",
      "### Bag  1\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  2\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  3\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  4\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  5\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  6\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  7\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  8\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  9\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  10\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  11\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  12\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  13\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  14\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  15\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  16\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  17\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  18\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  19\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 #########################\n",
      "### Bag  20\n",
      "#########################\n",
      "=> Fold 1 => Fold 2 => Fold 3 => Fold 4 => Fold 5 => Fold 6 => Fold 7 => Fold 8 => Fold 9 => Fold 10 => Fold 11 "
     ]
    }
   ],
   "source": [
    "BAGS = 20\n",
    "FOLDS = 11\n",
    "oof = np.zeros(len(train_df))\n",
    "models = {}\n",
    "\n",
    "params = {'n_estimators': 875, \n",
    "          'max_depth': 4, \n",
    "          'learning_rate': 0.1122658470071706, \n",
    "          'min_child_weight': 3, \n",
    "          'subsample': 0.6916380656264425, \n",
    "          'colsample_bytree': 0.9271370816204398, \n",
    "          'gamma': 0.849153073983306}\n",
    "\n",
    "# model = xgb.XGBClassifier(**params)\n",
    "\n",
    "for bag in range(BAGS):\n",
    "    print('#'*25)\n",
    "    print('### Bag ', bag+1)\n",
    "    print('#'*25)\n",
    "    models[bag] = []\n",
    "    skf = model_selection.StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=bag)\n",
    "    \n",
    "    for fold, (tr_idx, val_idx) in enumerate(skf.split(X=train_df[FEATURES], y=train_df['Class'])):\n",
    "        print(f\"=> Fold {fold+1}\", end=' ')\n",
    "        \n",
    "        # Downsample negative class to balance classes\n",
    "        y_train = train_df.loc[tr_idx, 'Class']\n",
    "        RMV = y_train.loc[y_train==0].sample(\n",
    "            frac=0.7, random_state=bag*BAGS+fold).index.values\n",
    "        tr_idx = np.setdiff1d(tr_idx, RMV)\n",
    "        \n",
    "        # train data\n",
    "        X_train = train_df.loc[tr_idx, FEATURES]\n",
    "        y_train = train_df.loc[tr_idx, 'Class']\n",
    "        \n",
    "        # valid data\n",
    "        X_valid = train_df.loc[val_idx, FEATURES]\n",
    "        y_valid = train_df.loc[val_idx, 'Class']\n",
    "        \n",
    "        # Train model\n",
    "        clf = ensemble.HistGradientBoostingClassifier()\n",
    "#         clf = xgb.XGBClassifier(**params)\n",
    "        clf.fit(X_train, y_train)\n",
    "        \n",
    "        # Hist Gradient oof predictions\n",
    "        oof[val_idx] += clf.predict_proba(X_valid)[:, 1]/BAGS\n",
    "        models[bag].append(clf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89b258e9",
   "metadata": {
    "papermill": {
     "duration": 0.026463,
     "end_time": "2023-08-07T01:11:06.069311",
     "exception": false,
     "start_time": "2023-08-07T01:11:06.042848",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Compute CV score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f7031cfc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:11:06.124707Z",
     "iopub.status.busy": "2023-08-07T01:11:06.124297Z",
     "iopub.status.idle": "2023-08-07T01:11:06.139350Z",
     "shell.execute_reply": "2023-08-07T01:11:06.138205Z"
    },
    "papermill": {
     "duration": 0.045841,
     "end_time": "2023-08-07T01:11:06.141880",
     "exception": false,
     "start_time": "2023-08-07T01:11:06.096039",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV score:  0.22312762031257014\n"
     ]
    }
   ],
   "source": [
    "def balanced_log_loss(y_true, y_pred):\n",
    "    nc = np.bincount(y_true)\n",
    "    return metrics.log_loss(y_true, y_pred, sample_weight = 1/nc[y_true], eps=1e-15)\n",
    "\n",
    "m = balanced_log_loss(train_df.Class.values, oof)\n",
    "print('CV score: ', m)\n",
    "\n",
    "# isotoninc, xgb -- 0.31319\n",
    "# base, xgb -- 0.36342\n",
    "# base, hgb -- 0.3929\n",
    "# isotonic, hgb -- 0.2854"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b24bc9c3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:11:06.197488Z",
     "iopub.status.busy": "2023-08-07T01:11:06.197114Z",
     "iopub.status.idle": "2023-08-07T01:11:06.202074Z",
     "shell.execute_reply": "2023-08-07T01:11:06.201053Z"
    },
    "papermill": {
     "duration": 0.035493,
     "end_time": "2023-08-07T01:11:06.204360",
     "exception": false,
     "start_time": "2023-08-07T01:11:06.168867",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.save('oof.npy', oof)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19275fb7",
   "metadata": {
    "papermill": {
     "duration": 0.026394,
     "end_time": "2023-08-07T01:11:06.257983",
     "exception": false,
     "start_time": "2023-08-07T01:11:06.231589",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# CV results\n",
    "## Without class weights:\n",
    "> CV: 0.21258, LB: 0.17\n",
    "\n",
    "## With class weights\n",
    "> CV: 0.20688, LB: 0.17"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "85089ba5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:11:06.313197Z",
     "iopub.status.busy": "2023-08-07T01:11:06.312795Z",
     "iopub.status.idle": "2023-08-07T01:11:06.753660Z",
     "shell.execute_reply": "2023-08-07T01:11:06.752527Z"
    },
    "papermill": {
     "duration": 0.471788,
     "end_time": "2023-08-07T01:11:06.756445",
     "exception": false,
     "start_time": "2023-08-07T01:11:06.284657",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAG7CAYAAADpF271AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAuVklEQVR4nO3deXQUZb7G8afJBkTSEpYskxDQC1cRRA0jEEaBYRMRZBivKF5NPCgoiiACAzr3BmcQRhwjVxFcLoKyCOMCcgSXoLKJCwRQtquMgCZCjGxJwJhA8t4/OKlJk7VDJ3k7+X7OqXPSXW9V/+rtTurJW9VVLmOMEQAAgEUa1XUBAAAA5yOgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaCgwVu0aJFcLpdcLpcOHTpU1+WgHvnhhx80ZswYXXrppWrcuLHzOVu1alVdlwZYj4ACv7J+/Xrnj/z06dOrtMz06dOdZdavX1+j9QHFfvjhB8XHx+ull17SgQMHlJ+f75P1fvXVV5o2bZp++9vfKjIyUsHBwYqIiFB8fLz+9Kc/aefOndVa73vvvafRo0erY8eOCg8PV+PGjRUbG6s+ffpo9uzZOnz4cKXrOHTokPO7VpWpqr/DaJgC67oAoL46dOiQ2rVrJ0lauHChkpKS6rYg1KoZM2bo6NGjCgwM1BNPPKHrr79eF110kSQpLi7O6/WdPHlSDz30kJYsWaLzb6GWlZWlrKwsbd++XU899ZTuuOMOPffcc7r44osrXe+ePXt077336rPPPis1LyMjQxkZGVq/fr3+8pe/aNq0aXr00Uflcrm8rh/wFgEFDV5SUhLhAT63bt06SdKwYcM0ZcqUC1rX4cOHdcMNN2jXrl2SpJiYGN1999363e9+p5YtW+rYsWP69NNP9corryg9PV1LlizRV199pQ8++EBRUVHlrnfTpk0aOnSoTp48KUmKj49XYmKirrzySl100UX68ccf9f777+vVV1/V6dOn9ec//1m7d+/Wa6+9pqCgoAprvvnmmzVjxowK27Ru3dq7jkCDQkABgBrw448/SpI6dOhwQes5e/asbrnlFiecjBw5Ui+++KIzGlOsf//+mjx5skaPHq1ly5Zp165duuWWW7Rx40YFBASUWm9GRoaGDRumkydPyuVy6e9//7sefvhhj9GR+Ph4DR06VFOmTNGwYcP01Vdfafny5WrTpo2efPLJCuu++OKL1alTpwvadjRsnIMCADWgoKBAkiodaahMSkqKc/jlxhtv1OLFi0uFk2KhoaFavHixbrzxRknSli1blJKSUmbb0aNH6/jx45KkmTNnauLEieUeumnbtq3WrVunmJgYSdJTTz2lL7744oK2C6gMAQUNXlW+xZOWlqZRo0apQ4cOCg0NdU4gjI+P1wMPPKDVq1d7nBfgcrmc808k6e67767SCYJFRUVasmSJbrzxRucEyFatWqlPnz6aN2+es9OryNGjRzV58mR16NBBTZo0UUREhPr376+VK1dWaXvbtm0rl8vlHPZKS0tTUlKS2rVrp5CQkFI7sQMHDujpp5/WkCFD1LZtWzVp0kRNmjRRXFycRowYoffff7/Ces+vp6CgQCkpKeratavcbrfCw8PVu3dvrVmzxmO53NxczZ49W1dffbXCwsJ08cUXq3///vroo48q7aOqqM57UXJbij3++OMe77s3hxMLCgr0zDPPSJIaN26sF198UY0aVfxnu1GjRnrhhRfUuHFjSdIzzzxTqtavvvpK7733niTpyiuv1OTJkyutpWXLlk4txhjNnj27ytsBVIsB/Mgnn3xiJBlJJjk5uUrLJCcnO8t88sknpeYvXLjQmX/w4MFS81NSUkyjRo2cNuVNubm5zjKVtS2r/mPHjpmePXtWuMzll19uDh06VO627ty507Rq1arc5UePHl3p9sbFxRlJJjEx0cyfP98EBgaWWk+xAwcOVGlb//M//9OcOXOmzJpL1vPVV1+Zbt26lbuep59+2hhjzPfff2+uuOKKMtu4XC6zePHicvuoKqr7XpTclvKmxMTEKtfxzjvvOMuNHDnSq224/fbbnWXfeecdj3kPP/ywM++ll16q8joLCwtNTEyMkWQaNWpkjh8/7jH/4MGD1dpOoCyMoAAV+PrrrzVp0iQVFRWpXbt2evrpp/XRRx9px44d2rRpk1555RXdeeedpYbcd+3apQ8++MB5PGPGDO3atctjGjt2rDO/sLBQN910kz799FNJUq9evfTGG29o27ZtWr16tYYNGyZJ2rdvn/r27atTp06VqvXEiRO64YYb9PPPP0uS7rjjDr333nvatm2bli9frh49euill17SCy+8UKVt37p1qx588EHFxMRo7ty5+uyzz7R582bNmjXLo+7g4GANGTJEzz77rNatW6ft27dr3bp1mjdvnq644gpJ0pIlS/TXv/610tccPXq00tLSNHbsWKWmpmrbtm363//9X+dEz8mTJ2v37t0aPny4Dhw4oKlTp2r9+vXaunWr5syZI7fbLWOMxo4dq6ysrCpt5/ku5L0YNmyY8/4Wu//++z3e9yeeeKLKtWzcuNH5eciQIV5tx9ChQ52fN23a5JP1NmrUSIMHD5Z0boRpy5YtXtUEeKWuExLgjZIjKPfff7/ZtWtXpdP9999f7RGU//qv/zKSTGhoqMnMzCy3rpMnT5rCwkKP50r+N7lw4cIKt2vu3LlO27vuussUFRWVavPoo486baZMmVJq/kMPPeTM//vf/15q/tmzZ83NN9/s8d98RSMokkznzp3NiRMnyq371KlT5vDhw+XOLyoqMklJSU4fnjx5slSbkv3vcrnMypUrS7X5+uuvTUBAgJFkWrVqZUJCQsznn39eqt2aNWucdaWkpJRbV0V88V4Y869RtKqO9JWlX79+znq++eYbr5b95ptvnGX79+/vMS8oKMhIMlFRUV7X9OKLLzrrfeKJJzzmlfzM33zzzRX+XmZkZHj92mhYCCjwKyUDSnUmbwPKvffeaySZq6++2utavQkol19+uZFkWrZsaXJycspsc/bsWXPZZZcZSaZ58+bm119/debl5eUZt9ttJJlrrrmmzJ2qMcZkZmaaxo0bVzmgbNy4scrbW55jx4454eLNN98sNb9k/48YMaLc9fTq1ctpN3Xq1HLbFdf/hz/8oVr1Xuh7UcwXAeWqq65y1pOdne3VsidPnnSWveaaa8p8vjqf61WrVjnLT5w40WNeyc98ZROHgFAZDvEAFSg+tLB37159+eWXNfIahw8f1r59+yRJt956q5o1a1Zmu4CAAN19992Szh3O2b59uzMvLS1N2dnZkqS77rqr3G9jREREaODAgVWqKzY2Vtddd12Vt0OSzpw5o4yMDO3bt0+7d+/W7t27dfjwYbVo0ULSuZMzK3LbbbeVO+/KK690fh4xYkSl7Q4cOOBN6ZJ88174Um5urvNzaGioV8uWbJ+Tk+OTdVa0XsDXCCjwW8nJyTLnRgErnJKTk6v9GrfffruCgoKUn5+vnj17asiQIXrhhRe0Z8+eUlfzrK7du3c7P3fr1q3CtiXnl1yu5M/x8fEVrqNr165VqqtkIKjImTNn9Pzzz6t79+666KKLFBsbq44dO6pz587OVHw+yNGjRytcV0XXDCl5VdSqtCu5I64qX7wXvlQyIJV13lFFSrYPCwvzyTorWu/5EhMTK/y9XLRokdevjYaFgAJU4LLLLtPrr7+u5s2b6+zZs3r33Xd1//33q1OnTmrdurXuvPPOUicgeqv4WhTSuRGOikRGRpa53IkTJ5yfK7s6Z6tWrapUV/PmzSttc/z4cfXo0UMPPvigvvjii0q/Bp2Xl1fh/KZNm5Y7r+TXa6vSrrCwsMLXKosv3gtfKh55kqTMzEyvlv3pp5/KXI/b7VZgYGC11lnRegFfI6AAlfjjH/+ogwcP6sUXX9Tw4cOdHfzRo0e1ZMkSXX/99UpKSlJRUdEFv1Zl9zjx1ahNVZR19dHzjR8/XmlpaZLOfYNl9erVOnTokH755RcVFRU5/y3HxsZKqt36L5QN70WXLl2cn3fs2OHVsiUPO5Vcj/Sv0bHMzEyvQ0pF6wV8iYACVIHb7dbo0aP11ltvKSsrS3v27NGsWbMUHR0tSXr11Vf13HPPVWvd4eHhzs+V7SxK/vdacrmSox2Vfb22+GvIFyonJ0crVqyQdO7y6ytXrtSQIUMUFxenJk2aeOzgS47w2MwX74UvXX/99c7Pq1ev9mrZku3PP5eouustKirS2rVrJZ0bqUpISPCqJsAbBBSgGjp27KipU6fq888/d04a/Mc//uHRpqp3fC15v5LKLh9e8kTdkssVX29EkrZt21bhOiqbX1X79+/XmTNnJFV8cus333xTrXMd6oIv3gtfuuGGG5xDTStXrlRGRkaVlktPT9eqVasknTsUdcMNN3jMT0xMdH5+/vnnq3w47K233nJquPnmm6t0GBCoLgIKcAFiY2OdEzbPPwG0+FLjkpSfn1/uOqKjo3X55ZdLkt54441yT+4sLCx0Tixs3ry5rrnmGmde8WXhJWnx4sXlHn746aefPC4gdyHOnj3r/PzLL7+U266qF4azgS/eC18KCQnR+PHjJUm//vqrxowZU+mhxKKiIt1333369ddfJUkTJkxQcHCwR5urrrpKAwYMkHTuYoRPPfVUpbUcPXpUEydOlHQufF/oHZqByhBQgAqsWrXKuRV9WdLT0/V///d/kuRx7x3p3AmExTuG7777rsLXeeCBBySdO/wybty4MgPG448/rr1790qS7r33XoWEhDjzGjdurLvuukvSuXMEyrpBXFFRkcaMGePsuC7Uv/3bvzmjRK+99lqZbd59991qH/qqKxf6Xvja5MmTnW8MrV27VnfddVe5I1KnT5/WXXfd5RyG6datmx555JEy27788svON54effRRzZkzp9xg+/3336tfv37O6MnEiRPVvXv3C9ksoHK1dcEVwBdq+148vXr1Mk2bNjX/8R//YebPn2/Wr19vduzYYT7++GMze/ZsExsb6yy7atWqUusuvp9LixYtzLJly8zevXvN/v37zf79+82xY8ecdmfPnjU9evRw1tW7d2/zxhtvmLS0NPPuu++a4cOHO/MuvfRSj/v+FDt27JiJjIx02t1xxx3m/fffN2lpaWbFihUmISHBSDLXXnut06as+/qUvBdPZQYPHuysa+DAgebtt98227ZtM2vXrjWjRo0yAQEBpn379s79gcpaZ2X3BipW8n2sSGJiopFk4uLiKq2/LL54L4zxzYXaimVkZJiOHTs664yJiTHJyckmNTXVbN++3aSmpprp06d7fB47duxY6dVa169fb8LCwpxlunbtap577jmzYcMGs23bNrN69WozduxYExoa6rS55ZZbTH5+fpnr41488CUCCvxKXQSU4nnlTQEBAWbmzJllvva7775rXC5XmcvVxc0Ck5KSzIIFC5zHZV2+35uA8sMPP5g2bdqU+3pt2rQxe/bsqXCdtgUUY3zzXvgyoBhjzPHjx83IkSPL/TwVTy6Xy9x+++2lbuRXnq+//rrCGzQWT02bNjXTp08vdUuHkggo8CUO8QAV+Mc//qGlS5cqKSlJV111lSIjIxUYGKiLLrpInTp10tixY7Vjxw5NmzatzOUHDx6sjz76SDfffLOio6MVFBRU7muFh4dr48aNWrx4sXNyZFBQkFq0aKHevXtr7ty52rlzp+Li4spdR5cuXbR371498sgjat++vUJCQtSyZUv16dNHy5Yt08KFCz2u/ll83kp1xcbGavv27Zo8ebI6dOigkJAQud1udenSRcnJydq5c6c6dux4Qa9RF3zxXvha8+bNtXTpUm3fvl1TpkzRNddco1atWikoKEitWrXS1VdfrUmTJiktLU3Lli2r8gmsnTt31ueff653331Xo0aN0mWXXSa3263g4GBFR0fr+uuv16xZs/Ttt98qOTnZ43o0QE1yGeNHFyYAcMHuueceLViwQDExMUpPT6/rcgCgTERhoAHJy8vTO++8I0mc5AjAagQUoB757rvvyv0mRmFhoe6//37n69Alr4UBALbhEA9QjyQlJenLL7/Ubbfdpm7duql169bKy8vT119/rZdfftm5THnfvn2Vmppa5YvJAUBtC6zrAgD41r59+yq8g3PPnj21YsUKwgkAqzGCAtQj33zzjd566y2lpqbq+++/188//6wzZ86oRYsW6tq1q0aMGKHbbruNb2IAsB4BBQAAWMcvD/EUFRXp8OHDatasGcPUAAD4CWOMcnNzFR0dXelIrl8GlMOHDys2NrauywAAANWQnp6umJiYCtv4ZUBp1qyZpHMbGBYWVsfVAACAqsjJyVFsbKyzH6+IXwaU4sM6YWFhBBQAAPxMVU7P4FR+AABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsE1nUBNmo7dY3H40N/G1xHlQAA0DAxggIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYx6uAMmvWLP32t79Vs2bN1Lp1aw0bNkzffPONRxtjjKZPn67o6Gg1adJEvXv31p49ezza5Ofna9y4cWrZsqVCQ0M1dOhQZWRkXPjWAACAesGrgLJhwwY98MAD+vzzz5WamqqzZ89qwIABOn36tNNm9uzZSklJ0dy5c7V161ZFRkaqf//+ys3NddpMmDBBK1eu1PLly7V582adOnVKN910kwoLC323ZQAAwG+5jDGmugv//PPPat26tTZs2KDrr79exhhFR0drwoQJ+tOf/iTp3GhJRESEnnzySY0ZM0bZ2dlq1aqVFi9erBEjRkiSDh8+rNjYWK1du1YDBw6s9HVzcnLkdruVnZ2tsLCw6pZfrrZT13g8PvS3wT5/DQAAGhpv9t8XdA5Kdna2JCk8PFySdPDgQWVmZmrAgAFOm5CQEPXq1UtbtmyRJKWlpenMmTMebaKjo9WpUyenzfny8/OVk5PjMQEAgPqr2gHFGKOJEyfqd7/7nTp16iRJyszMlCRFRER4tI2IiHDmZWZmKjg4WM2bNy+3zflmzZolt9vtTLGxsdUtGwAA+IFqB5QHH3xQX3/9tV5//fVS81wul8djY0yp585XUZtp06YpOzvbmdLT06tbNgAA8APVCijjxo3T6tWr9cknnygmJsZ5PjIyUpJKjYRkZWU5oyqRkZEqKCjQiRMnym1zvpCQEIWFhXlMAACg/vIqoBhj9OCDD+rtt9/Wxx9/rHbt2nnMb9eunSIjI5Wamuo8V1BQoA0bNighIUGSFB8fr6CgII82R44c0e7du502AACgYQv0pvEDDzygZcuW6Z133lGzZs2ckRK3260mTZrI5XJpwoQJmjlzptq3b6/27dtr5syZatq0qUaOHOm0HTVqlB555BG1aNFC4eHhmjRpkjp37qx+/fr5fgsBAIDf8SqgzJ8/X5LUu3dvj+cXLlyopKQkSdKUKVOUl5ensWPH6sSJE+rWrZs+/PBDNWvWzGn/zDPPKDAwULfeeqvy8vLUt29fLVq0SAEBARe2NQAAoF64oOug1BWugwIAgP+pteugAAAA1AQCCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALCO1wFl48aNGjJkiKKjo+VyubRq1SqP+UlJSXK5XB5T9+7dPdrk5+dr3LhxatmypUJDQzV06FBlZGRc0IYAAID6w+uAcvr0aXXp0kVz584tt80NN9ygI0eOONPatWs95k+YMEErV67U8uXLtXnzZp06dUo33XSTCgsLvd8CAABQ7wR6u8CgQYM0aNCgCtuEhIQoMjKyzHnZ2dlasGCBFi9erH79+kmSlixZotjYWK1bt04DBw70tiQAAFDP1Mg5KOvXr1fr1q3VoUMH3XvvvcrKynLmpaWl6cyZMxowYIDzXHR0tDp16qQtW7aUub78/Hzl5OR4TAAAoP7yeUAZNGiQli5dqo8//lhPP/20tm7dqt///vfKz8+XJGVmZio4OFjNmzf3WC4iIkKZmZllrnPWrFlyu93OFBsb6+uyAQCARbw+xFOZESNGOD936tRJXbt2VVxcnNasWaPhw4eXu5wxRi6Xq8x506ZN08SJE53HOTk5hBQAAOqxGv+acVRUlOLi4rR//35JUmRkpAoKCnTixAmPdllZWYqIiChzHSEhIQoLC/OYAABA/VXjAeXYsWNKT09XVFSUJCk+Pl5BQUFKTU112hw5ckS7d+9WQkJCTZcDAAD8gNeHeE6dOqV//vOfzuODBw9q586dCg8PV3h4uKZPn64//vGPioqK0qFDh/Too4+qZcuW+sMf/iBJcrvdGjVqlB555BG1aNFC4eHhmjRpkjp37ux8qwcAADRsXgeUbdu2qU+fPs7j4nNDEhMTNX/+fO3atUuvvfaaTp48qaioKPXp00crVqxQs2bNnGWeeeYZBQYG6tZbb1VeXp769u2rRYsWKSAgwAebBAAA/J3LGGPqughv5eTkyO12Kzs7u0bOR2k7dY3H40N/G+zz1wAAoKHxZv/NvXgAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOl4HlI0bN2rIkCGKjo6Wy+XSqlWrPOYbYzR9+nRFR0erSZMm6t27t/bs2ePRJj8/X+PGjVPLli0VGhqqoUOHKiMj44I2BAAA1B9eB5TTp0+rS5cumjt3bpnzZ8+erZSUFM2dO1dbt25VZGSk+vfvr9zcXKfNhAkTtHLlSi1fvlybN2/WqVOndNNNN6mwsLD6WwIAAOqNQG8XGDRokAYNGlTmPGOM5syZo8cee0zDhw+XJL366quKiIjQsmXLNGbMGGVnZ2vBggVavHix+vXrJ0lasmSJYmNjtW7dOg0cOPACNgcAANQHPj0H5eDBg8rMzNSAAQOc50JCQtSrVy9t2bJFkpSWlqYzZ854tImOjlanTp2cNufLz89XTk6OxwQAAOovnwaUzMxMSVJERITH8xEREc68zMxMBQcHq3nz5uW2Od+sWbPkdrudKTY21pdlAwAAy9TIt3hcLpfHY2NMqefOV1GbadOmKTs725nS09N9VisAALCPTwNKZGSkJJUaCcnKynJGVSIjI1VQUKATJ06U2+Z8ISEhCgsL85gAAED95dOA0q5dO0VGRio1NdV5rqCgQBs2bFBCQoIkKT4+XkFBQR5tjhw5ot27dzttAABAw+b1t3hOnTqlf/7zn87jgwcPaufOnQoPD1ebNm00YcIEzZw5U+3bt1f79u01c+ZMNW3aVCNHjpQkud1ujRo1So888ohatGih8PBwTZo0SZ07d3a+1QMAABo2rwPKtm3b1KdPH+fxxIkTJUmJiYlatGiRpkyZory8PI0dO1YnTpxQt27d9OGHH6pZs2bOMs8884wCAwN16623Ki8vT3379tWiRYsUEBDgg00CAAD+zmWMMXVdhLdycnLkdruVnZ1dI+ejtJ26xuPxob8N9vlrAADQ0Hiz/+ZePAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsE5gXRfgD9pOXVPquUN/G1wHlQAA0DAwggIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHZ8HlOnTp8vlcnlMkZGRznxjjKZPn67o6Gg1adJEvXv31p49e3xdBgAA8GM1MoJyxRVX6MiRI860a9cuZ97s2bOVkpKiuXPnauvWrYqMjFT//v2Vm5tbE6UAAAA/FFgjKw0M9Bg1KWaM0Zw5c/TYY49p+PDhkqRXX31VERERWrZsmcaMGVPm+vLz85Wfn+88zsnJqYmyAQCAJWpkBGX//v2Kjo5Wu3btdNttt+nAgQOSpIMHDyozM1MDBgxw2oaEhKhXr17asmVLueubNWuW3G63M8XGxtZE2QAAwBI+DyjdunXTa6+9pg8++EAvv/yyMjMzlZCQoGPHjikzM1OSFBER4bFMRESEM68s06ZNU3Z2tjOlp6f7umwAAGARnx/iGTRokPNz586d1aNHD1166aV69dVX1b17d0mSy+XyWMYYU+q5kkJCQhQSEuLrUgEAgKVq/GvGoaGh6ty5s/bv3++cl3L+aElWVlapURUAANBw1XhAyc/P1759+xQVFaV27dopMjJSqampzvyCggJt2LBBCQkJNV0KAADwEz4/xDNp0iQNGTJEbdq0UVZWlmbMmKGcnBwlJibK5XJpwoQJmjlzptq3b6/27dtr5syZatq0qUaOHOnrUgAAgJ/yeUDJyMjQ7bffrqNHj6pVq1bq3r27Pv/8c8XFxUmSpkyZory8PI0dO1YnTpxQt27d9OGHH6pZs2a+LgUAAPgplzHG1HUR3srJyZHb7VZ2drbCwsJ8vv62U9dU2ubQ3wb7/HUBAKjPvNl/cy8eAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsE1jXBfirtlPXeDw+9LfBdVQJAAD1DyMoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKzD3Yx95Py7G0vc4RgAgOoioAAA0MDZ+E82h3gAAIB1CCgAAMA6HOKpQecPmdX1cBkAAP6CERQAAGAdRlBqkY0nIQEA6jd/3fcQUCzjrx8kAAB8iUM8AADAOgQUAABgHQIKAACwDgEFAABYh5NkAQBoYMr6QoZtGEEBAADWIaAAAADrcIinjvnDMBsAwA4N6RYqjKAAAADrMIJSTzSkVA0AOKc+X32cgOIHbDsMVJ9/IQDAFrb97a9tHOIBAADWYQSlnqpq8mbkAwBqHofhvUdAQaVsG2b01S86fzD+hcN2DQPvc8Wq8rfOV/3Fe1E5DvEAAADr1OkIyrx58/TUU0/pyJEjuuKKKzRnzhxdd911dVkS/FBN/tdTnVEWX/1n5A//YTEK5Xv+8L7Xpur2R019Nv3h/bFt1Lu66iygrFixQhMmTNC8efPUs2dPvfjiixo0aJD27t2rNm3a1FVZDY6vPsi2H3apL7+w57MtnFVXfQiCtXl4wJfq8rCGrz6b1VVTfxfq69+b2lZnh3hSUlI0atQo3XPPPbr88ss1Z84cxcbGav78+XVVEgAAsESdjKAUFBQoLS1NU6dO9Xh+wIAB2rJlS6n2+fn5ys/Pdx5nZ2dLknJycmqkvqL8X2pkvQ1JVd6bTskfVNqmzcNv+KKcKqnKa1Vlu8r6/PhqO6qznrKW2f34QI/HVfnMV7d/zn+fz3/tsl6/ujXX1OeurJrPV5U+rEr/VKWeqqrO++zL169MdfvDV6/lq7/1dfn7XZNqYh9bvE5jTOWNTR348ccfjSTz6aefejz/xBNPmA4dOpRqn5ycbCQxMTExMTEx1YMpPT290qxQpyfJulwuj8fGmFLPSdK0adM0ceJE53FRUZGOHz+uFi1alNn+QuTk5Cg2Nlbp6ekKCwvz6brxL/Rz7aCfaw99XTvo59pRU/1sjFFubq6io6MrbVsnAaVly5YKCAhQZmamx/NZWVmKiIgo1T4kJEQhISEez1188cU1WaLCwsL48NcC+rl20M+1h76uHfRz7aiJfna73VVqVycnyQYHBys+Pl6pqakez6empiohIaEuSgIAABaps0M8EydO1J133qmuXbuqR48eeumll/TDDz/ovvvuq6uSAACAJeosoIwYMULHjh3TX/7yFx05ckSdOnXS2rVrFRcXV1clSTp3OCk5ObnUISX4Fv1cO+jn2kNf1w76uXbY0M8uY6ryXR8AAIDaw714AACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYp0EGlHnz5qldu3Zq3Lix4uPjtWnTpgrbb9iwQfHx8WrcuLEuueQSvfDCC7VUqX/zpp/ffvtt9e/fX61atVJYWJh69OihDz6omZuG1Tfefp6LffrppwoMDNRVV11VswXWE972c35+vh577DHFxcUpJCREl156qV555ZVaqta/edvXS5cuVZcuXdS0aVNFRUXp7rvv1rFjx2qpWv+0ceNGDRkyRNHR0XK5XFq1alWly9T6vtAnd//zI8uXLzdBQUHm5ZdfNnv37jXjx483oaGh5vvvvy+z/YEDB0zTpk3N+PHjzd69e83LL79sgoKCzJtvvlnLlfsXb/t5/Pjx5sknnzRffvml+fbbb820adNMUFCQ2b59ey1X7l+87ediJ0+eNJdccokZMGCA6dKlS+0U68eq089Dhw413bp1M6mpqebgwYPmiy++KHWDVJTmbV9v2rTJNGrUyPzP//yPOXDggNm0aZO54oorzLBhw2q5cv+ydu1a89hjj5m33nrLSDIrV66ssH1d7AsbXEC59tprzX333efx3GWXXWamTp1aZvspU6aYyy67zOO5MWPGmO7du9dYjfWBt/1clo4dO5rHH3/c16XVK9Xt5xEjRpg///nPJjk5mYBSBd7283vvvWfcbrc5duxYbZRXr3jb10899ZS55JJLPJ579tlnTUxMTI3VWN9UJaDUxb6wQR3iKSgoUFpamgYMGODx/IABA7Rly5Yyl/nss89KtR84cKC2bdumM2fO1Fit/qw6/Xy+oqIi5ebmKjw8vCZKrBeq288LFy7Ud999p+Tk5JousV6oTj+vXr1aXbt21ezZs/Wb3/xGHTp00KRJk5SXl1cbJfut6vR1QkKCMjIytHbtWhlj9NNPP+nNN9/U4MGDa6PkBqMu9oV1dqn7unD06FEVFhaWumNyREREqTsrF8vMzCyz/dmzZ3X06FFFRUXVWL3+qjr9fL6nn35ap0+f1q233loTJdYL1enn/fv3a+rUqdq0aZMCAxvUr3+1VaefDxw4oM2bN6tx48ZauXKljh49qrFjx+r48eOch1KB6vR1QkKCli5dqhEjRujXX3/V2bNnNXToUD333HO1UXKDURf7wgY1glLM5XJ5PDbGlHqusvZlPQ9P3vZzsddff13Tp0/XihUr1Lp165oqr96oaj8XFhZq5MiRevzxx9WhQ4faKq/e8ObzXFRUJJfLpaVLl+raa6/VjTfeqJSUFC1atIhRlCrwpq/37t2rhx56SP/93/+ttLQ0vf/++zp48CA3nq0Btb0vbFD/QrVs2VIBAQGlknhWVlapZFgsMjKyzPaBgYFq0aJFjdXqz6rTz8VWrFihUaNG6Y033lC/fv1qsky/520/5+bmatu2bdqxY4cefPBBSed2pMYYBQYG6sMPP9Tvf//7Wqndn1Tn8xwVFaXf/OY3crvdznOXX365jDHKyMhQ+/bta7Rmf1Wdvp41a5Z69uypyZMnS5KuvPJKhYaG6rrrrtOMGTMY5faRutgXNqgRlODgYMXHxys1NdXj+dTUVCUkJJS5TI8ePUq1//DDD9W1a1cFBQXVWK3+rDr9LJ0bOUlKStKyZcs4flwF3vZzWFiYdu3apZ07dzrTfffdp3//93/Xzp071a1bt9oq3a9U5/Pcs2dPHT58WKdOnXKe+/bbb9WoUSPFxMTUaL3+rDp9/csvv6hRI89dWUBAgKR//YePC1cn+8IaO/3WUsVfYVuwYIHZu3evmTBhggkNDTWHDh0yxhgzdepUc+eddzrti79a9fDDD5u9e/eaBQsW8DXjKvC2n5ctW2YCAwPN888/b44cOeJMJ0+erKtN8Ave9vP5+BZP1Xjbz7m5uSYmJsbccsstZs+ePWbDhg2mffv25p577qmrTfAb3vb1woULTWBgoJk3b5757rvvzObNm03Xrl3NtddeW1eb4Bdyc3PNjh07zI4dO4wkk5KSYnbs2OF8nduGfWGDCyjGGPP888+buLg4ExwcbK655hqzYcMGZ15iYqLp1auXR/v169ebq6++2gQHB5u2bdua+fPn13LF/smbfu7Vq5eRVGpKTEys/cL9jLef55IIKFXnbT/v27fP9OvXzzRp0sTExMSYiRMnml9++aWWq/ZP3vb1s88+azp27GiaNGlioqKizB133GEyMjJquWr/8sknn1T4N9eGfaHLGMbAAACAXRrUOSgAAMA/EFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDr/D4qSSkr8CxyFAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(oof, bins=100)\n",
    "plt.title('Histogram of OOF', size=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf0283d6",
   "metadata": {
    "papermill": {
     "duration": 0.027299,
     "end_time": "2023-08-07T01:11:06.811472",
     "exception": false,
     "start_time": "2023-08-07T01:11:06.784173",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Infer Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0da5d335",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:11:06.868832Z",
     "iopub.status.busy": "2023-08-07T01:11:06.867674Z",
     "iopub.status.idle": "2023-08-07T01:11:08.101871Z",
     "shell.execute_reply": "2023-08-07T01:11:08.100978Z"
    },
    "papermill": {
     "duration": 1.26587,
     "end_time": "2023-08-07T01:11:08.104537",
     "exception": false,
     "start_time": "2023-08-07T01:11:06.838667",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "preds = np.zeros(len(test_df))\n",
    "\n",
    "for bag in range(BAGS):\n",
    "    for clf in models[bag]:\n",
    "        X_test = test_df[FEATURES]\n",
    "        preds += clf.predict_proba(X_test)[:, 1] / FOLDS / BAGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f34a3eca",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:11:08.163958Z",
     "iopub.status.busy": "2023-08-07T01:11:08.163545Z",
     "iopub.status.idle": "2023-08-07T01:11:08.168856Z",
     "shell.execute_reply": "2023-08-07T01:11:08.167560Z"
    },
    "papermill": {
     "duration": 0.037498,
     "end_time": "2023-08-07T01:11:08.171123",
     "exception": false,
     "start_time": "2023-08-07T01:11:08.133625",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "preds[preds > 0.74] = 1\n",
    "preds[preds < 0.26] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "fcf6d74b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-07T01:11:08.227669Z",
     "iopub.status.busy": "2023-08-07T01:11:08.227242Z",
     "iopub.status.idle": "2023-08-07T01:11:08.249290Z",
     "shell.execute_reply": "2023-08-07T01:11:08.248010Z"
    },
    "papermill": {
     "duration": 0.053439,
     "end_time": "2023-08-07T01:11:08.251853",
     "exception": false,
     "start_time": "2023-08-07T01:11:08.198414",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>class_0</th>\n",
       "      <th>class_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00eed32682bb</td>\n",
       "      <td>0.672654</td>\n",
       "      <td>0.327346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>010ebe33f668</td>\n",
       "      <td>0.672654</td>\n",
       "      <td>0.327346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02fa521e1838</td>\n",
       "      <td>0.672654</td>\n",
       "      <td>0.327346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>040e15f562a2</td>\n",
       "      <td>0.672654</td>\n",
       "      <td>0.327346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>046e85c7cc7f</td>\n",
       "      <td>0.672654</td>\n",
       "      <td>0.327346</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Id   class_0   class_1\n",
       "0  00eed32682bb  0.672654  0.327346\n",
       "1  010ebe33f668  0.672654  0.327346\n",
       "2  02fa521e1838  0.672654  0.327346\n",
       "3  040e15f562a2  0.672654  0.327346\n",
       "4  046e85c7cc7f  0.672654  0.327346"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_df['class_0'] = 1 - preds\n",
    "sample_df['class_1'] = preds\n",
    "\n",
    "sample_df.to_csv('submission.csv', index=False)\n",
    "pd.read_csv('submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "188ff24c",
   "metadata": {
    "papermill": {
     "duration": 0.027193,
     "end_time": "2023-08-07T01:11:08.306587",
     "exception": false,
     "start_time": "2023-08-07T01:11:08.279394",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 67.829502,
   "end_time": "2023-08-07T01:11:09.457589",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-08-07T01:10:01.628087",
   "version": "2.4.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "00f260efe10245e982f2dc36a2574485": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "24029c966820488594748b5494034235": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "45b8002d045c4f1780762e75aa3bdc8d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ab593152987040f1a35e65de510ee497",
       "placeholder": "​",
       "style": "IPY_MODEL_24029c966820488594748b5494034235",
       "value": " 56/56 [00:00&lt;00:00, 692.63it/s]"
      }
     },
     "460d70ae51f6409ba6d7fd151a2d9b3b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_79278fee2aa5495fbecacbee8bff9386",
        "IPY_MODEL_9d1ca17d82c6467187b11d9769827ac8",
        "IPY_MODEL_45b8002d045c4f1780762e75aa3bdc8d"
       ],
       "layout": "IPY_MODEL_8a3cb2a19f4041f5abee7309ef6ccd88"
      }
     },
     "79278fee2aa5495fbecacbee8bff9386": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e5984066ad5c45e9baf1512046dff1e2",
       "placeholder": "​",
       "style": "IPY_MODEL_00f260efe10245e982f2dc36a2574485",
       "value": "100%"
      }
     },
     "8a15d792661c4a0c88027e8a2a63f963": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "8a3cb2a19f4041f5abee7309ef6ccd88": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "98638552cb0440c9b46db89c33685281": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9d1ca17d82c6467187b11d9769827ac8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_98638552cb0440c9b46db89c33685281",
       "max": 56.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_8a15d792661c4a0c88027e8a2a63f963",
       "value": 56.0
      }
     },
     "ab593152987040f1a35e65de510ee497": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e5984066ad5c45e9baf1512046dff1e2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}