Blame for Analysis_code/final_test/final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "import random\n",
    "from collections import Counter\n",
    "import sys\n",
    "sys.path.append('../../../../../../../../mnt/workspace/LightGBM/python-package')\n",
    "from lightgbm import LGBMClassifier\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.inspection import permutation_importance\n",
    "from sklearn.metrics import confusion_matrix,accuracy_score, recall_score, precision_score\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from xgboost import XGBClassifier\n",
    "from warnings import filterwarnings\n",
    "filterwarnings('ignore')\n",
    "import sys\n",
    "sys.path.append('../')\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import optuna\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "from ft_transformer import FTTransformer\n",
    "from resnet_like import ResNetLike\n",
    "from deepgbm import DeepGBM\n",
    "from pytorch_tabnet.tab_model import TabNetClassifier\n",
    "from sklearn.metrics import accuracy_score, f1_score, roc_auc_score, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python 및 Numpy 시드 고정\n",
    "seed = 42\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "# PyTorch 시드 고정\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)  # Multi-GPU 환경에서 동일한 시드 적용\n",
    "\n",
    "# PyTorch 연산의 결정적 모드 설정\n",
    "torch.backends.cudnn.deterministic = True  # 실행마다 동일한 결과를 보장\n",
    "torch.backends.cudnn.benchmark = True  # 성능 최적화를 활성화 (가능한 한 빠른 연산 수행)\n",
    "\n",
    "# 전처리 함수\n",
    "def preprocessing(df):\n",
    "    df = df[df.columns].copy()\n",
    "    df.loc[df['wind_dir']=='정온', 'wind_dir'] = \"0\"\n",
    "    df['wind_dir'] = df['wind_dir'].astype('int')\n",
    "    df['lm_cloudcover'] = df['lm_cloudcover'].astype('int')\n",
    "    df['cloudcover'] = df['cloudcover'].astype('int')\n",
    "    return df\n",
    "\n",
    "# 데이터셋 준비 함수\n",
    "def prepare_dataset(region, data_sample='pure', target='multi', fold=3):\n",
    "\n",
    "    # 데이터 경로 지정\n",
    "    dat_path = f\"../../data/data_for_modeling/{region}_train.csv\"\n",
    "    if data_sample == 'pure':\n",
    "        train_path = dat_path\n",
    "    else:\n",
    "        train_path = f'../../data/data_oversampled/{data_sample}/{data_sample}_{fold}_{region}.csv'\n",
    "        train_path = f'../../data/data_oversampled/{data_sample}/{data_sample}_{fold}_{region}.csv'\n",
    "    test_path = f\"../../data/data_for_modeling/{region}_test.csv\"\n",
    "    drop_col = ['binary_class','multi_class','visi','year']\n",
    "    target_col = f'{target}_class'\n",
    "    \n",
    "    # 데이터 로드\n",
    "    region_dat = preprocessing(pd.read_csv(dat_path, index_col=0))\n",
    "    if data_sample == 'pure':\n",
    "        region_train = region_dat.loc[~region_dat['year'].isin([2021-fold]), :]\n",
    "    else:\n",
    "        region_train = preprocessing(pd.read_csv(train_path))\n",
    "    region_val = region_dat.loc[region_dat['year'].isin([2021-fold]), :]\n",
    "    region_test = preprocessing(pd.read_csv(test_path))\n",
    "\n",
    "    # 컬럼 정렬 (일관성 유지)\n",
    "    common_columns = region_train.columns.to_list()\n",
    "    train_data = region_train[common_columns]\n",
    "    val_data = region_val[common_columns]\n",
    "    test_data = region_test[common_columns]\n",
    "\n",
    "    # 설명변수 & 타겟 분리\n",
    "    X_train = train_data.drop(columns=drop_col)\n",
    "    y_train = train_data[target_col]\n",
    "    X_val = val_data.drop(columns=drop_col)\n",
    "    y_val = val_data[target_col]\n",
    "    X_test = test_data.drop(columns=drop_col)\n",
    "    y_test = test_data[target_col]\n",
    "\n",
    "    # 범주형 & 연속형 변수 분리\n",
    "    categorical_cols = X_train.select_dtypes(include=['object', 'category', 'int64']).columns\n",
    "    numerical_cols = X_train.select_dtypes(include=['float64']).columns\n",
    "\n",
    "    # 범주형 변수 Label Encoding\n",
    "    label_encoders = {}\n",
    "    for col in categorical_cols:\n",
    "        le = LabelEncoder()\n",
    "        le.fit(X_train[col])  # Train 데이터 기준으로 학습\n",
    "        label_encoders[col] = le\n",
    "\n",
    "    # 변환 적용\n",
    "    for col in categorical_cols:\n",
    "        X_train[col] = label_encoders[col].transform(X_train[col])\n",
    "        X_val[col] = label_encoders[col].transform(X_val[col])\n",
    "        X_test[col] = label_encoders[col].transform(X_test[col])\n",
    "\n",
    "    # 연속형 변수 Standard Scaling\n",
    "    scaler = StandardScaler()\n",
    "    scaler.fit(X_train[numerical_cols])  # Train 데이터 기준으로 학습\n",
    "\n",
    "    # 변환 적용\n",
    "    X_train[numerical_cols] = scaler.transform(X_train[numerical_cols])\n",
    "    X_val[numerical_cols] = scaler.transform(X_val[numerical_cols])\n",
    "    X_test[numerical_cols] = scaler.transform(X_test[numerical_cols])\n",
    "\n",
    "    return X_train, X_val, X_test, y_train, y_val, y_test, categorical_cols, numerical_cols\n",
    "\n",
    "# 데이터 변환 및 dataloader 생성 함수\n",
    "def prepare_dataloader(region, data_sample='pure', target='multi', fold=3, random_state=None):\n",
    "\n",
    "    # 데이터 경로 지정\n",
    "    dat_path = f\"../../data/data_for_modeling/{region}_train.csv\"\n",
    "    if data_sample == 'pure':\n",
    "        train_path = dat_path\n",
    "    else:\n",
    "        train_path = f'../../data/data_oversampled/{data_sample}/{data_sample}_{fold}_{region}.csv'\n",
    "        train_path = f'../../data/data_oversampled/{data_sample}/{data_sample}_{fold}_{region}.csv'\n",
    "    test_path = f\"../../data/data_for_modeling/{region}_test.csv\"\n",
    "    drop_col = ['binary_class','multi_class','visi','year']\n",
    "    target_col = f'{target}_class'\n",
    "    \n",
    "    # 데이터 로드\n",
    "    region_dat = preprocessing(pd.read_csv(dat_path, index_col=0))\n",
    "    if data_sample == 'pure':\n",
    "        region_train = region_dat.loc[~region_dat['year'].isin([2021-fold]), :]\n",
    "    else:\n",
    "        region_train = preprocessing(pd.read_csv(train_path))\n",
    "    region_val = region_dat.loc[region_dat['year'].isin([2021-fold]), :]\n",
    "    region_test = preprocessing(pd.read_csv(test_path))\n",
    "\n",
    "    # 컬럼 정렬 (일관성 유지)\n",
    "    common_columns = region_train.columns.to_list()\n",
    "    train_data = region_train[common_columns]\n",
    "    val_data = region_val[common_columns]\n",
    "    test_data = region_test[common_columns]\n",
    "\n",
    "    # 설명변수 & 타겟 분리\n",
    "    X_train = train_data.drop(columns=drop_col)\n",
    "    y_train = train_data[target_col]\n",
    "    X_val = val_data.drop(columns=drop_col)\n",
    "    y_val = val_data[target_col]\n",
    "    X_test = test_data.drop(columns=drop_col)\n",
    "    y_test = test_data[target_col]\n",
    "\n",
    "    # 범주형 & 연속형 변수 분리\n",
    "    categorical_cols = X_train.select_dtypes(include=['object', 'category', 'int64']).columns\n",
    "    numerical_cols = X_train.select_dtypes(include=['float64']).columns\n",
    "\n",
    "    # 범주형 변수 Label Encoding\n",
    "    label_encoders = {}\n",
    "    for col in categorical_cols:\n",
    "        le = LabelEncoder()\n",
    "        le.fit(X_train[col])  # Train 데이터 기준으로 학습\n",
    "        label_encoders[col] = le\n",
    "\n",
    "    # 변환 적용\n",
    "    for col in categorical_cols:\n",
    "        X_train[col] = label_encoders[col].transform(X_train[col])\n",
    "        X_val[col] = label_encoders[col].transform(X_val[col])\n",
    "        X_test[col] = label_encoders[col].transform(X_test[col])\n",
    "\n",
    "    # 연속형 변수 Standard Scaling\n",
    "    scaler = StandardScaler()\n",
    "    scaler.fit(X_train[numerical_cols])  # Train 데이터 기준으로 학습\n",
    "\n",
    "    # 변환 적용\n",
    "    X_train[numerical_cols] = scaler.transform(X_train[numerical_cols])\n",
    "    X_val[numerical_cols] = scaler.transform(X_val[numerical_cols])\n",
    "    X_test[numerical_cols] = scaler.transform(X_test[numerical_cols])\n",
    "\n",
    "    # 연속형 변수와 범주형 변수 분리\n",
    "    X_train_num = torch.tensor(X_train[numerical_cols].values, dtype=torch.float32)\n",
    "    X_train_cat = torch.tensor(X_train[categorical_cols].values, dtype=torch.long)\n",
    "\n",
    "    X_val_num = torch.tensor(X_val[numerical_cols].values, dtype=torch.float32)\n",
    "    X_val_cat = torch.tensor(X_val[categorical_cols].values, dtype=torch.long)\n",
    "\n",
    "    X_test_num = torch.tensor(X_test[numerical_cols].values, dtype=torch.float32)\n",
    "    X_test_cat = torch.tensor(X_test[categorical_cols].values, dtype=torch.long)\n",
    "\n",
    "    # 레이블 변환\n",
    "    if target == \"binary\":\n",
    "        y_train_tensor = torch.tensor(y_train.values, dtype=torch.float32)  # 이진 분류 → float32\n",
    "        y_val_tensor = torch.tensor(y_val.values, dtype=torch.float32)\n",
    "        y_test_tensor = torch.tensor(y_test.values, dtype=torch.float32)\n",
    "    elif target == \"multi\":\n",
    "        y_train_tensor = torch.tensor(y_train.values, dtype=torch.long)  # 다중 분류 → long\n",
    "        y_val_tensor = torch.tensor(y_val.values, dtype=torch.long)\n",
    "        y_test_tensor = torch.tensor(y_test.values, dtype=torch.long)\n",
    "    else:\n",
    "        raise ValueError(\"target must be 'binary' or 'multi'\")\n",
    "\n",
    "    # TensorDataset 생성\n",
    "    train_dataset = TensorDataset(X_train_num, X_train_cat, y_train_tensor)\n",
    "    val_dataset = TensorDataset(X_val_num, X_val_cat, y_val_tensor)\n",
    "    test_dataset = TensorDataset(X_test_num, X_test_cat, y_test_tensor)\n",
    "\n",
    "    # DataLoader 생성\n",
    "    if random_state == None:\n",
    "        train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n",
    "    else:\n",
    "        train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True, generator=torch.Generator().manual_seed(random_state))\n",
    "    val_loader = DataLoader(val_dataset, batch_size=64, shuffle=False)\n",
    "    test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)\n",
    "    \n",
    "    return X_train, categorical_cols, numerical_cols, train_loader, val_loader, test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "# 디바이스 설정 (CUDA 사용 가능하면 GPU로, 아니면 CPU로)\n",
    "import glob\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "def calculate_csi(Y_test, pred):\n",
    "\n",
    "    cm = confusion_matrix(Y_test, pred)  # 변수 이름을 cm으로 변경\n",
    "    # 혼동 행렬에서 H, F, M 추출\n",
    "    H = (cm[0, 0] + cm[1, 1])\n",
    "    \n",
    "    F = (cm[1, 0] + cm[2, 0] +\n",
    "         cm[0, 1] + cm[2, 1])\n",
    "\n",
    "    M = (cm[0, 2] + cm[1, 2])\n",
    "    \n",
    "    # CSI 계산\n",
    "    CSI = H / (H + F + M + 1e-10)\n",
    "    return CSI\n",
    "\n",
    "def csi_metric(y_true, pred):\n",
    "    y_pred_binary = np.argmax(pred, axis=1)\n",
    "    score = calculate_csi(y_true, y_pred_binary)\n",
    "    return 'CSI', score, True  # higher_better=True\n",
    "\n",
    "\n",
    "def eval_metric_csi(y_true, pred_prob):\n",
    "\n",
    "    pred = np.argmax(pred_prob, axis=1)\n",
    "    y_true = y_true\n",
    "    y_pred = pred\n",
    "    csi = calculate_csi(y_true, y_pred)\n",
    "    return -1*csi\n",
    "\n",
    "\n",
    "from sklearn.metrics import matthews_corrcoef, accuracy_score\n",
    "\n",
    "def multiclass_mcc(y_val, y_pred):\n",
    "    \"\"\"\n",
    "    다중 분류에서도 sklearn의 matthews_corrcoef를 그대로 사용할 수 있음.\n",
    "    \"\"\"\n",
    "    return matthews_corrcoef(y_val, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "# 디바이스 설정 (CUDA 사용 가능하면 GPU로, 아니면 CPU로)\n",
    "import glob\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "def calculate_csi(Y_test, pred):\n",
    "\n",
    "    cm = confusion_matrix(Y_test, pred)  # 변수 이름을 cm으로 변경\n",
    "    # 혼동 행렬에서 H, F, M 추출\n",
    "    H = (cm[0, 0] + cm[1, 1])\n",
    "    \n",
    "    F = (cm[1, 0] + cm[2, 0] +\n",
    "         cm[0, 1] + cm[2, 1])\n",
    "\n",
    "    M = (cm[0, 2] + cm[1, 2])\n",
    "    \n",
    "    # CSI 계산\n",
    "    CSI = H / (H + F + M + 1e-10)\n",
    "    return CSI\n",
    "\n",
    "# Soft Voting 앙상블\n",
    "def get_proba(region, model_choose, data_sample, fold, target='multi'):\n",
    "    _, _, _, _,val_loader , test_loader = prepare_dataloader(region=region, data_sample=data_sample, target=target,fold=fold ,random_state=120)\n",
    "\n",
    "    folder_path = f'../save_model/{model_choose}/{data_sample}'\n",
    "    model_paths = [path for path in glob.glob(f'{folder_path}/*.pth') if f'{region}' in path]\n",
    "\n",
    "    model = torch.load(model_paths[fold-1], weights_only=False).to(device)\n",
    "    model.eval()\n",
    "\n",
    "    test_preds = []\n",
    "\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for x_num_batch, x_cat_batch, _ in test_loader:\n",
    "            output = model(x_num_batch.to(device), x_cat_batch.to(device))\n",
    "            output = torch.softmax(output, dim=1)\n",
    "            test_preds.extend(output.cpu().numpy())\n",
    "\n",
    "\n",
    "    return test_preds\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_seoul = pd.read_csv(\"../../data/data_for_modeling/seoul_train.csv\")\n",
    "df_seoul_test = pd.read_csv(\"../../data/data_for_modeling/seoul_test.csv\")\n",
    "\n",
    "df_busan = pd.read_csv(\"../../data/data_for_modeling/busan_train.csv\")\n",
    "df_busan_test = pd.read_csv(\"../../data/data_for_modeling/busan_test.csv\")\n",
    "\n",
    "df_daegu = pd.read_csv(\"../../data/data_for_modeling/daegu_train.csv\")\n",
    "df_daegu_test = pd.read_csv(\"../../data/data_for_modeling/daegu_test.csv\")\n",
    "\n",
    "df_daejeon = pd.read_csv(\"../../data/data_for_modeling/daejeon_train.csv\")\n",
    "df_daejeon_test = pd.read_csv(\"../../data/data_for_modeling/daejeon_test.csv\")\n",
    "\n",
    "df_incheon = pd.read_csv(\"../../data/data_for_modeling/incheon_train.csv\")\n",
    "df_incheon_test = pd.read_csv(\"../../data/data_for_modeling/incheon_test.csv\")\n",
    "\n",
    "df_gwangju = pd.read_csv(\"../../data/data_for_modeling/gwangju_train.csv\")\n",
    "df_gwangju_test = pd.read_csv(\"../../data/data_for_modeling/gwangju_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessing_df(df):\n",
    "    df = df[df.columns].copy()\n",
    "    df['year'] = df['year'].astype('int')\n",
    "    df['month'] = df['month'].astype('int')\n",
    "    df['hour'] = df['hour'].astype('int')\n",
    "    df['binary_class'] = df['binary_class'].astype('int')\n",
    "    df['multi_class'] = df['multi_class'].astype('int')\n",
    "\n",
    "    df.loc[df['wind_dir']=='정온', 'wind_dir'] = \"0\"\n",
    "    df['wind_dir'] = df['wind_dir'].astype('int')\n",
    "    df= df[['temp_C', 'precip_mm', 'wind_speed', 'wind_dir', 'hm',\n",
    "       'vap_pressure', 'dewpoint_C', 'loc_pressure', 'sea_pressure',\n",
    "       'solarRad', 'snow_cm', 'cloudcover', 'lm_cloudcover', 'low_cloudbase',\n",
    "       'groundtemp', 'O3', 'NO2', 'PM10', 'PM25', 'year',\n",
    "       'month', 'hour', 'ground_temp - temp_C', 'hour_sin', 'hour_cos',\n",
    "       'month_sin', 'month_cos','multi_class']]\n",
    "    return df\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_seoul_test= preprocessing_df(df_seoul_test).copy()\n",
    "df_busan_test= preprocessing_df(df_busan_test).copy()\n",
    "df_daegu_test= preprocessing_df(df_daegu_test).copy()\n",
    "df_gwangju_test= preprocessing_df(df_gwangju_test).copy()\n",
    "df_daejeon_test= preprocessing_df(df_daejeon_test).copy()\n",
    "df_incheon_test= preprocessing_df(df_incheon_test).copy()\n",
    "\n",
    "df_seoul= preprocessing_df(df_seoul).copy()\n",
    "df_busan= preprocessing_df(df_busan).copy()\n",
    "df_daegu= preprocessing_df(df_daegu).copy()\n",
    "df_gwangju= preprocessing_df(df_gwangju).copy()\n",
    "df_daejeon= preprocessing_df(df_daejeon).copy()\n",
    "df_incheon= preprocessing_df(df_incheon).copy()\n",
    "\n",
    "df_seoul_test.drop(columns=['year'], inplace=True)\n",
    "df_busan_test.drop(columns=['year'], inplace=True)\n",
    "df_daegu_test.drop(columns=['year'], inplace=True)\n",
    "df_daejeon_test.drop(columns=['year'], inplace=True)\n",
    "df_incheon_test.drop(columns=['year'], inplace=True)\n",
    "df_gwangju_test.drop(columns=['year'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "\n",
    "lgb_seoul= joblib.load('../save_model/LGB_optima/lgb_seoul_smote.pkl')\n",
    "lgb_busan= joblib.load('../save_model/LGB_optima/lgb_busan_smote.pkl')\n",
    "lgb_incheon= joblib.load('../save_model/LGB_optima/lgb_incheon_smote.pkl')\n",
    "lgb_daegu= joblib.load('../save_model/LGB_optima/lgb_daegu_smote.pkl')\n",
    "lgb_daejeon= joblib.load('../save_model/LGB_optima/lgb_daejeon_smote.pkl')\n",
    "lgb_gwangju= joblib.load('../save_model/LGB_optima/lgb_gwangju_smote.pkl')\n",
    "\n",
    "xgb_seoul= joblib.load('../save_model/XGB_optima/xgb_seoul_smote.pkl')\n",
    "xgb_busan= joblib.load('../save_model/XGB_optima/xgb_busan_ctgan20000.pkl')\n",
    "xgb_incheon= joblib.load('../save_model/XGB_optima/xgb_incheon_smote.pkl')\n",
    "xgb_daegu= joblib.load('../save_model/XGB_optima/xgb_daegu_smote.pkl')\n",
    "xgb_daejeon= joblib.load('../save_model/XGB_optima/xgb_daejeon_smote.pkl')\n",
    "xgb_gwangju= joblib.load('../save_model/XGB_optima/xgb_gwangju_smote.pkl')\n",
    "\n",
    "lgb_seoul_1= lgb_seoul[0]\n",
    "lgb_seoul_2= lgb_seoul[1]\n",
    "lgb_seoul_3= lgb_seoul[2]\n",
    "\n",
    "lgb_busan_1= lgb_busan[0]\n",
    "lgb_busan_2= lgb_busan[1]\n",
    "lgb_busan_3= lgb_busan[2]\n",
    "\n",
    "lgb_incheon_1= lgb_incheon[0]\n",
    "lgb_incheon_2= lgb_incheon[1]\n",
    "lgb_incheon_3= lgb_incheon[2]\n",
    "\n",
    "lgb_daegu_1= lgb_daegu[0]\n",
    "lgb_daegu_2= lgb_daegu[1]\n",
    "lgb_daegu_3= lgb_daegu[2]\n",
    "\n",
    "lgb_daejeon_1= lgb_daejeon[0]\n",
    "lgb_daejeon_2= lgb_daejeon[1]\n",
    "lgb_daejeon_3= lgb_daejeon[2]\n",
    "\n",
    "lgb_gwangju_1= lgb_gwangju[0]\n",
    "lgb_gwangju_2= lgb_gwangju[1]\n",
    "lgb_gwangju_3= lgb_gwangju[2]\n",
    "\n",
    "\n",
    "xgb_seoul_1= xgb_seoul[0]\n",
    "xgb_seoul_2= xgb_seoul[1]\n",
    "xgb_seoul_3= xgb_seoul[2]\n",
    "\n",
    "xgb_busan_1= xgb_busan[0]\n",
    "xgb_busan_2= xgb_busan[1]\n",
    "xgb_busan_3= xgb_busan[2]\n",
    "\n",
    "xgb_incheon_1= xgb_incheon[0]\n",
    "xgb_incheon_2= xgb_incheon[1]\n",
    "xgb_incheon_3= xgb_incheon[2]\n",
    "\n",
    "xgb_daegu_1= xgb_daegu[0]\n",
    "xgb_daegu_2= xgb_daegu[1]\n",
    "xgb_daegu_3= xgb_daegu[2]\n",
    "\n",
    "xgb_daejeon_1= xgb_daejeon[0]\n",
    "xgb_daejeon_2= xgb_daejeon[1]\n",
    "xgb_daejeon_3= xgb_daejeon[2]\n",
    "\n",
    "xgb_gwangju_1= xgb_gwangju[0]\n",
    "xgb_gwangju_2= xgb_gwangju[1]\n",
    "xgb_gwangju_3= xgb_gwangju[2]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Soft Voting**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **서울**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "voting = []\n",
    "mcc = []\n",
    "accuracy = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['temp_C', 'precip_mm', 'wind_speed', 'wind_dir', 'hm', 'vap_pressure',\n",
       "       'dewpoint_C', 'loc_pressure', 'sea_pressure', 'solarRad', 'snow_cm',\n",
       "       'cloudcover', 'lm_cloudcover', 'low_cloudbase', 'groundtemp', 'O3',\n",
       "       'NO2', 'PM10', 'PM25', 'month', 'hour', 'ground_temp - temp_C',\n",
       "       'hour_sin', 'hour_cos', 'month_sin', 'month_cos', 'multi_class'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_seoul_test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CSI score of soft(test) : 0.3248062015503624\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import mode  # 최다 득표수 계산을 위한 mode 함수\n",
    "\n",
    "probas = []\n",
    "\n",
    "# 1 Fold\n",
    "test_preds  = get_proba('seoul', 'deepgbm', 'ctgan20000', 1)\n",
    "probas.append(test_preds)\n",
    "test_preds  = get_proba('seoul', 'resnet_like', 'smote', 1)\n",
    "probas.append(test_preds)\n",
    "# probas.append(xgb_seoul_1.predict_proba(df_seoul_test.iloc[:,:-1]))\n",
    "\n",
    "# 2 Fold\n",
    "test_preds  = get_proba('seoul', 'deepgbm', 'ctgan20000', 2)\n",
    "probas.append(test_preds)\n",
    "test_preds  = get_proba('seoul', 'resnet_like', 'smote', 2)\n",
    "probas.append(test_preds)\n",
    "# probas.append(xgb_seoul_2.predict_proba(df_seoul_test.iloc[:,:-1]))\n",
    "\n",
    "# 3 Fold\n",
    "test_preds  = get_proba('seoul', 'deepgbm', 'ctgan20000', 3)\n",
    "probas.append(test_preds)\n",
    "test_preds  = get_proba('seoul', 'resnet_like', 'smote', 3)\n",
    "probas.append(test_preds)\n",
    "# probas.append(xgb_seoul_3.predict_proba(df_seoul_test.iloc[:,:-1]))\n",
    "\n",
    "voting.append(calculate_csi(df_seoul_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "mcc.append(multiclass_mcc(df_seoul_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "accuracy.append(accuracy_score(df_seoul_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "\n",
    "\n",
    "print(\"CSI score of soft(test) :\", calculate_csi(df_seoul_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   2,   11,    0],\n",
       "       [   6,  417,   58],\n",
       "       [   7,  789, 7470]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(df_seoul_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **부산**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CSI score of soft(test) : 0.46608315098458075\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import mode  # 최다 득표수 계산을 위한 mode 함수\n",
    "\n",
    "probas = []\n",
    "\n",
    "# 1 Fold\n",
    "test_preds  = get_proba('busan', 'deepgbm', 'pure', 1)\n",
    "probas.append(test_preds)\n",
    "test_preds  = get_proba('busan', 'resnet_like', 'ctgan10000', 1)\n",
    "probas.append(test_preds)\n",
    "\n",
    "\n",
    "# 2 Fold\n",
    "test_preds  = get_proba('busan', 'deepgbm', 'pure', 2)\n",
    "probas.append(test_preds)\n",
    "test_preds  = get_proba('busan', 'resnet_like', 'ctgan10000', 2)\n",
    "probas.append(test_preds)\n",
    "\n",
    "\n",
    "# 3 Fold\n",
    "test_preds  = get_proba('busan', 'deepgbm', 'pure', 3)\n",
    "probas.append(test_preds)\n",
    "test_preds  = get_proba('busan', 'resnet_like', 'ctgan10000', 3)\n",
    "probas.append(test_preds)\n",
    "\n",
    "\n",
    "voting.append(calculate_csi(df_busan_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "mcc.append(multiclass_mcc(df_busan_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "accuracy.append(accuracy_score(df_busan_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "\n",
    "print(\"CSI score of soft(test) :\", calculate_csi(df_busan_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  11,   13,    0],\n",
       "       [  11,  202,   68],\n",
       "       [   2,  150, 8303]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(df_busan_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **인천**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CSI score of hard(test) : 0.572269457161506\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import mode  # 최다 득표수 계산을 위한 mode 함수\n",
    "\n",
    "\n",
    "# 1 Fold\n",
    "probas = []\n",
    "test_preds = get_proba('incheon', 'deepgbm', 'pure', 1)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('incheon', 'resnet_like', 'smote', 1)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('incheon', 'ft_transformer', 'pure', 1)\n",
    "probas.append(test_preds)\n",
    "\n",
    "probas.append(lgb_incheon_1.predict_proba(df_incheon_test.iloc[:,:-1]))\n",
    "probas.append(xgb_incheon_1.predict_proba(df_incheon_test.iloc[:,:-1]))\n",
    "\n",
    "# 2 Fold\n",
    "test_preds = get_proba('incheon', 'deepgbm', 'pure', 2)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('incheon', 'resnet_like', 'smote', 2)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('incheon', 'ft_transformer', 'pure', 2)\n",
    "probas.append(test_preds)\n",
    "\n",
    "probas.append(lgb_incheon_2.predict_proba(df_incheon_test.iloc[:,:-1]))\n",
    "probas.append(xgb_incheon_2.predict_proba(df_incheon_test.iloc[:,:-1]))\n",
    "\n",
    "# 3 Fold\n",
    "test_preds = get_proba('incheon', 'deepgbm', 'pure', 3)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('incheon', 'resnet_like', 'smote', 3)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('incheon', 'ft_transformer', 'pure', 3)\n",
    "probas.append(test_preds)\n",
    "\n",
    "probas.append(lgb_incheon_3.predict_proba(df_incheon_test.iloc[:,:-1]))\n",
    "probas.append(xgb_incheon_3.predict_proba(df_incheon_test.iloc[:,:-1]))\n",
    "\n",
    "\n",
    "\n",
    "voting.append(calculate_csi(df_incheon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "mcc.append(multiclass_mcc(df_incheon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "accuracy.append(accuracy_score(df_incheon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "\n",
    "\n",
    "print(\"CSI score of hard(test) :\", calculate_csi(df_incheon_test.iloc[:,-1],mode(np.argmax(probas, axis=2), axis=0).mode[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  87,   74,   21],\n",
       "       [  22,  788,  395],\n",
       "       [   2,  140, 7231]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(df_incheon_test.iloc[:,-1],mode(np.argmax(probas, axis=2), axis=0).mode[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **대구**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CSI score of soft(test) : 0.2852112676055334\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import mode  # 최다 득표수 계산을 위한 mode 함수\n",
    "\n",
    "probas= []\n",
    "\n",
    "# 1 Fold\n",
    "test_preds = get_proba('daegu', 'deepgbm', 'smote', 1)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('daegu', 'ft_transformer', 'pure', 1)\n",
    "probas.append(test_preds)\n",
    "\n",
    "# 2 Fold\n",
    "test_preds = get_proba('daegu', 'deepgbm', 'smote', 2)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('daegu', 'ft_transformer', 'pure', 2)\n",
    "probas.append(test_preds)\n",
    "\n",
    "# 3 Fold\n",
    "test_preds = get_proba('daegu', 'deepgbm', 'smote', 3)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('daegu', 'ft_transformer', 'pure', 3)\n",
    "probas.append(test_preds)\n",
    "\n",
    "voting.append(calculate_csi(df_daegu_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "mcc.append(multiclass_mcc(df_daegu_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "accuracy.append(accuracy_score(df_daegu_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "\n",
    "print(\"CSI score of soft(test) :\", calculate_csi(df_daegu_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   1,    0,    0],\n",
       "       [   1,   80,   47],\n",
       "       [   2,  153, 8476]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(df_daegu_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **대전**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CSI score of soft(test) : 0.31884057971011603\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import mode  # 최다 득표수 계산을 위한 mode 함수\n",
    "\n",
    "probas = []\n",
    "\n",
    "# 1 Fold\n",
    "test_preds = get_proba('daejeon', 'deepgbm', 'pure', 1)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('daejeon', 'ft_transformer', 'pure', 1)\n",
    "probas.append(test_preds)\n",
    "\n",
    "\n",
    "# 2 Fold\n",
    "test_preds = get_proba('daejeon', 'deepgbm', 'pure', 2)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('daejeon', 'ft_transformer', 'pure', 2)\n",
    "probas.append(test_preds)\n",
    "\n",
    "\n",
    "# 3 Fold\n",
    "test_preds = get_proba('daejeon', 'deepgbm', 'pure', 3)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('daejeon', 'ft_transformer', 'pure', 3)\n",
    "probas.append(test_preds)\n",
    "\n",
    "voting.append(calculate_csi(df_daejeon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "mcc.append(multiclass_mcc(df_daejeon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "accuracy.append(accuracy_score(df_daejeon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "\n",
    "\n",
    "print(\"CSI score of soft(test) :\", calculate_csi(df_daejeon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  15,   23,   15],\n",
       "       [  10,  337,  271],\n",
       "       [   0,  433, 7656]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(df_daejeon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **광주**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CSI score of soft(test) : 0.4759725400457121\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import mode  # 최다 득표수 계산을 위한 mode 함수\n",
    "\n",
    "probas = []\n",
    "\n",
    "# 1 Fold\n",
    "test_preds = get_proba('gwangju', 'deepgbm', 'pure', 1)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('gwangju', 'ft_transformer', 'pure', 1)\n",
    "probas.append(test_preds)\n",
    "\n",
    "\n",
    "# 2 Fold\n",
    "test_preds = get_proba('gwangju', 'deepgbm', 'pure', 2)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('gwangju', 'ft_transformer', 'pure', 2)\n",
    "probas.append(test_preds)\n",
    "\n",
    "\n",
    "# 3 Fold\n",
    "test_preds = get_proba('gwangju', 'deepgbm', 'pure', 3)\n",
    "probas.append(test_preds)\n",
    "test_preds = get_proba('gwangju', 'ft_transformer', 'pure', 3)\n",
    "probas.append(test_preds)\n",
    "\n",
    "voting.append(calculate_csi(df_gwangju_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "mcc.append(multiclass_mcc(df_gwangju_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "accuracy.append(accuracy_score(df_gwangju_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n",
    "\n",
    "\n",
    "print(\"CSI score of soft(test) :\", calculate_csi(df_gwangju_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  10,   12,    8],\n",
       "       [   2,  406,  235],\n",
       "       [   0,  201, 7886]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(df_gwangju_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAHDCAYAAAATEUquAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDfUlEQVR4nO3deVwVdf///+cBZBfEDcgQUtw33EMvxQpDLcsrKysL5GvqpWJeF5VFlmiLWJbppaZZH9PcMs1WU0vUSrM0FZdCxd00cClFUKHg/fujH+fqCDhiynF53G+3c6t5z3tmXnPOeM55MjPvYzPGGAEAAAAASuXi7AIAAAAA4EpHcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAXFPWr1+vdu3aycfHRzabTWlpac4uSSNHjpTNZnN2GeVq1apVstlsWrVqVZmXnTFjhmw2m/bt23fJ6wKAi0VwAoDLaPfu3RowYIBq1aolT09P+fn5qX379powYYLOnDlj75efn68JEyaoefPm8vPzU6VKldSoUSP1799f27dvt/cr+kL5ww8/WG573759io+PV+3ateXp6amgoCB17NhRycnJl2VfrwS///677rvvPv366696/fXXNWvWLIWGhhbr99hjj8lms2nXrl2lrmv48OGy2WzasmXLBW379OnTGjly5EUFhcupT58+stls8vPzczjmimRkZMhms8lms+nVV191QoUAcHUgOAHAZbJ48WI1adJE77//vrp3766JEycqJSVFNWvW1JNPPqmhQ4fa+/bs2VOPP/64GjdurDFjxmjUqFHq2LGjlixZou+++67M2961a5eaN2+uZcuW6cEHH9SkSZM0ePBgValSRS+//PKl3M0ryu7du7V//3498cQT6t+/vx5++GEFBAQU69e7d29J0ty5c0td17x589SkSRM1bdr0grZ9+vRpjRo1qsTg9Oyzz5YYWsqLm5ubTp8+rU8//bTYvDlz5sjT09MJVQHA1cXN2QUAwLVo7969euCBBxQaGqoVK1YoODjYPm/w4MHatWuXFi9eLOnPS8s+++wzvfTSS3rmmWcc1jNp0iSdOHGizNt//fXXlZOTo7S0tGJnXI4cOVL2HfobcnNz5ePjUy7bKtq3SpUqnbdf27ZtFR4ernnz5mnEiBHF5q9du1Z79+7VmDFjLkldbm5ucnNz3keuh4eH2rdvr3nz5un+++93mDd37lzdcccd+uCDD5xUHQBcHTjjBACXwSuvvKKcnBz93//9n0NoKhIeHm4/47R7925JUvv27Yv1c3V1VZUqVcq8/d27d+vGG28s8TK16tWrF2tbsmSJoqKiVLFiRfn5+al169bFzsYsWLBALVu2lJeXl6pWraqHH35Yhw4dcujTp08f+fr6avfu3erWrZsqVqxoP7tTWFio8ePHq1GjRvL09FRgYKAGDBig33777YL2acWKFerQoYN8fHxUqVIl3X333UpPT3fYdlRUlCTpvvvuk81mU6dOnUpdX+/evbV9+3Zt3Lix2Ly5c+fKZrPpwQcflPRnIOvbt68CAwPl6empZs2aaebMmfb++/btU7Vq1SRJo0aNsl/6NnLkSEkl3+Nks9mUkJCgjz76SI0bN5aHh4caNWqkpUuXFqtn1apVatWqlTw9PVW7dm29+eabZb5v6qGHHtKSJUscgvj69euVkZGhhx56qMRl9uzZo/vuu0+VK1eWt7e3br75Znvg/6uff/5ZPXr0kI+Pj6pXr67//Oc/ysvLK3Gd33//vbp06SJ/f395e3srKipKa9asueD9AABnITgBwGXw6aefqlatWmrXrp1l36JwM2fOHP3xxx+XZPuhoaE6ePCgVqxYYdl3xowZuuOOO/Trr78qKSlJY8aMUUREhMMX+BkzZuj++++Xq6urUlJS1K9fPy1atEj/+Mc/ip0R++OPPxQTE6Pq1avr1VdfVc+ePSVJAwYM0JNPPmm/xys+Pl5z5sxRTEyMfv/99/PWuHz5csXExOjIkSMaOXKkEhMT9e2336p9+/b2AQQGDBhgP2P32GOPadasWRo+fHip6yztcr2CggK9//776tChg2rWrKkzZ86oU6dOmjVrlnr37q2xY8fK399fffr00YQJEyRJ1apV05QpUyRJ//znPzVr1izNmjVL99xzz3n3a/Xq1Ro0aJAeeOABvfLKKzp79qx69uyp48eP2/ts2rRJXbp00fHjxzVq1Cj17dtXzz//vD766KPzrvtc99xzj2w2mxYtWmRvmzt3rurXr68WLVoU65+VlaV27dpp2bJlGjRokF566SWdPXtWd911lz788EN7vzNnzui2227TsmXLlJCQoOHDh+ubb77RsGHDiq1zxYoV6tixo7Kzs5WcnKzRo0frxIkTuvXWW7Vu3boy7Q8AlDsDALikTp48aSSZu++++4L6FxYWmqioKCPJBAYGmgcffNBMnjzZ7N+/v1jfd955x0gy69evP+86t23bZry8vIwkExERYYYOHWo++ugjk5ub69DvxIkTpmLFiqZt27bmzJkzxeoyxpj8/HxTvXp107hxY4c+n332mZFkRowYYW+Li4szkszTTz/tsK5vvvnGSDJz5sxxaF+6dGmJ7eeKiIgw1atXN8ePH7e3bd682bi4uJjY2Fh728qVK40ks2DBgvOur0jr1q3NjTfeaAoKCorV9OabbxpjjBk/fryRZGbPnm3vk5+fbyIjI42vr6/Jzs42xhhz9OhRI8kkJycX205ycrI59yNXknF3dze7du1y2CdJZuLEifa27t27G29vb3Po0CF7W0ZGhnFzcyu2zpLExcUZHx8fY4wx9957r7ntttuMMcYUFBSYoKAgM2rUKLN3714jyYwdO9a+3L///W8jyXzzzTf2tlOnTpmbbrrJhIWF2Z+zoufn/ffft/fLzc014eHhRpJZuXKlMebP46lOnTomJibGfmwZY8zp06fNTTfdZDp37mxvKzrO9+7da7l/AFBeOOMEAJdYdna2JKlixYoX1N9ms2nZsmV68cUXFRAQoHnz5mnw4MEKDQ1Vr169Luoep0aNGiktLU0PP/yw9u3bpwkTJqhHjx4KDAzUW2+9Ze/35Zdf6tSpU3r66aeLDRBQdBnYDz/8oCNHjmjQoEEOfe644w7Vr1+/xEu3Bg4c6DC9YMEC+fv7q3Pnzjp27Jj90bJlS/n6+mrlypWl7ssvv/yitLQ09enTR5UrV7a3N23aVJ07d9bnn39etifnLx5++GH9/PPP+vrrr+1tc+fOlbu7u+677z5J0ueff66goCD7ZXuSVKFCBT322GPKycnRV199ddHbj46OVu3ate3TTZs2lZ+fn/bs2SPpz7Nfy5cvV48ePXTDDTfY+4WHh6tr165l3t5DDz2kVatWKTMzUytWrFBmZmapl+l9/vnnatOmjf7xj3/Y23x9fdW/f3/t27dPP/30k71fcHCw7r33Xns/b29v9e/f32F9aWlp9ssCjx8/bj8GcnNzddttt+nrr79WYWFhmfcJAMoLwQkALjE/Pz9J0qlTpy54GQ8PDw0fPlzp6ek6fPiw5s2bp5tvvlnvv/++EhISLqqOunXratasWTp27Ji2bNmi0aNHy83NTf3799fy5csl/e/+qsaNG5e6nv3790uS6tWrV2xe/fr17fOLuLm56cYbb3Roy8jI0MmTJ1W9enVVq1bN4ZGTk3PeASvOt/0GDRrYv3xfjAceeECurq72y/XOnj2rDz/8UF27drWPxrd//37VqVNHLi6OH5kNGjRwqO9i1KxZs1hbQECA/b6vI0eO6MyZMwoPDy/Wr6Q2K0X3nc2fP19z5sxR69atS13P/v37S33Oi+YX/Tc8PLzY/VbnLpuRkSFJiouLK3YMvP3228rLy9PJkyfLvE8AUF4YVQ8ALjE/Pz/dcMMN2rZt20UtHxwcrAceeEA9e/ZUo0aN9P7772vGjBkXPSqbq6urmjRpoiZNmigyMlK33HKL5syZo+jo6ItanxUPD49iIaOwsFDVq1fXnDlzSlymaGCF8la9enV17txZH3zwgSZPnqxPP/1Up06dst//dLm5urqW2G6MuSzb8/Dw0D333KOZM2dqz5499sErykPR2aSxY8cqIiKixD6+vr7lVg8AlBXBCQAugzvvvFPTpk3T2rVrFRkZeVHrqFChgpo2baqMjAwdO3ZMQUFBf7uuVq1aSfrz8jdJ9svEtm3bVuqZh6LBK3bs2KFbb73VYd6OHTtKHLnvXLVr19by5cvVvn17eXl5lanmv27/XNu3b1fVqlX/1nDnvXv31tKlS7VkyRLNnTtXfn5+6t69u8P2t2zZosLCQodAWPTDxEX1lWWEuwtVvXp1eXp6lvhDvef78d7zeeihhzR9+nS5uLjogQceKLVfaGhoqc950fyi/27btk3GGIfn4Nxli441Pz+/yxbaAeBy4lI9ALgMhg0bJh8fHz366KPKysoqNn/37t32EdkyMjJ04MCBYn1OnDihtWvXKiAgoMxnZL755psSR6oruh+o6DKq22+/XRUrVlRKSorOnj3r0LforEerVq1UvXp1TZ061WGI6SVLlig9PV133HGHZT3333+/CgoK9MILLxSb98cff5z3Pq7g4GBFRERo5syZDv22bdumL774Qt26dbPc/vn06NFD3t7eeuONN7RkyRLdc889DvdydevWTZmZmZo/f75DzRMnTpSvr699CHRvb29Juqh70krj6uqq6OhoffTRRzp8+LC9fdeuXVqyZMlFrfOWW27RCy+8oEmTJp03jHfr1k3r1q3T2rVr7W25ubmaNm2awsLC1LBhQ3u/w4cPa+HChfZ+p0+f1rRp0xzW17JlS9WuXVuvvvqqcnJyim3v6NGjF7U/AFBeOOMEAJdB7dq1NXfuXPXq1UsNGjRQbGysGjdurPz8fH377bdasGCB+vTpI0navHmzHnroIXXt2lUdOnRQ5cqVdejQIc2cOVOHDx/W+PHjS72kqzQvv/yyNmzYoHvuuUdNmzaVJG3cuFHvvvuuKleurH//+9+S/vzr/+uvv65HH31UrVu31kMPPaSAgABt3rxZp0+f1syZM1WhQgW9/PLLio+PV1RUlB588EFlZWVpwoQJCgsL03/+8x/LeqKiojRgwAClpKQoLS1Nt99+uypUqKCMjAwtWLBAEyZMcBhc4Fxjx45V165dFRkZqb59++rMmTOaOHGi/P39//blZr6+vurRo4f9PqdzL9Pr37+/3nzzTfXp00cbNmxQWFiYFi5cqDVr1mj8+PH2QUC8vLzUsGFDzZ8/X3Xr1lXlypXVuHHj894/diFGjhypL774Qu3bt9fAgQNVUFCgSZMmqXHjxkpLSyvz+lxcXPTss89a9nv66ac1b948de3aVY899pgqV66smTNnau/evfrggw/sZ9/69eunSZMmKTY2Vhs2bFBwcLBmzZplD5J/3e7bb7+trl27qlGjRoqPj1eNGjV06NAhrVy5Un5+fvr000/LvD8AUG6cPKofAFzTdu7cafr162fCwsKMu7u7qVixomnfvr2ZOHGiOXv2rDHGmKysLDNmzBgTFRVlgoODjZubmwkICDC33nqrWbhwocP6LnQ48jVr1pjBgwebxo0bG39/f1OhQgVTs2ZN06dPH7N79+5i/T/55BPTrl074+XlZfz8/EybNm3MvHnzHPrMnz/fNG/e3Hh4eJjKlSub3r17m59//tmhz1+Hvi7JtGnTTMuWLY2Xl5epWLGiadKkiRk2bJg5fPjweffHGGOWL19u2rdvb6+xe/fu5qeffnLoU9bhyIssXrzYSDLBwcEOQ5MXycrKMvHx8aZq1arG3d3dNGnSxLzzzjvF+n377bemZcuWxt3d3WFo8tKGIx88eHCxdYSGhpq4uDiHttTUVNO8eXPj7u5uateubd5++23z+OOPG09PT8t9s3pNjDElDkdujDG7d+829957r6lUqZLx9PQ0bdq0MZ999lmx5ffv32/uuusu4+3tbapWrWqGDh1qH9a9aDjyIps2bTL33HOPqVKlivHw8DChoaHm/vvvN6mpqfY+DEcO4EpkM+Yy3YEKAAAumx49eujHH3+0j1YHALi8uMcJAIAr3JkzZxymMzIy9Pnnn6tTp07OKQgArkOccQIA4AoXHBysPn36qFatWtq/f7+mTJmivLw8bdq0SXXq1HF2eQBwXWBwCAAArnBdunTRvHnzlJmZKQ8PD0VGRmr06NGEJgAoR5xxAgAAAAAL3OMEAAAAABYITgAAAABgwen3OE2ePFljx45VZmammjVrpokTJ6pNmzal9j9x4oSGDx+uRYsW6ddff1VoaKjGjx9/wb8cX1hYqMOHD6tixYqy2WyXajcAAAAAXGWMMTp16pRuuOEG+w97l8apwWn+/PlKTEzU1KlT1bZtW40fP14xMTHasWOHqlevXqx/fn6+OnfurOrVq2vhwoWqUaOG9u/fr0qVKl3wNg8fPqyQkJBLuBcAAAAArmYHDx7UjTfeeN4+Th0com3btmrdurUmTZok6c+zQSEhIRoyZIiefvrpYv2nTp2qsWPHavv27apQocJFbfPkyZOqVKmSDh48KD8/v79VPwAAAICrV3Z2tkJCQnTixAn5+/uft6/Tzjjl5+drw4YNSkpKsre5uLgoOjpaa9euLXGZTz75RJGRkRo8eLA+/vhjVatWTQ899JCeeuopubq6XtB2iy7P8/PzIzgBAAAAuKBbeJwWnI4dO6aCggIFBgY6tAcGBmr79u0lLrNnzx6tWLFCvXv31ueff65du3Zp0KBB+v3335WcnFziMnl5ecrLy7NPZ2dnX7qdAAAAAHBduKpG1SssLFT16tU1bdo0tWzZUr169dLw4cM1derUUpdJSUmRv7+//cH9TQAAAADKymnBqWrVqnJ1dVVWVpZDe1ZWloKCgkpcJjg4WHXr1nW4LK9BgwbKzMxUfn5+icskJSXp5MmT9sfBgwcv3U4AAAAAuC44LTi5u7urZcuWSk1NtbcVFhYqNTVVkZGRJS7Tvn177dq1S4WFhfa2nTt3Kjg4WO7u7iUu4+HhYb+fifuaAAAAAFwMp16ql5iYqLfeekszZ85Uenq6Bg4cqNzcXMXHx0uSYmNjHQaPGDhwoH799VcNHTpUO3fu1OLFizV69GgNHjzYWbsAAAAA4Drg1N9x6tWrl44ePaoRI0YoMzNTERERWrp0qX3AiAMHDjj8EFVISIiWLVum//znP2ratKlq1KihoUOH6qmnnnLWLgAAAAC4Djj1d5ycITs7W/7+/jp58iSX7QEAAADXsbJkg6tqVD0AAAAAcAaCEwAAAABYIDgBAAAAgAWCEwAAAABYIDgBAAAAgAWCEwAAAABYIDgBAAAAgAWCEwAAAABYIDgBAAAAgAU3ZxcAAFeT9hPbO7sElGLNkDXOLgEAcA3jjBMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWLgigtPkyZMVFhYmT09PtW3bVuvWrSu174wZM2Sz2Rwenp6e5VgtAAAAgOuN04PT/PnzlZiYqOTkZG3cuFHNmjVTTEyMjhw5Uuoyfn5++uWXX+yP/fv3l2PFAAAAAK43Tg9O48aNU79+/RQfH6+GDRtq6tSp8vb21vTp00tdxmazKSgoyP4IDAwsx4oBAAAAXG+cGpzy8/O1YcMGRUdH29tcXFwUHR2ttWvXlrpcTk6OQkNDFRISorvvvls//vhjqX3z8vKUnZ3t8AAAAACAsnBqcDp27JgKCgqKnTEKDAxUZmZmicvUq1dP06dP18cff6zZs2ersLBQ7dq1088//1xi/5SUFPn7+9sfISEhl3w/AAAAAFzbnH6pXllFRkYqNjZWERERioqK0qJFi1StWjW9+eabJfZPSkrSyZMn7Y+DBw+Wc8UAAAAArnZuztx41apV5erqqqysLIf2rKwsBQUFXdA6KlSooObNm2vXrl0lzvfw8JCHh8ffrhUAAADA9cupZ5zc3d3VsmVLpaam2tsKCwuVmpqqyMjIC1pHQUGBtm7dquDg4MtVJgAAAIDrnFPPOElSYmKi4uLi1KpVK7Vp00bjx49Xbm6u4uPjJUmxsbGqUaOGUlJSJEnPP/+8br75ZoWHh+vEiRMaO3as9u/fr0cffdSZuwEAAADgGub04NSrVy8dPXpUI0aMUGZmpiIiIrR06VL7gBEHDhyQi8v/Toz99ttv6tevnzIzMxUQEKCWLVvq22+/VcOGDZ21CwAAAACucTZjjHF2EeUpOztb/v7+OnnypPz8/JxdDoCrTPuJ7Z1dAkqxZsgaZ5cAALjKlCUbXHWj6gEAAABAeSM4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFN2cXAAAAAFxNXnr4XmeXgFIMn73wsq2bM04AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWGI4cuEAHnm/i7BJwHjVHbHV2CQAA4BrGGScAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALV0Rwmjx5ssLCwuTp6am2bdtq3bp1F7Tce++9J5vNph49elzeAgEAAABc15wenObPn6/ExEQlJydr48aNatasmWJiYnTkyJHzLrdv3z498cQT6tChQzlVCgAAAOB65fTgNG7cOPXr10/x8fFq2LChpk6dKm9vb02fPr3UZQoKCtS7d2+NGjVKtWrVKsdqAQAAAFyPnBqc8vPztWHDBkVHR9vbXFxcFB0drbVr15a63PPPP6/q1aurb9++5VEmAAAAgOucmzM3fuzYMRUUFCgwMNChPTAwUNu3by9xmdWrV+v//u//lJaWdkHbyMvLU15enn06Ozv7ousFAAAAcH1y+qV6ZXHq1Ck98sgjeuutt1S1atULWiYlJUX+/v72R0hIyGWuEgAAAMC1xqlnnKpWrSpXV1dlZWU5tGdlZSkoKKhY/927d2vfvn3q3r27va2wsFCS5Obmph07dqh27doOyyQlJSkxMdE+nZ2dTXgCAAAAUCZODU7u7u5q2bKlUlNT7UOKFxYWKjU1VQkJCcX6169fX1u3bnVoe/bZZ3Xq1ClNmDChxEDk4eEhDw+Py1I/AAAAgOuDU4OTJCUmJiouLk6tWrVSmzZtNH78eOXm5io+Pl6SFBsbqxo1aiglJUWenp5q3Lixw/KVKlWSpGLtAAAAAHCpOD049erVS0ePHtWIESOUmZmpiIgILV261D5gxIEDB+TiclXdigUAAADgGuP04CRJCQkJJV6aJ0mrVq0677IzZsy49AUBAAAAwF9wKgcAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMCCm7MLuFq0fPJdZ5eA89gwNtbZJQAAAOAaxhknAAAAALBAcAIAAAAAC1yqBwAAUAaTHv/U2SWgFAmvdXd2CbiGccYJAAAAACyU6YzTli1bLqhf06ZNL6oYAAAAALgSlSk4RUREyGazyRhTbF5Ru81mU0FBwSUrEAAAAACcrUzBae/evZerDgAAAAC4YpUpOIWGhl6uOgAAAADgilWmwSGOHTum/fv3O7T9+OOPio+P1/3336+5c+de0uIAAAAA4EpQpuA0ZMgQ/fe//7VPHzlyRB06dND69euVl5enPn36aNasWZe8SAAAAABwpjIFp++++0533XWXffrdd99V5cqVlZaWpo8//lijR4/W5MmTy1zE5MmTFRYWJk9PT7Vt21br1q0rte+iRYvUqlUrVapUST4+PoqIiCCsAQAAALisyhScMjMzFRYWZp9esWKF7rnnHrm5/Xmr1F133aWMjIwyFTB//nwlJiYqOTlZGzduVLNmzRQTE6MjR46U2L9y5coaPny41q5dqy1btig+Pl7x8fFatmxZmbYLAAAAABeqTMHJz89PJ06csE+vW7dObdu2tU/bbDbl5eWVqYBx48apX79+io+PV8OGDTV16lR5e3tr+vTpJfbv1KmT/vnPf6pBgwaqXbu2hg4dqqZNm2r16tVl2i4AAAAAXKgyBaebb75Z//3vf1VYWKiFCxfq1KlTuvXWW+3zd+7cqZCQkAteX35+vjZs2KDo6Oj/FeTioujoaK1du9ZyeWOMUlNTtWPHDnXs2LHEPnl5ecrOznZ4AAAAAEBZlCk4Pf/88/rkk0/k5eWlXr16adiwYQoICLDPf++99xQVFXXB6zt27JgKCgoUGBjo0B4YGKjMzMxSlzt58qR8fX3l7u6uO+64QxMnTlTnzp1L7JuSkiJ/f3/7oyzBDgAAAACkMv6OU7NmzZSenq41a9YoKCjI4TI9SXrwwQfVoEGDS1pgSSpWrKi0tDTl5OQoNTVViYmJqlWrljp16lSsb1JSkhITE+3T2dnZhCcAAAAAZVKm4LRixQolJCTou+++k5+fn8O8kydP6sknn9TUqVN10003XdD6qlatKldXV2VlZTm0Z2VlKSgoqNTlXFxcFB4eLkmKiIhQenq6UlJSSgxOHh4e8vDwuKB6AAAAAKAkZbpUb/z48erXr1+x0CRJ/v7+GjBggMaNG3fB63N3d1fLli2VmppqbyssLFRqaqoiIyMveD2FhYVlHpQCAAAAAC5UmYLT5s2b1aVLl1Ln33777dqwYUOZCkhMTNRbb72lmTNnKj09XQMHDlRubq7i4+MlSbGxsUpKSrL3T0lJ0Zdffqk9e/YoPT1dr732mmbNmqWHH364TNsFAAAAgAtVpkv1srKyVKFChdJX5uamo0ePlqmAXr166ejRoxoxYoQyMzMVERGhpUuX2geMOHDggFxc/pfvcnNzNWjQIP3888/y8vJS/fr1NXv2bPXq1atM2wUAAACAC1Wm4FSjRg1t27bNfn/RubZs2aLg4OAyF5GQkKCEhIQS561atcph+sUXX9SLL75Y5m0AAAAAwMUq06V63bp103PPPaezZ88Wm3fmzBklJyfrzjvvvGTFAQAAAMCVoExnnJ599lktWrRIdevWVUJCgurVqydJ2r59uyZPnqyCggINHz78shQKAAAAAM5SpuAUGBiob7/9VgMHDlRSUpKMMZIkm82mmJgYTZ48udiP2QIAAADA1a5MwUmSQkND9fnnn+u3337Trl27ZIxRnTp1FBAQcDnqAwDgivJVxyhnl4BSRH39lbNLAHANK3NwKhIQEKDWrVtfyloAAAAA4IpUpsEhAAAAAOB6RHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAtXRHCaPHmywsLC5OnpqbZt22rdunWl9n3rrbfUoUMHBQQEKCAgQNHR0eftDwAAAAB/l9OD0/z585WYmKjk5GRt3LhRzZo1U0xMjI4cOVJi/1WrVunBBx/UypUrtXbtWoWEhOj222/XoUOHyrlyAAAAANcLpwencePGqV+/foqPj1fDhg01depUeXt7a/r06SX2nzNnjgYNGqSIiAjVr19fb7/9tgoLC5WamlrOlQMAAAC4Xjg1OOXn52vDhg2Kjo62t7m4uCg6Olpr1669oHWcPn1av//+uypXrlzi/Ly8PGVnZzs8AAAAAKAsnBqcjh07poKCAgUGBjq0BwYGKjMz84LW8dRTT+mGG25wCF9/lZKSIn9/f/sjJCTkb9cNAAAA4Pri9Ev1/o4xY8bovffe04cffihPT88S+yQlJenkyZP2x8GDB8u5SgAAAABXOzdnbrxq1apydXVVVlaWQ3tWVpaCgoLOu+yrr76qMWPGaPny5WratGmp/Tw8POTh4XFJ6gUAAABwfXLqGSd3d3e1bNnSYWCHooEeIiMjS13ulVde0QsvvKClS5eqVatW5VEqAAAAgOuYU884SVJiYqLi4uLUqlUrtWnTRuPHj1dubq7i4+MlSbGxsapRo4ZSUlIkSS+//LJGjBihuXPnKiwszH4vlK+vr3x9fZ22HwAAAACuXU4PTr169dLRo0c1YsQIZWZmKiIiQkuXLrUPGHHgwAG5uPzvxNiUKVOUn5+ve++912E9ycnJGjlyZHmWDgAAAOA64fTgJEkJCQlKSEgocd6qVascpvft23f5CwIAAACAv7iqR9UDAAAAgPJAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAAC04PTpMnT1ZYWJg8PT3Vtm1brVu3rtS+P/74o3r27KmwsDDZbDaNHz++/AoFAAAAcN1yanCaP3++EhMTlZycrI0bN6pZs2aKiYnRkSNHSux/+vRp1apVS2PGjFFQUFA5VwsAAADgeuXU4DRu3Dj169dP8fHxatiwoaZOnSpvb29Nnz69xP6tW7fW2LFj9cADD8jDw6OcqwUAAABwvXJacMrPz9eGDRsUHR39v2JcXBQdHa21a9c6qywAAAAAKMbNWRs+duyYCgoKFBgY6NAeGBio7du3X7Lt5OXlKS8vzz6dnZ19ydYNAAAA4Prg9MEhLreUlBT5+/vbHyEhIc4uCQAAAMBVxmnBqWrVqnJ1dVVWVpZDe1ZW1iUd+CEpKUknT560Pw4ePHjJ1g0AAADg+uC04OTu7q6WLVsqNTXV3lZYWKjU1FRFRkZesu14eHjIz8/P4QEAAAAAZeG0e5wkKTExUXFxcWrVqpXatGmj8ePHKzc3V/Hx8ZKk2NhY1ahRQykpKZL+HFDip59+sv//oUOHlJaWJl9fX4WHhzttPwAAAABc25wanHr16qWjR49qxIgRyszMVEREhJYuXWofMOLAgQNycfnfSbHDhw+refPm9ulXX31Vr776qqKiorRq1aryLh8AAADAdcKpwUmSEhISlJCQUOK8c8NQWFiYjDHlUBUAAAAA/M81P6oeAAAAAPxdBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALV0Rwmjx5ssLCwuTp6am2bdtq3bp15+2/YMEC1a9fX56enmrSpIk+//zzcqoUAAAAwPXI6cFp/vz5SkxMVHJysjZu3KhmzZopJiZGR44cKbH/t99+qwcffFB9+/bVpk2b1KNHD/Xo0UPbtm0r58oBAAAAXC+cHpzGjRunfv36KT4+Xg0bNtTUqVPl7e2t6dOnl9h/woQJ6tKli5588kk1aNBAL7zwglq0aKFJkyaVc+UAAAAArhduztx4fn6+NmzYoKSkJHubi4uLoqOjtXbt2hKXWbt2rRITEx3aYmJi9NFHH5XYPy8vT3l5efbpkydPSpKys7PLVGtB3pky9Uf5KuvreTFOnS247NvAxSuPY0CS/jjzR7lsB2VXXsdA7h8cA1eq8joGzuSdLpftoOzK6xg4+/vv5bIdlF1Zj4Gi/sYYy75ODU7Hjh1TQUGBAgMDHdoDAwO1ffv2EpfJzMwssX9mZmaJ/VNSUjRq1Khi7SEhIRdZNa5E/hP/5ewS4Gwp/s6uAE7m/xTHwHXPn2PgejdssrMrgLO9+P7FvQ+cOnVK/hbvIU4NTuUhKSnJ4QxVYWGhfv31V1WpUkU2m82JlTlPdna2QkJCdPDgQfn5+Tm7HDgBxwA4BsAxAInjABwDxhidOnVKN9xwg2VfpwanqlWrytXVVVlZWQ7tWVlZCgoKKnGZoKCgMvX38PCQh4eHQ1ulSpUuvuhriJ+f33X5DwT/wzEAjgFwDEDiOMD1fQxYnWkq4tTBIdzd3dWyZUulpqba2woLC5WamqrIyMgSl4mMjHToL0lffvllqf0BAAAA4O9y+qV6iYmJiouLU6tWrdSmTRuNHz9eubm5io+PlyTFxsaqRo0aSklJkSQNHTpUUVFReu2113THHXfovffe0w8//KBp06Y5czcAAAAAXMOcHpx69eqlo0ePasSIEcrMzFRERISWLl1qHwDiwIEDcnH534mxdu3aae7cuXr22Wf1zDPPqE6dOvroo4/UuHFjZ+3CVcfDw0PJycnFLmHE9YNjABwD4BiAxHEAjoGysJkLGXsPAAAAAK5jTv8BXAAAAAC40hGcAAAAAMACwQkAAAAALBCccFFWrVolm82mEydOOLsUAOcxY8YMfrsOQKk6deqkf//7384uA9eIa/37IcHpKnT06FENHDhQNWvWlIeHh4KCghQTE6M1a9Y4uzRcRn369JHNZrM/qlSpoi5dumjLli3OLg2XWJ8+fdSjRw9nl4ErzF/fAypUqKDAwEB17txZ06dPV2FhobPLwyVUnq/1okWL9MILL1zSdeLvy8zM1NChQxUeHi5PT08FBgaqffv2mjJlik6fPu3s8krVrl07/fLLLxf8g7JXG4LTVahnz57atGmTZs6cqZ07d+qTTz5Rp06ddPz4cWeXhsusS5cu+uWXX/TLL78oNTVVbm5uuvPOO51dFoByUvQesG/fPi1ZskS33HKLhg4dqjvvvFN//PGHs8vDJVRer3XlypVVsWLFS7Y+/H179uxR8+bN9cUXX2j06NHatGmT1q5dq2HDhumzzz7T8uXLnV1iqdzd3RUUFCSbzebsUi4LgtNV5sSJE/rmm2/08ssv65ZbblFoaKjatGmjpKQk3XXXXfY+jz76qKpVqyY/Pz/deuut2rx5s8N6pkyZotq1a8vd3V316tXTrFmz7PP27dsnm82mtLQ0h+3abDatWrWqPHYTpSg6wxgUFKSIiAg9/fTTOnjwoI4ePVri6fG0tDTZbDbt27dPkrR//351795dAQEB8vHxUaNGjfT5559LkgoKCtS3b1/ddNNN8vLyUr169TRhwgSH7RedCXn11VcVHBysKlWqaPDgwfr999/L6ym47nTq1EmPPfaYhg0bpsqVKysoKEgjR4506HPixAkNGDBAgYGB8vT0VOPGjfXZZ5859Fm2bJkaNGggX19f+xeyv3r77bfVoEEDeXp6qn79+nrjjTcc5m/dulW33nqrvLy8VKVKFfXv3185OTn2+Rwb5aPoPaBGjRpq0aKFnnnmGX388cdasmSJZsyYIUkaN26cmjRpIh8fH4WEhGjQoEEOr5UkrV69Wh06dJCXl5dCQkL02GOPKTc31z7/l19+0R133CEvLy/ddNNNmjt3rsLCwjR+/HhJfE6Uh/J6rc+9VC8vL09PPPGEatSoIR8fH7Vt27bYa/rBBx+oUaNG8vDwUFhYmF577TWH+WFhYRo9erT+3//7f6pYsaJq1qypadOmXdLn51o2aNAgubm56YcfftD999+vBg0aqFatWrr77ru1ePFide/eXU888YTDH07Hjx8vm82mpUuX2tvCw8P19ttvS5LWr1+vzp07q2rVqvL391dUVJQ2btzosF2bzaa3335b//znP+Xt7a06derok08+cejzySefqE6dOvL09NQtt9yimTNnOnz3OPe7yMiRIxUREeGwjvHjxyssLOzSPFnljOB0lfH19ZWvr68++ugj5eXlldjnvvvu05EjR7RkyRJt2LBBLVq00G233aZff/1VkvThhx9q6NChevzxx7Vt2zYNGDBA8fHxWrlyZXnuCv6mnJwczZ49W+Hh4apSpcoFLTN48GDl5eXp66+/1tatW/Xyyy/L19dXklRYWKgbb7xRCxYs0E8//aQRI0bomWee0fvvv++wjpUrV2r37t1auXKlZs6cqRkzZtg/xHF5zJw5Uz4+Pvr+++/1yiuv6Pnnn9eXX34p6c/XrWvXrlqzZo1mz56tn376SWPGjJGrq6t9+dOnT+vVV1/VrFmz9PXXX+vAgQN64okn7PPnzJmjESNG6KWXXlJ6erpGjx6t5557TjNnzpQk5ebmKiYmRgEBAVq/fr0WLFig5cuXKyEhwaFOjg3nuPXWW9WsWTMtWrRIkuTi4qL//ve/+vHHHzVz5kytWLFCw4YNs/ffvXu3unTpop49e2rLli2aP3++Vq9e7fB6xsbG6vDhw1q1apU++OADTZs2TUeOHCn3fYOjy/FanyshIUFr167Ve++9py1btui+++5Tly5dlJGRIUnasGGD7r//fj3wwAPaunWrRo4cqeeee67Yv/XXXntNrVq10qZNmzRo0CANHDhQO3bsuPRPyjXm+PHj+uKLLzR48GD5+PiU2MdmsykqKkqrV69WQUGBJOmrr75S1apV7SH30KFD2r17tzp16iRJOnXqlOLi4rR69Wp99913qlOnjrp166ZTp045rHvUqFG6//77tWXLFnXr1k29e/e2f3/cu3ev7r33XvXo0UObN2/WgAEDNHz48MvzRFypDK46CxcuNAEBAcbT09O0a9fOJCUlmc2bNxtjjPnmm2+Mn5+fOXv2rMMytWvXNm+++aYxxph27dqZfv36Ocy/7777TLdu3Ywxxuzdu9dIMps2bbLP/+2334wks3LlSmOMMStXrjSSzG+//XZ5dhLFxMXFGVdXV+Pj42N8fHyMJBMcHGw2bNhgjCn5Ndm0aZORZPbu3WuMMaZJkyZm5MiRF7zNwYMHm549ezrUEBoaav744w9723333Wd69er193YODuLi4szdd99tjDEmKirK/OMf/3CY37p1a/PUU08ZY4xZtmyZcXFxMTt27ChxXe+8846RZHbt2mVvmzx5sgkMDLRP165d28ydO9dhuRdeeMFERkYaY4yZNm2aCQgIMDk5Ofb5ixcvNi4uLiYzM9NeM8fG5fXX4+JcvXr1Mg0aNChx3oIFC0yVKlXs03379jX9+/d36PPNN98YFxcXc+bMGZOenm4kmfXr19vnZ2RkGEnm9ddfN8Zc2OcELl55vdbG/PkeM3ToUGOMMfv37zeurq7m0KFDDsvcdtttJikpyRhjzEMPPWQ6d+7sMP/JJ580DRs2tE+Hhoaahx9+2D5dWFhoqlevbqZMmXKevYYxxnz33XdGklm0aJFDe5UqVeyf/8OGDTO//fabcXFxMevXrzeFhYWmcuXKJiUlxbRt29YYY8zs2bNNjRo1St1OQUGBqVixovn000/tbZLMs88+a5/OyckxksySJUuMMcY89dRTpnHjxg7rGT58uMN3j3O/iyQnJ5tmzZo5LPP666+b0NDQsjwtVwzOOF2FevbsqcOHD+uTTz5Rly5dtGrVKrVo0UIzZszQ5s2blZOToypVqtjPTvn6+mrv3r3avXu3JCk9PV3t27d3WGf79u2Vnp7ujN1BGdxyyy1KS0tTWlqa1q1bp5iYGHXt2lX79++/oOUfe+wxvfjii2rfvr2Sk5OLDSwxefJktWzZUtWqVZOvr6+mTZumAwcOOPRp1KiRw9mM4OBg/hJ9mTVt2tRh+q/PeVpamm688UbVrVu31OW9vb1Vu3btEpfPzc3V7t271bdvX4f3jBdffNHhPaNZs2YOf/1s3769CgsLHf6CzLHhPMYY+z0Fy5cv12233aYaNWqoYsWKeuSRR3T8+HH7DeWbN2/WjBkzHF7vmJgYFRYWau/evdqxY4fc3NzUokUL+/rDw8MVEBDglH2Do0v5Wp9r69atKigoUN26dR2W+eqrryy/Q2RkZNjPfkiO71s2m01BQUG8H/wN69atU1pamho1aqS8vDxVqlRJzZo106pVq7R161a5u7urf//+2rRpk3JycvTVV18pKirKvnxWVpb69eunOnXqyN/fX35+fsrJySn2Gf/X183Hx0d+fn72123Hjh1q3bq1Q/82bdpcxr2+8rg5uwBcHE9PT3Xu3FmdO3fWc889p0cffVTJyckaNGiQgoODS7zG/EKHJHZx+TNPG2PsbdyncGXw8fFReHi4ffrtt9+Wv7+/3nrrLd1+++2Szv+6Pfroo4qJidHixYv1xRdfKCUlRa+99pqGDBmi9957T0888YRee+01RUZGqmLFiho7dqy+//57h3VUqFDBYdpmszGi12V2vufcy8vropYvOk6K7od466231LZtW4d+fw1Bf7dOXF7p6em66aabtG/fPt15550aOHCgXnrpJVWuXFmrV69W3759lZ+fL29vb+Xk5GjAgAF67LHHiq2nZs2a2rlzp+X2+Jxwnkv5Wp8rJydHrq6u2rBhQ7F//0WXdV8o3g8uTnh4uGw2W7HLGmvVqiXJ8T2/U6dOWrVqlTw8PBQVFaXKlSurQYMGWr16tb766is9/vjj9r5xcXE6fvy4JkyYoNDQUHl4eCgyMlL5+fkO27nUr5uLi4vD+4R0db9XEJyuEQ0bNtRHH32kFi1aKDMzU25ubqXeeNegQQOtWbNGcXFx9rY1a9aoYcOGkqRq1apJ+vPm4ObNm0uSww3AuHLYbDa5uLjozJkzDq9b0V+GS3rdQkJC9K9//Uv/+te/lJSUpLfeektDhgzRmjVr1K5dOw0aNMjet+gvjLhyNW3aVD///LN27tx53rNOpQkMDNQNN9ygPXv2qHfv3iX2adCggWbMmKHc3Fz7Wac1a9bIxcVF9erV+1v14+9bsWKFtm7dqv/85z/asGGDCgsL9dprr9nDzbn3KbZo0UI//fSTwx9h/qpevXr6448/tGnTJrVs2VKStGvXLv3222/2PnxOOMelfq3P1bx5cxUUFOjIkSPq0KFDiX2KvkP81Zo1a1S3bt0y/7EFxVWpUkWdO3fWpEmTNGTIkFLvc5KkqKgoTZ8+XW5uburSpYukP8PUvHnztHPnTvv9TdKfr9Ebb7yhbt26SZIOHjyoY8eOlam2evXq2QeUKrJ+/frzLlOtWjVlZmY6nCm9mt8ruFTvKnP8+HHdeuutmj17trZs2aK9e/dqwYIFeuWVV3T33XcrOjpakZGR6tGjh7744gvt27dP3377rYYPH64ffvhBkvTkk09qxowZmjJlijIyMjRu3DgtWrTIfrO4l5eXbr75Zo0ZM0bp6en66quv9Oyzzzpzt/H/y8vLU2ZmpjIzM5Wenq4hQ4YoJydH3bt3V3h4uEJCQjRy5EhlZGRo8eLFxUY6+ve//61ly5Zp79692rhxo1auXKkGDRpIkurUqaMffvhBy5Yt086dO/Xcc89ZviHC+aKiotSxY0f17NlTX375pfbu3aslS5Y4jKxkZdSoUUpJSdF///tf7dy5U1u3btU777yjcePGSZJ69+4tT09PxcXFadu2bVq5cqWGDBmiRx55RIGBgZdr11CCoveAQ4cOaePGjRo9erTuvvtu3XnnnYqNjVV4eLh+//13TZw4UXv27NGsWbM0depUh3U89dRT+vbbb5WQkKC0tDRlZGTo448/tg8YUL9+fUVHR6t///5at26dNm3apP79+8vLy8v+xYfPicuvPF7rc9WtW1e9e/dWbGysFi1apL1792rdunVKSUnR4sWLJUmPP/64UlNT9cILL2jnzp2aOXOmJk2a5DDgDP6eN954Q3/88YdatWql+fPnKz09XTt27NDs2bO1fft2e0Dt2LGjTp06pc8++8wekjp16qQ5c+YoODjY4Y9pderU0axZs5Senq7vv/9evXv3vqArFv5qwIAB2r59u5566int3LlT77//vn1QkNKGH+/UqZOOHj2qV155Rbt379bkyZO1ZMmSsj8pVwrn3V6Fi3H27Fnz9NNPmxYtWhh/f3/j7e1t6tWrZ5599llz+vRpY4wx2dnZZsiQIeaGG24wFSpUMCEhIaZ3797mwIED9vW88cYbplatWqZChQqmbt265t1333XYzk8//WQiIyONl5eXiYiIMF988QWDQzhZXFyckWR/VKxY0bRu3dosXLjQ3mf16tWmSZMmxtPT03To0MEsWLDAYXCIhIQEU7t2bePh4WGqVatmHnnkEXPs2DFjzJ/HVp8+fYy/v7+pVKmSGThwoHn66acdbuos6YbloUOHmqioqMu899eXcweHKLpxu8jdd99t4uLi7NPHjx838fHxpkqVKsbT09M0btzYfPbZZ8aYPweH8Pf3d1j+ww8/NOe+/c+ZM8dEREQYd3d3ExAQYDp27Ohwc/KWLVvMLbfcYjw9PU3lypVNv379zKlTp0qsuQjHxqX11/cANzc3U61aNRMdHW2mT59uCgoK7P3GjRtngoODjZeXl4mJiTHvvvtusffrdevWmc6dOxtfX1/j4+NjmjZtal566SX7/MOHD5uuXbsaDw8PExoaaubOnWuqV69upk6dau9j9TmBi1eer/W57zH5+flmxIgRJiwszFSoUMEEBwebf/7zn2bLli32PgsXLjQNGzY0FSpUMDVr1jRjx451qD80NNQ+kEiRZs2ameTk5Evy/FwPDh8+bBISEsxNN91kKlSoYHx9fU2bNm3M2LFjTW5urr1fs2bNTFBQkH36+PHjxmazmQceeMBhfRs3bjStWrUynp6epk6dOmbBggXFXidJ5sMPP3RYzt/f37zzzjv26Y8//tiEh4cbDw8P06lTJzNlyhQjyT7YSEnfD6dMmWJCQkKMj4+PiY2NNS+99NJVOziEzZhzLjwEAAD4i59//lkhISH2wQhw7YiMjNRtt92mF1980dml4Cr00ksvaerUqTp48KCkP38zsGvXrjp79qzc3d2dXN2lxz1OAADAwYoVK5STk6MmTZrol19+0bBhwxQWFqaOHTs6uzRcInl5edq6dat+/PHHEgePAEryxhtvqHXr1qpSpYrWrFmjsWPH2i/9zMrK0scff6w6depck6FJIjgBAIBz/P7773rmmWe0Z88eVaxYUe3atdOcOXOKjbiFq9eSJUsUGxuru+66S/fee6+zy8FVIiMjQy+++KJ+/fVX1axZU48//riSkpIkyf6Dum+88YaTq7x8uFQPAAAAACwwqh4AAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWPj/AJ3jGjFFTxgpAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "plt.figure(figsize=(10,5))\n",
    "sns.barplot(x=['Seoul', 'Busan', 'Incheon', 'Daegu', 'Daejeon', 'Gwangju'], y=voting)\n",
    "plt.title('CSI Score of Voting Model')\n",
    "plt.ylabel('CSI')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4078715882283252"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(voting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.3248062015503624,\n",
       " 0.46608315098458075,\n",
       " 0.5763157894736463,\n",
       " 0.2852112676055334,\n",
       " 0.31884057971011603,\n",
       " 0.4759725400457121]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.5106142349456536,\n",
       " 0.640202275543952,\n",
       " 0.709448778435959,\n",
       " 0.45579515959653394,\n",
       " 0.453960121993875,\n",
       " 0.6218724605270242]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mcc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.9005707762557078,\n",
       " 0.9721461187214612,\n",
       " 0.9264840182648402,\n",
       " 0.9768264840182649,\n",
       " 0.9141552511415525,\n",
       " 0.9477168949771689]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}