{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "from sklearn.preprocessing import StandardScaler, LabelEncoder\n", "import torch\n", "from torch.utils.data import DataLoader, TensorDataset\n", "import random\n", "from collections import Counter\n", "import sys\n", "sys.path.append('../../../../../../../../mnt/workspace/LightGBM/python-package')\n", "from lightgbm import LGBMClassifier\n", "import numpy as np\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.inspection import permutation_importance\n", "from sklearn.metrics import confusion_matrix,accuracy_score, recall_score, precision_score\n", "from sklearn.model_selection import StratifiedKFold\n", "from xgboost import XGBClassifier\n", "from warnings import filterwarnings\n", "filterwarnings('ignore')\n", "import sys\n", "sys.path.append('../')\n", "import torch\n", "import torch.nn as nn\n", "import torch.optim as optim\n", "import optuna\n", "import pandas as pd\n", "import numpy as np\n", "import random\n", "from ft_transformer import FTTransformer\n", "from resnet_like import ResNetLike\n", "from deepgbm import DeepGBM\n", "from pytorch_tabnet.tab_model import TabNetClassifier\n", "from sklearn.metrics import accuracy_score, f1_score, roc_auc_score, confusion_matrix" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# Python 및 Numpy 시드 고정\n", "seed = 42\n", "random.seed(seed)\n", "np.random.seed(seed)\n", "\n", "# PyTorch 시드 고정\n", "torch.manual_seed(seed)\n", "torch.cuda.manual_seed(seed)\n", "torch.cuda.manual_seed_all(seed) # Multi-GPU 환경에서 동일한 시드 적용\n", "\n", "# PyTorch 연산의 결정적 모드 설정\n", "torch.backends.cudnn.deterministic = True # 실행마다 동일한 결과를 보장\n", "torch.backends.cudnn.benchmark = True # 성능 최적화를 활성화 (가능한 한 빠른 연산 수행)\n", "\n", "# 전처리 함수\n", "def preprocessing(df):\n", " df = df[df.columns].copy()\n", " df.loc[df['wind_dir']=='정온', 'wind_dir'] = \"0\"\n", " df['wind_dir'] = df['wind_dir'].astype('int')\n", " df['lm_cloudcover'] = df['lm_cloudcover'].astype('int')\n", " df['cloudcover'] = df['cloudcover'].astype('int')\n", " return df\n", "\n", "# 데이터셋 준비 함수\n", "def prepare_dataset(region, data_sample='pure', target='multi', fold=3):\n", "\n", " # 데이터 경로 지정\n", " dat_path = f\"../../data/data_for_modeling/{region}_train.csv\"\n", " if data_sample == 'pure':\n", " train_path = dat_path\n", " else:\n", " train_path = f'../../data/data_oversampled/{data_sample}/{data_sample}_{fold}_{region}.csv'\n", " train_path = f'../../data/data_oversampled/{data_sample}/{data_sample}_{fold}_{region}.csv'\n", " test_path = f\"../../data/data_for_modeling/{region}_test.csv\"\n", " drop_col = ['binary_class','multi_class','visi','year']\n", " target_col = f'{target}_class'\n", " \n", " # 데이터 로드\n", " region_dat = preprocessing(pd.read_csv(dat_path, index_col=0))\n", " if data_sample == 'pure':\n", " region_train = region_dat.loc[~region_dat['year'].isin([2021-fold]), :]\n", " else:\n", " region_train = preprocessing(pd.read_csv(train_path))\n", " region_val = region_dat.loc[region_dat['year'].isin([2021-fold]), :]\n", " region_test = preprocessing(pd.read_csv(test_path))\n", "\n", " # 컬럼 정렬 (일관성 유지)\n", " common_columns = region_train.columns.to_list()\n", " train_data = region_train[common_columns]\n", " val_data = region_val[common_columns]\n", " test_data = region_test[common_columns]\n", "\n", " # 설명변수 & 타겟 분리\n", " X_train = train_data.drop(columns=drop_col)\n", " y_train = train_data[target_col]\n", " X_val = val_data.drop(columns=drop_col)\n", " y_val = val_data[target_col]\n", " X_test = test_data.drop(columns=drop_col)\n", " y_test = test_data[target_col]\n", "\n", " # 범주형 & 연속형 변수 분리\n", " categorical_cols = X_train.select_dtypes(include=['object', 'category', 'int64']).columns\n", " numerical_cols = X_train.select_dtypes(include=['float64']).columns\n", "\n", " # 범주형 변수 Label Encoding\n", " label_encoders = {}\n", " for col in categorical_cols:\n", " le = LabelEncoder()\n", " le.fit(X_train[col]) # Train 데이터 기준으로 학습\n", " label_encoders[col] = le\n", "\n", " # 변환 적용\n", " for col in categorical_cols:\n", " X_train[col] = label_encoders[col].transform(X_train[col])\n", " X_val[col] = label_encoders[col].transform(X_val[col])\n", " X_test[col] = label_encoders[col].transform(X_test[col])\n", "\n", " # 연속형 변수 Standard Scaling\n", " scaler = StandardScaler()\n", " scaler.fit(X_train[numerical_cols]) # Train 데이터 기준으로 학습\n", "\n", " # 변환 적용\n", " X_train[numerical_cols] = scaler.transform(X_train[numerical_cols])\n", " X_val[numerical_cols] = scaler.transform(X_val[numerical_cols])\n", " X_test[numerical_cols] = scaler.transform(X_test[numerical_cols])\n", "\n", " return X_train, X_val, X_test, y_train, y_val, y_test, categorical_cols, numerical_cols\n", "\n", "# 데이터 변환 및 dataloader 생성 함수\n", "def prepare_dataloader(region, data_sample='pure', target='multi', fold=3, random_state=None):\n", "\n", " # 데이터 경로 지정\n", " dat_path = f\"../../data/data_for_modeling/{region}_train.csv\"\n", " if data_sample == 'pure':\n", " train_path = dat_path\n", " else:\n", " train_path = f'../../data/data_oversampled/{data_sample}/{data_sample}_{fold}_{region}.csv'\n", " train_path = f'../../data/data_oversampled/{data_sample}/{data_sample}_{fold}_{region}.csv'\n", " test_path = f\"../../data/data_for_modeling/{region}_test.csv\"\n", " drop_col = ['binary_class','multi_class','visi','year']\n", " target_col = f'{target}_class'\n", " \n", " # 데이터 로드\n", " region_dat = preprocessing(pd.read_csv(dat_path, index_col=0))\n", " if data_sample == 'pure':\n", " region_train = region_dat.loc[~region_dat['year'].isin([2021-fold]), :]\n", " else:\n", " region_train = preprocessing(pd.read_csv(train_path))\n", " region_val = region_dat.loc[region_dat['year'].isin([2021-fold]), :]\n", " region_test = preprocessing(pd.read_csv(test_path))\n", "\n", " # 컬럼 정렬 (일관성 유지)\n", " common_columns = region_train.columns.to_list()\n", " train_data = region_train[common_columns]\n", " val_data = region_val[common_columns]\n", " test_data = region_test[common_columns]\n", "\n", " # 설명변수 & 타겟 분리\n", " X_train = train_data.drop(columns=drop_col)\n", " y_train = train_data[target_col]\n", " X_val = val_data.drop(columns=drop_col)\n", " y_val = val_data[target_col]\n", " X_test = test_data.drop(columns=drop_col)\n", " y_test = test_data[target_col]\n", "\n", " # 범주형 & 연속형 변수 분리\n", " categorical_cols = X_train.select_dtypes(include=['object', 'category', 'int64']).columns\n", " numerical_cols = X_train.select_dtypes(include=['float64']).columns\n", "\n", " # 범주형 변수 Label Encoding\n", " label_encoders = {}\n", " for col in categorical_cols:\n", " le = LabelEncoder()\n", " le.fit(X_train[col]) # Train 데이터 기준으로 학습\n", " label_encoders[col] = le\n", "\n", " # 변환 적용\n", " for col in categorical_cols:\n", " X_train[col] = label_encoders[col].transform(X_train[col])\n", " X_val[col] = label_encoders[col].transform(X_val[col])\n", " X_test[col] = label_encoders[col].transform(X_test[col])\n", "\n", " # 연속형 변수 Standard Scaling\n", " scaler = StandardScaler()\n", " scaler.fit(X_train[numerical_cols]) # Train 데이터 기준으로 학습\n", "\n", " # 변환 적용\n", " X_train[numerical_cols] = scaler.transform(X_train[numerical_cols])\n", " X_val[numerical_cols] = scaler.transform(X_val[numerical_cols])\n", " X_test[numerical_cols] = scaler.transform(X_test[numerical_cols])\n", "\n", " # 연속형 변수와 범주형 변수 분리\n", " X_train_num = torch.tensor(X_train[numerical_cols].values, dtype=torch.float32)\n", " X_train_cat = torch.tensor(X_train[categorical_cols].values, dtype=torch.long)\n", "\n", " X_val_num = torch.tensor(X_val[numerical_cols].values, dtype=torch.float32)\n", " X_val_cat = torch.tensor(X_val[categorical_cols].values, dtype=torch.long)\n", "\n", " X_test_num = torch.tensor(X_test[numerical_cols].values, dtype=torch.float32)\n", " X_test_cat = torch.tensor(X_test[categorical_cols].values, dtype=torch.long)\n", "\n", " # 레이블 변환\n", " if target == \"binary\":\n", " y_train_tensor = torch.tensor(y_train.values, dtype=torch.float32) # 이진 분류 → float32\n", " y_val_tensor = torch.tensor(y_val.values, dtype=torch.float32)\n", " y_test_tensor = torch.tensor(y_test.values, dtype=torch.float32)\n", " elif target == \"multi\":\n", " y_train_tensor = torch.tensor(y_train.values, dtype=torch.long) # 다중 분류 → long\n", " y_val_tensor = torch.tensor(y_val.values, dtype=torch.long)\n", " y_test_tensor = torch.tensor(y_test.values, dtype=torch.long)\n", " else:\n", " raise ValueError(\"target must be 'binary' or 'multi'\")\n", "\n", " # TensorDataset 생성\n", " train_dataset = TensorDataset(X_train_num, X_train_cat, y_train_tensor)\n", " val_dataset = TensorDataset(X_val_num, X_val_cat, y_val_tensor)\n", " test_dataset = TensorDataset(X_test_num, X_test_cat, y_test_tensor)\n", "\n", " # DataLoader 생성\n", " if random_state == None:\n", " train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n", " else:\n", " train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True, generator=torch.Generator().manual_seed(random_state))\n", " val_loader = DataLoader(val_dataset, batch_size=64, shuffle=False)\n", " test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)\n", " \n", " return X_train, categorical_cols, numerical_cols, train_loader, val_loader, test_loader" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import os\n", "import torch\n", "# 디바이스 설정 (CUDA 사용 가능하면 GPU로, 아니면 CPU로)\n", "import glob\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "\n", "def calculate_csi(Y_test, pred):\n", "\n", " cm = confusion_matrix(Y_test, pred) # 변수 이름을 cm으로 변경\n", " # 혼동 행렬에서 H, F, M 추출\n", " H = (cm[0, 0] + cm[1, 1])\n", " \n", " F = (cm[1, 0] + cm[2, 0] +\n", " cm[0, 1] + cm[2, 1])\n", "\n", " M = (cm[0, 2] + cm[1, 2])\n", " \n", " # CSI 계산\n", " CSI = H / (H + F + M + 1e-10)\n", " return CSI\n", "\n", "def csi_metric(y_true, pred):\n", " y_pred_binary = np.argmax(pred, axis=1)\n", " score = calculate_csi(y_true, y_pred_binary)\n", " return 'CSI', score, True # higher_better=True\n", "\n", "\n", "def eval_metric_csi(y_true, pred_prob):\n", "\n", " pred = np.argmax(pred_prob, axis=1)\n", " y_true = y_true\n", " y_pred = pred\n", " csi = calculate_csi(y_true, y_pred)\n", " return -1*csi\n", "\n", "\n", "from sklearn.metrics import matthews_corrcoef, accuracy_score\n", "\n", "def multiclass_mcc(y_val, y_pred):\n", " \"\"\"\n", " 다중 분류에서도 sklearn의 matthews_corrcoef를 그대로 사용할 수 있음.\n", " \"\"\"\n", " return matthews_corrcoef(y_val, y_pred)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "import os\n", "import torch\n", "# 디바이스 설정 (CUDA 사용 가능하면 GPU로, 아니면 CPU로)\n", "import glob\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "import warnings\n", "warnings.filterwarnings('ignore')\n", "\n", "def calculate_csi(Y_test, pred):\n", "\n", " cm = confusion_matrix(Y_test, pred) # 변수 이름을 cm으로 변경\n", " # 혼동 행렬에서 H, F, M 추출\n", " H = (cm[0, 0] + cm[1, 1])\n", " \n", " F = (cm[1, 0] + cm[2, 0] +\n", " cm[0, 1] + cm[2, 1])\n", "\n", " M = (cm[0, 2] + cm[1, 2])\n", " \n", " # CSI 계산\n", " CSI = H / (H + F + M + 1e-10)\n", " return CSI\n", "\n", "# Soft Voting 앙상블\n", "def get_proba(region, model_choose, data_sample, fold, target='multi'):\n", " _, _, _, _,val_loader , test_loader = prepare_dataloader(region=region, data_sample=data_sample, target=target,fold=fold ,random_state=120)\n", "\n", " folder_path = f'../save_model/{model_choose}/{data_sample}'\n", " model_paths = [path for path in glob.glob(f'{folder_path}/*.pth') if f'{region}' in path]\n", "\n", " model = torch.load(model_paths[fold-1], weights_only=False).to(device)\n", " model.eval()\n", "\n", " test_preds = []\n", "\n", "\n", " with torch.no_grad():\n", " for x_num_batch, x_cat_batch, _ in test_loader:\n", " output = model(x_num_batch.to(device), x_cat_batch.to(device))\n", " output = torch.softmax(output, dim=1)\n", " test_preds.extend(output.cpu().numpy())\n", "\n", "\n", " return test_preds\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "df_seoul = pd.read_csv(\"../../data/data_for_modeling/seoul_train.csv\")\n", "df_seoul_test = pd.read_csv(\"../../data/data_for_modeling/seoul_test.csv\")\n", "\n", "df_busan = pd.read_csv(\"../../data/data_for_modeling/busan_train.csv\")\n", "df_busan_test = pd.read_csv(\"../../data/data_for_modeling/busan_test.csv\")\n", "\n", "df_daegu = pd.read_csv(\"../../data/data_for_modeling/daegu_train.csv\")\n", "df_daegu_test = pd.read_csv(\"../../data/data_for_modeling/daegu_test.csv\")\n", "\n", "df_daejeon = pd.read_csv(\"../../data/data_for_modeling/daejeon_train.csv\")\n", "df_daejeon_test = pd.read_csv(\"../../data/data_for_modeling/daejeon_test.csv\")\n", "\n", "df_incheon = pd.read_csv(\"../../data/data_for_modeling/incheon_train.csv\")\n", "df_incheon_test = pd.read_csv(\"../../data/data_for_modeling/incheon_test.csv\")\n", "\n", "df_gwangju = pd.read_csv(\"../../data/data_for_modeling/gwangju_train.csv\")\n", "df_gwangju_test = pd.read_csv(\"../../data/data_for_modeling/gwangju_test.csv\")" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def preprocessing_df(df):\n", " df = df[df.columns].copy()\n", " df['year'] = df['year'].astype('int')\n", " df['month'] = df['month'].astype('int')\n", " df['hour'] = df['hour'].astype('int')\n", " df['binary_class'] = df['binary_class'].astype('int')\n", " df['multi_class'] = df['multi_class'].astype('int')\n", "\n", " df.loc[df['wind_dir']=='정온', 'wind_dir'] = \"0\"\n", " df['wind_dir'] = df['wind_dir'].astype('int')\n", " df= df[['temp_C', 'precip_mm', 'wind_speed', 'wind_dir', 'hm',\n", " 'vap_pressure', 'dewpoint_C', 'loc_pressure', 'sea_pressure',\n", " 'solarRad', 'snow_cm', 'cloudcover', 'lm_cloudcover', 'low_cloudbase',\n", " 'groundtemp', 'O3', 'NO2', 'PM10', 'PM25', 'year',\n", " 'month', 'hour', 'ground_temp - temp_C', 'hour_sin', 'hour_cos',\n", " 'month_sin', 'month_cos','multi_class']]\n", " return df\n", "\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "df_seoul_test= preprocessing_df(df_seoul_test).copy()\n", "df_busan_test= preprocessing_df(df_busan_test).copy()\n", "df_daegu_test= preprocessing_df(df_daegu_test).copy()\n", "df_gwangju_test= preprocessing_df(df_gwangju_test).copy()\n", "df_daejeon_test= preprocessing_df(df_daejeon_test).copy()\n", "df_incheon_test= preprocessing_df(df_incheon_test).copy()\n", "\n", "df_seoul= preprocessing_df(df_seoul).copy()\n", "df_busan= preprocessing_df(df_busan).copy()\n", "df_daegu= preprocessing_df(df_daegu).copy()\n", "df_gwangju= preprocessing_df(df_gwangju).copy()\n", "df_daejeon= preprocessing_df(df_daejeon).copy()\n", "df_incheon= preprocessing_df(df_incheon).copy()\n", "\n", "df_seoul_test.drop(columns=['year'], inplace=True)\n", "df_busan_test.drop(columns=['year'], inplace=True)\n", "df_daegu_test.drop(columns=['year'], inplace=True)\n", "df_daejeon_test.drop(columns=['year'], inplace=True)\n", "df_incheon_test.drop(columns=['year'], inplace=True)\n", "df_gwangju_test.drop(columns=['year'], inplace=True)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "import joblib\n", "\n", "lgb_seoul= joblib.load('../save_model/LGB_optima/lgb_seoul_smote.pkl')\n", "lgb_busan= joblib.load('../save_model/LGB_optima/lgb_busan_smote.pkl')\n", "lgb_incheon= joblib.load('../save_model/LGB_optima/lgb_incheon_smote.pkl')\n", "lgb_daegu= joblib.load('../save_model/LGB_optima/lgb_daegu_smote.pkl')\n", "lgb_daejeon= joblib.load('../save_model/LGB_optima/lgb_daejeon_smote.pkl')\n", "lgb_gwangju= joblib.load('../save_model/LGB_optima/lgb_gwangju_smote.pkl')\n", "\n", "xgb_seoul= joblib.load('../save_model/XGB_optima/xgb_seoul_smote.pkl')\n", "xgb_busan= joblib.load('../save_model/XGB_optima/xgb_busan_ctgan20000.pkl')\n", "xgb_incheon= joblib.load('../save_model/XGB_optima/xgb_incheon_smote.pkl')\n", "xgb_daegu= joblib.load('../save_model/XGB_optima/xgb_daegu_smote.pkl')\n", "xgb_daejeon= joblib.load('../save_model/XGB_optima/xgb_daejeon_smote.pkl')\n", "xgb_gwangju= joblib.load('../save_model/XGB_optima/xgb_gwangju_smote.pkl')\n", "\n", "lgb_seoul_1= lgb_seoul[0]\n", "lgb_seoul_2= lgb_seoul[1]\n", "lgb_seoul_3= lgb_seoul[2]\n", "\n", "lgb_busan_1= lgb_busan[0]\n", "lgb_busan_2= lgb_busan[1]\n", "lgb_busan_3= lgb_busan[2]\n", "\n", "lgb_incheon_1= lgb_incheon[0]\n", "lgb_incheon_2= lgb_incheon[1]\n", "lgb_incheon_3= lgb_incheon[2]\n", "\n", "lgb_daegu_1= lgb_daegu[0]\n", "lgb_daegu_2= lgb_daegu[1]\n", "lgb_daegu_3= lgb_daegu[2]\n", "\n", "lgb_daejeon_1= lgb_daejeon[0]\n", "lgb_daejeon_2= lgb_daejeon[1]\n", "lgb_daejeon_3= lgb_daejeon[2]\n", "\n", "lgb_gwangju_1= lgb_gwangju[0]\n", "lgb_gwangju_2= lgb_gwangju[1]\n", "lgb_gwangju_3= lgb_gwangju[2]\n", "\n", "\n", "xgb_seoul_1= xgb_seoul[0]\n", "xgb_seoul_2= xgb_seoul[1]\n", "xgb_seoul_3= xgb_seoul[2]\n", "\n", "xgb_busan_1= xgb_busan[0]\n", "xgb_busan_2= xgb_busan[1]\n", "xgb_busan_3= xgb_busan[2]\n", "\n", "xgb_incheon_1= xgb_incheon[0]\n", "xgb_incheon_2= xgb_incheon[1]\n", "xgb_incheon_3= xgb_incheon[2]\n", "\n", "xgb_daegu_1= xgb_daegu[0]\n", "xgb_daegu_2= xgb_daegu[1]\n", "xgb_daegu_3= xgb_daegu[2]\n", "\n", "xgb_daejeon_1= xgb_daejeon[0]\n", "xgb_daejeon_2= xgb_daejeon[1]\n", "xgb_daejeon_3= xgb_daejeon[2]\n", "\n", "xgb_gwangju_1= xgb_gwangju[0]\n", "xgb_gwangju_2= xgb_gwangju[1]\n", "xgb_gwangju_3= xgb_gwangju[2]\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **Soft Voting**" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **서울**" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "voting = []\n", "mcc = []\n", "accuracy = []\n" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Index(['temp_C', 'precip_mm', 'wind_speed', 'wind_dir', 'hm', 'vap_pressure',\n", " 'dewpoint_C', 'loc_pressure', 'sea_pressure', 'solarRad', 'snow_cm',\n", " 'cloudcover', 'lm_cloudcover', 'low_cloudbase', 'groundtemp', 'O3',\n", " 'NO2', 'PM10', 'PM25', 'month', 'hour', 'ground_temp - temp_C',\n", " 'hour_sin', 'hour_cos', 'month_sin', 'month_cos', 'multi_class'],\n", " dtype='object')" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_seoul_test.columns" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CSI score of soft(test) : 0.3248062015503624\n" ] } ], "source": [ "from scipy.stats import mode # 최다 득표수 계산을 위한 mode 함수\n", "\n", "probas = []\n", "\n", "# 1 Fold\n", "test_preds = get_proba('seoul', 'deepgbm', 'ctgan20000', 1)\n", "probas.append(test_preds)\n", "test_preds = get_proba('seoul', 'resnet_like', 'smote', 1)\n", "probas.append(test_preds)\n", "# probas.append(xgb_seoul_1.predict_proba(df_seoul_test.iloc[:,:-1]))\n", "\n", "# 2 Fold\n", "test_preds = get_proba('seoul', 'deepgbm', 'ctgan20000', 2)\n", "probas.append(test_preds)\n", "test_preds = get_proba('seoul', 'resnet_like', 'smote', 2)\n", "probas.append(test_preds)\n", "# probas.append(xgb_seoul_2.predict_proba(df_seoul_test.iloc[:,:-1]))\n", "\n", "# 3 Fold\n", "test_preds = get_proba('seoul', 'deepgbm', 'ctgan20000', 3)\n", "probas.append(test_preds)\n", "test_preds = get_proba('seoul', 'resnet_like', 'smote', 3)\n", "probas.append(test_preds)\n", "# probas.append(xgb_seoul_3.predict_proba(df_seoul_test.iloc[:,:-1]))\n", "\n", "voting.append(calculate_csi(df_seoul_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "mcc.append(multiclass_mcc(df_seoul_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "accuracy.append(accuracy_score(df_seoul_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "\n", "\n", "print(\"CSI score of soft(test) :\", calculate_csi(df_seoul_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 2, 11, 0],\n", " [ 6, 417, 58],\n", " [ 7, 789, 7470]])" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "confusion_matrix(df_seoul_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **부산**" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CSI score of soft(test) : 0.46608315098458075\n" ] } ], "source": [ "from scipy.stats import mode # 최다 득표수 계산을 위한 mode 함수\n", "\n", "probas = []\n", "\n", "# 1 Fold\n", "test_preds = get_proba('busan', 'deepgbm', 'pure', 1)\n", "probas.append(test_preds)\n", "test_preds = get_proba('busan', 'resnet_like', 'ctgan10000', 1)\n", "probas.append(test_preds)\n", "\n", "\n", "# 2 Fold\n", "test_preds = get_proba('busan', 'deepgbm', 'pure', 2)\n", "probas.append(test_preds)\n", "test_preds = get_proba('busan', 'resnet_like', 'ctgan10000', 2)\n", "probas.append(test_preds)\n", "\n", "\n", "# 3 Fold\n", "test_preds = get_proba('busan', 'deepgbm', 'pure', 3)\n", "probas.append(test_preds)\n", "test_preds = get_proba('busan', 'resnet_like', 'ctgan10000', 3)\n", "probas.append(test_preds)\n", "\n", "\n", "voting.append(calculate_csi(df_busan_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "mcc.append(multiclass_mcc(df_busan_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "accuracy.append(accuracy_score(df_busan_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "\n", "print(\"CSI score of soft(test) :\", calculate_csi(df_busan_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 11, 13, 0],\n", " [ 11, 202, 68],\n", " [ 2, 150, 8303]])" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "confusion_matrix(df_busan_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **인천**" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CSI score of hard(test) : 0.572269457161506\n" ] } ], "source": [ "from scipy.stats import mode # 최다 득표수 계산을 위한 mode 함수\n", "\n", "\n", "# 1 Fold\n", "probas = []\n", "test_preds = get_proba('incheon', 'deepgbm', 'pure', 1)\n", "probas.append(test_preds)\n", "test_preds = get_proba('incheon', 'resnet_like', 'smote', 1)\n", "probas.append(test_preds)\n", "test_preds = get_proba('incheon', 'ft_transformer', 'pure', 1)\n", "probas.append(test_preds)\n", "\n", "probas.append(lgb_incheon_1.predict_proba(df_incheon_test.iloc[:,:-1]))\n", "probas.append(xgb_incheon_1.predict_proba(df_incheon_test.iloc[:,:-1]))\n", "\n", "# 2 Fold\n", "test_preds = get_proba('incheon', 'deepgbm', 'pure', 2)\n", "probas.append(test_preds)\n", "test_preds = get_proba('incheon', 'resnet_like', 'smote', 2)\n", "probas.append(test_preds)\n", "test_preds = get_proba('incheon', 'ft_transformer', 'pure', 2)\n", "probas.append(test_preds)\n", "\n", "probas.append(lgb_incheon_2.predict_proba(df_incheon_test.iloc[:,:-1]))\n", "probas.append(xgb_incheon_2.predict_proba(df_incheon_test.iloc[:,:-1]))\n", "\n", "# 3 Fold\n", "test_preds = get_proba('incheon', 'deepgbm', 'pure', 3)\n", "probas.append(test_preds)\n", "test_preds = get_proba('incheon', 'resnet_like', 'smote', 3)\n", "probas.append(test_preds)\n", "test_preds = get_proba('incheon', 'ft_transformer', 'pure', 3)\n", "probas.append(test_preds)\n", "\n", "probas.append(lgb_incheon_3.predict_proba(df_incheon_test.iloc[:,:-1]))\n", "probas.append(xgb_incheon_3.predict_proba(df_incheon_test.iloc[:,:-1]))\n", "\n", "\n", "\n", "voting.append(calculate_csi(df_incheon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "mcc.append(multiclass_mcc(df_incheon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "accuracy.append(accuracy_score(df_incheon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "\n", "\n", "print(\"CSI score of hard(test) :\", calculate_csi(df_incheon_test.iloc[:,-1],mode(np.argmax(probas, axis=2), axis=0).mode[0]))\n" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 87, 74, 21],\n", " [ 22, 788, 395],\n", " [ 2, 140, 7231]])" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "confusion_matrix(df_incheon_test.iloc[:,-1],mode(np.argmax(probas, axis=2), axis=0).mode[0])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **대구**" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CSI score of soft(test) : 0.2852112676055334\n" ] } ], "source": [ "from scipy.stats import mode # 최다 득표수 계산을 위한 mode 함수\n", "\n", "probas= []\n", "\n", "# 1 Fold\n", "test_preds = get_proba('daegu', 'deepgbm', 'smote', 1)\n", "probas.append(test_preds)\n", "test_preds = get_proba('daegu', 'ft_transformer', 'pure', 1)\n", "probas.append(test_preds)\n", "\n", "# 2 Fold\n", "test_preds = get_proba('daegu', 'deepgbm', 'smote', 2)\n", "probas.append(test_preds)\n", "test_preds = get_proba('daegu', 'ft_transformer', 'pure', 2)\n", "probas.append(test_preds)\n", "\n", "# 3 Fold\n", "test_preds = get_proba('daegu', 'deepgbm', 'smote', 3)\n", "probas.append(test_preds)\n", "test_preds = get_proba('daegu', 'ft_transformer', 'pure', 3)\n", "probas.append(test_preds)\n", "\n", "voting.append(calculate_csi(df_daegu_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "mcc.append(multiclass_mcc(df_daegu_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "accuracy.append(accuracy_score(df_daegu_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "\n", "print(\"CSI score of soft(test) :\", calculate_csi(df_daegu_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 1, 0, 0],\n", " [ 1, 80, 47],\n", " [ 2, 153, 8476]])" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "confusion_matrix(df_daegu_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **대전**" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CSI score of soft(test) : 0.31884057971011603\n" ] } ], "source": [ "from scipy.stats import mode # 최다 득표수 계산을 위한 mode 함수\n", "\n", "probas = []\n", "\n", "# 1 Fold\n", "test_preds = get_proba('daejeon', 'deepgbm', 'pure', 1)\n", "probas.append(test_preds)\n", "test_preds = get_proba('daejeon', 'ft_transformer', 'pure', 1)\n", "probas.append(test_preds)\n", "\n", "\n", "# 2 Fold\n", "test_preds = get_proba('daejeon', 'deepgbm', 'pure', 2)\n", "probas.append(test_preds)\n", "test_preds = get_proba('daejeon', 'ft_transformer', 'pure', 2)\n", "probas.append(test_preds)\n", "\n", "\n", "# 3 Fold\n", "test_preds = get_proba('daejeon', 'deepgbm', 'pure', 3)\n", "probas.append(test_preds)\n", "test_preds = get_proba('daejeon', 'ft_transformer', 'pure', 3)\n", "probas.append(test_preds)\n", "\n", "voting.append(calculate_csi(df_daejeon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "mcc.append(multiclass_mcc(df_daejeon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "accuracy.append(accuracy_score(df_daejeon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "\n", "\n", "print(\"CSI score of soft(test) :\", calculate_csi(df_daejeon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 15, 23, 15],\n", " [ 10, 337, 271],\n", " [ 0, 433, 7656]])" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "confusion_matrix(df_daejeon_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **광주**" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CSI score of soft(test) : 0.4759725400457121\n" ] } ], "source": [ "from scipy.stats import mode # 최다 득표수 계산을 위한 mode 함수\n", "\n", "probas = []\n", "\n", "# 1 Fold\n", "test_preds = get_proba('gwangju', 'deepgbm', 'pure', 1)\n", "probas.append(test_preds)\n", "test_preds = get_proba('gwangju', 'ft_transformer', 'pure', 1)\n", "probas.append(test_preds)\n", "\n", "\n", "# 2 Fold\n", "test_preds = get_proba('gwangju', 'deepgbm', 'pure', 2)\n", "probas.append(test_preds)\n", "test_preds = get_proba('gwangju', 'ft_transformer', 'pure', 2)\n", "probas.append(test_preds)\n", "\n", "\n", "# 3 Fold\n", "test_preds = get_proba('gwangju', 'deepgbm', 'pure', 3)\n", "probas.append(test_preds)\n", "test_preds = get_proba('gwangju', 'ft_transformer', 'pure', 3)\n", "probas.append(test_preds)\n", "\n", "voting.append(calculate_csi(df_gwangju_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "mcc.append(multiclass_mcc(df_gwangju_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "accuracy.append(accuracy_score(df_gwangju_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))\n", "\n", "\n", "print(\"CSI score of soft(test) :\", calculate_csi(df_gwangju_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1)))" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 10, 12, 8],\n", " [ 2, 406, 235],\n", " [ 0, 201, 7886]])" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "confusion_matrix(df_gwangju_test.iloc[:,-1], np.argmax(np.mean(probas, axis=0), axis=1))" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAHDCAYAAAATEUquAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDfUlEQVR4nO3deVwVdf///+cBZBfEDcgQUtw33EMvxQpDLcsrKysL5GvqpWJeF5VFlmiLWJbppaZZH9PcMs1WU0vUSrM0FZdCxd00cClFUKHg/fujH+fqCDhiynF53G+3c6t5z3tmXnPOeM55MjPvYzPGGAEAAAAASuXi7AIAAAAA4EpHcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAXFPWr1+vdu3aycfHRzabTWlpac4uSSNHjpTNZnN2GeVq1apVstlsWrVqVZmXnTFjhmw2m/bt23fJ6wKAi0VwAoDLaPfu3RowYIBq1aolT09P+fn5qX379powYYLOnDlj75efn68JEyaoefPm8vPzU6VKldSoUSP1799f27dvt/cr+kL5ww8/WG573759io+PV+3ateXp6amgoCB17NhRycnJl2VfrwS///677rvvPv366696/fXXNWvWLIWGhhbr99hjj8lms2nXrl2lrmv48OGy2WzasmXLBW379OnTGjly5EUFhcupT58+stls8vPzczjmimRkZMhms8lms+nVV191QoUAcHUgOAHAZbJ48WI1adJE77//vrp3766JEycqJSVFNWvW1JNPPqmhQ4fa+/bs2VOPP/64GjdurDFjxmjUqFHq2LGjlixZou+++67M2961a5eaN2+uZcuW6cEHH9SkSZM0ePBgValSRS+//PKl3M0ryu7du7V//3498cQT6t+/vx5++GEFBAQU69e7d29J0ty5c0td17x589SkSRM1bdr0grZ9+vRpjRo1qsTg9Oyzz5YYWsqLm5ubTp8+rU8//bTYvDlz5sjT09MJVQHA1cXN2QUAwLVo7969euCBBxQaGqoVK1YoODjYPm/w4MHatWuXFi9eLOnPS8s+++wzvfTSS3rmmWcc1jNp0iSdOHGizNt//fXXlZOTo7S0tGJnXI4cOVL2HfobcnNz5ePjUy7bKtq3SpUqnbdf27ZtFR4ernnz5mnEiBHF5q9du1Z79+7VmDFjLkldbm5ucnNz3keuh4eH2rdvr3nz5un+++93mDd37lzdcccd+uCDD5xUHQBcHTjjBACXwSuvvKKcnBz93//9n0NoKhIeHm4/47R7925JUvv27Yv1c3V1VZUqVcq8/d27d+vGG28s8TK16tWrF2tbsmSJoqKiVLFiRfn5+al169bFzsYsWLBALVu2lJeXl6pWraqHH35Yhw4dcujTp08f+fr6avfu3erWrZsqVqxoP7tTWFio8ePHq1GjRvL09FRgYKAGDBig33777YL2acWKFerQoYN8fHxUqVIl3X333UpPT3fYdlRUlCTpvvvuk81mU6dOnUpdX+/evbV9+3Zt3Lix2Ly5c+fKZrPpwQcflPRnIOvbt68CAwPl6empZs2aaebMmfb++/btU7Vq1SRJo0aNsl/6NnLkSEkl3+Nks9mUkJCgjz76SI0bN5aHh4caNWqkpUuXFqtn1apVatWqlTw9PVW7dm29+eabZb5v6qGHHtKSJUscgvj69euVkZGhhx56qMRl9uzZo/vuu0+VK1eWt7e3br75Znvg/6uff/5ZPXr0kI+Pj6pXr67//Oc/ysvLK3Gd33//vbp06SJ/f395e3srKipKa9asueD9AABnITgBwGXw6aefqlatWmrXrp1l36JwM2fOHP3xxx+XZPuhoaE6ePCgVqxYYdl3xowZuuOOO/Trr78qKSlJY8aMUUREhMMX+BkzZuj++++Xq6urUlJS1K9fPy1atEj/+Mc/ip0R++OPPxQTE6Pq1avr1VdfVc+ePSVJAwYM0JNPPmm/xys+Pl5z5sxRTEyMfv/99/PWuHz5csXExOjIkSMaOXKkEhMT9e2336p9+/b2AQQGDBhgP2P32GOPadasWRo+fHip6yztcr2CggK9//776tChg2rWrKkzZ86oU6dOmjVrlnr37q2xY8fK399fffr00YQJEyRJ1apV05QpUyRJ//znPzVr1izNmjVL99xzz3n3a/Xq1Ro0aJAeeOABvfLKKzp79qx69uyp48eP2/ts2rRJXbp00fHjxzVq1Cj17dtXzz//vD766KPzrvtc99xzj2w2mxYtWmRvmzt3rurXr68WLVoU65+VlaV27dpp2bJlGjRokF566SWdPXtWd911lz788EN7vzNnzui2227TsmXLlJCQoOHDh+ubb77RsGHDiq1zxYoV6tixo7Kzs5WcnKzRo0frxIkTuvXWW7Vu3boy7Q8AlDsDALikTp48aSSZu++++4L6FxYWmqioKCPJBAYGmgcffNBMnjzZ7N+/v1jfd955x0gy69evP+86t23bZry8vIwkExERYYYOHWo++ugjk5ub69DvxIkTpmLFiqZt27bmzJkzxeoyxpj8/HxTvXp107hxY4c+n332mZFkRowYYW+Li4szkszTTz/tsK5vvvnGSDJz5sxxaF+6dGmJ7eeKiIgw1atXN8ePH7e3bd682bi4uJjY2Fh728qVK40ks2DBgvOur0jr1q3NjTfeaAoKCorV9OabbxpjjBk/fryRZGbPnm3vk5+fbyIjI42vr6/Jzs42xhhz9OhRI8kkJycX205ycrI59yNXknF3dze7du1y2CdJZuLEifa27t27G29vb3Po0CF7W0ZGhnFzcyu2zpLExcUZHx8fY4wx9957r7ntttuMMcYUFBSYoKAgM2rUKLN3714jyYwdO9a+3L///W8jyXzzzTf2tlOnTpmbbrrJhIWF2Z+zoufn/ffft/fLzc014eHhRpJZuXKlMebP46lOnTomJibGfmwZY8zp06fNTTfdZDp37mxvKzrO9+7da7l/AFBeOOMEAJdYdna2JKlixYoX1N9ms2nZsmV68cUXFRAQoHnz5mnw4MEKDQ1Vr169Luoep0aNGiktLU0PP/yw9u3bpwkTJqhHjx4KDAzUW2+9Ze/35Zdf6tSpU3r66aeLDRBQdBnYDz/8oCNHjmjQoEEOfe644w7Vr1+/xEu3Bg4c6DC9YMEC+fv7q3Pnzjp27Jj90bJlS/n6+mrlypWl7ssvv/yitLQ09enTR5UrV7a3N23aVJ07d9bnn39etifnLx5++GH9/PPP+vrrr+1tc+fOlbu7u+677z5J0ueff66goCD7ZXuSVKFCBT322GPKycnRV199ddHbj46OVu3ate3TTZs2lZ+fn/bs2SPpz7Nfy5cvV48ePXTDDTfY+4WHh6tr165l3t5DDz2kVatWKTMzUytWrFBmZmapl+l9/vnnatOmjf7xj3/Y23x9fdW/f3/t27dPP/30k71fcHCw7r33Xns/b29v9e/f32F9aWlp9ssCjx8/bj8GcnNzddttt+nrr79WYWFhmfcJAMoLwQkALjE/Pz9J0qlTpy54GQ8PDw0fPlzp6ek6fPiw5s2bp5tvvlnvv/++EhISLqqOunXratasWTp27Ji2bNmi0aNHy83NTf3799fy5csl/e/+qsaNG5e6nv3790uS6tWrV2xe/fr17fOLuLm56cYbb3Roy8jI0MmTJ1W9enVVq1bN4ZGTk3PeASvOt/0GDRrYv3xfjAceeECurq72y/XOnj2rDz/8UF27drWPxrd//37VqVNHLi6OH5kNGjRwqO9i1KxZs1hbQECA/b6vI0eO6MyZMwoPDy/Wr6Q2K0X3nc2fP19z5sxR69atS13P/v37S33Oi+YX/Tc8PLzY/VbnLpuRkSFJiouLK3YMvP3228rLy9PJkyfLvE8AUF4YVQ8ALjE/Pz/dcMMN2rZt20UtHxwcrAceeEA9e/ZUo0aN9P7772vGjBkXPSqbq6urmjRpoiZNmigyMlK33HKL5syZo+jo6ItanxUPD49iIaOwsFDVq1fXnDlzSlymaGCF8la9enV17txZH3zwgSZPnqxPP/1Up06dst//dLm5urqW2G6MuSzb8/Dw0D333KOZM2dqz5499sErykPR2aSxY8cqIiKixD6+vr7lVg8AlBXBCQAugzvvvFPTpk3T2rVrFRkZeVHrqFChgpo2baqMjAwdO3ZMQUFBf7uuVq1aSfrz8jdJ9svEtm3bVuqZh6LBK3bs2KFbb73VYd6OHTtKHLnvXLVr19by5cvVvn17eXl5lanmv27/XNu3b1fVqlX/1nDnvXv31tKlS7VkyRLNnTtXfn5+6t69u8P2t2zZosLCQodAWPTDxEX1lWWEuwtVvXp1eXp6lvhDvef78d7zeeihhzR9+nS5uLjogQceKLVfaGhoqc950fyi/27btk3GGIfn4Nxli441Pz+/yxbaAeBy4lI9ALgMhg0bJh8fHz366KPKysoqNn/37t32EdkyMjJ04MCBYn1OnDihtWvXKiAgoMxnZL755psSR6oruh+o6DKq22+/XRUrVlRKSorOnj3r0LforEerVq1UvXp1TZ061WGI6SVLlig9PV133HGHZT3333+/CgoK9MILLxSb98cff5z3Pq7g4GBFRERo5syZDv22bdumL774Qt26dbPc/vn06NFD3t7eeuONN7RkyRLdc889DvdydevWTZmZmZo/f75DzRMnTpSvr699CHRvb29Juqh70krj6uqq6OhoffTRRzp8+LC9fdeuXVqyZMlFrfOWW27RCy+8oEmTJp03jHfr1k3r1q3T2rVr7W25ubmaNm2awsLC1LBhQ3u/w4cPa+HChfZ+p0+f1rRp0xzW17JlS9WuXVuvvvqqcnJyim3v6NGjF7U/AFBeOOMEAJdB7dq1NXfuXPXq1UsNGjRQbGysGjdurPz8fH377bdasGCB+vTpI0navHmzHnroIXXt2lUdOnRQ5cqVdejQIc2cOVOHDx/W+PHjS72kqzQvv/yyNmzYoHvuuUdNmzaVJG3cuFHvvvuuKleurH//+9+S/vzr/+uvv65HH31UrVu31kMPPaSAgABt3rxZp0+f1syZM1WhQgW9/PLLio+PV1RUlB588EFlZWVpwoQJCgsL03/+8x/LeqKiojRgwAClpKQoLS1Nt99+uypUqKCMjAwtWLBAEyZMcBhc4Fxjx45V165dFRkZqb59++rMmTOaOHGi/P39//blZr6+vurRo4f9PqdzL9Pr37+/3nzzTfXp00cbNmxQWFiYFi5cqDVr1mj8+PH2QUC8vLzUsGFDzZ8/X3Xr1lXlypXVuHHj894/diFGjhypL774Qu3bt9fAgQNVUFCgSZMmqXHjxkpLSyvz+lxcXPTss89a9nv66ac1b948de3aVY899pgqV66smTNnau/evfrggw/sZ9/69eunSZMmKTY2Vhs2bFBwcLBmzZplD5J/3e7bb7+trl27qlGjRoqPj1eNGjV06NAhrVy5Un5+fvr000/LvD8AUG6cPKofAFzTdu7cafr162fCwsKMu7u7qVixomnfvr2ZOHGiOXv2rDHGmKysLDNmzBgTFRVlgoODjZubmwkICDC33nqrWbhwocP6LnQ48jVr1pjBgwebxo0bG39/f1OhQgVTs2ZN06dPH7N79+5i/T/55BPTrl074+XlZfz8/EybNm3MvHnzHPrMnz/fNG/e3Hh4eJjKlSub3r17m59//tmhz1+Hvi7JtGnTTMuWLY2Xl5epWLGiadKkiRk2bJg5fPjweffHGGOWL19u2rdvb6+xe/fu5qeffnLoU9bhyIssXrzYSDLBwcEOQ5MXycrKMvHx8aZq1arG3d3dNGnSxLzzzjvF+n377bemZcuWxt3d3WFo8tKGIx88eHCxdYSGhpq4uDiHttTUVNO8eXPj7u5uateubd5++23z+OOPG09PT8t9s3pNjDElDkdujDG7d+829957r6lUqZLx9PQ0bdq0MZ999lmx5ffv32/uuusu4+3tbapWrWqGDh1qH9a9aDjyIps2bTL33HOPqVKlivHw8DChoaHm/vvvN6mpqfY+DEcO4EpkM+Yy3YEKAAAumx49eujHH3+0j1YHALi8uMcJAIAr3JkzZxymMzIy9Pnnn6tTp07OKQgArkOccQIA4AoXHBysPn36qFatWtq/f7+mTJmivLw8bdq0SXXq1HF2eQBwXWBwCAAArnBdunTRvHnzlJmZKQ8PD0VGRmr06NGEJgAoR5xxAgAAAAAL3OMEAAAAABYITgAAAABgwen3OE2ePFljx45VZmammjVrpokTJ6pNmzal9j9x4oSGDx+uRYsW6ddff1VoaKjGjx9/wb8cX1hYqMOHD6tixYqy2WyXajcAAAAAXGWMMTp16pRuuOEG+w97l8apwWn+/PlKTEzU1KlT1bZtW40fP14xMTHasWOHqlevXqx/fn6+OnfurOrVq2vhwoWqUaOG9u/fr0qVKl3wNg8fPqyQkJBLuBcAAAAArmYHDx7UjTfeeN4+Th0com3btmrdurUmTZok6c+zQSEhIRoyZIiefvrpYv2nTp2qsWPHavv27apQocJFbfPkyZOqVKmSDh48KD8/v79VPwAAAICrV3Z2tkJCQnTixAn5+/uft6/Tzjjl5+drw4YNSkpKsre5uLgoOjpaa9euLXGZTz75RJGRkRo8eLA+/vhjVatWTQ899JCeeuopubq6XtB2iy7P8/PzIzgBAAAAuKBbeJwWnI4dO6aCggIFBgY6tAcGBmr79u0lLrNnzx6tWLFCvXv31ueff65du3Zp0KBB+v3335WcnFziMnl5ecrLy7NPZ2dnX7qdAAAAAHBduKpG1SssLFT16tU1bdo0tWzZUr169dLw4cM1derUUpdJSUmRv7+//cH9TQAAAADKymnBqWrVqnJ1dVVWVpZDe1ZWloKCgkpcJjg4WHXr1nW4LK9BgwbKzMxUfn5+icskJSXp5MmT9sfBgwcv3U4AAAAAuC44LTi5u7urZcuWSk1NtbcVFhYqNTVVkZGRJS7Tvn177dq1S4WFhfa2nTt3Kjg4WO7u7iUu4+HhYb+fifuaAAAAAFwMp16ql5iYqLfeekszZ85Uenq6Bg4cqNzcXMXHx0uSYmNjHQaPGDhwoH799VcNHTpUO3fu1OLFizV69GgNHjzYWbsAAAAA4Drg1N9x6tWrl44ePaoRI0YoMzNTERERWrp0qX3AiAMHDjj8EFVISIiWLVum//znP2ratKlq1KihoUOH6qmnnnLWLgAAAAC4Djj1d5ycITs7W/7+/jp58iSX7QEAAADXsbJkg6tqVD0AAAAAcAaCEwAAAABYIDgBAAAAgAWCEwAAAABYIDgBAAAAgAWCEwAAAABYIDgBAAAAgAWCEwAAAABYIDgBAAAAgAU3ZxcAAFeT9hPbO7sElGLNkDXOLgEAcA3jjBMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWLgigtPkyZMVFhYmT09PtW3bVuvWrSu174wZM2Sz2Rwenp6e5VgtAAAAgOuN04PT/PnzlZiYqOTkZG3cuFHNmjVTTEyMjhw5Uuoyfn5++uWXX+yP/fv3l2PFAAAAAK43Tg9O48aNU79+/RQfH6+GDRtq6tSp8vb21vTp00tdxmazKSgoyP4IDAwsx4oBAAAAXG+cGpzy8/O1YcMGRUdH29tcXFwUHR2ttWvXlrpcTk6OQkNDFRISorvvvls//vhjqX3z8vKUnZ3t8AAAAACAsnBqcDp27JgKCgqKnTEKDAxUZmZmicvUq1dP06dP18cff6zZs2ersLBQ7dq1088//1xi/5SUFPn7+9sfISEhl3w/AAAAAFzbnH6pXllFRkYqNjZWERERioqK0qJFi1StWjW9+eabJfZPSkrSyZMn7Y+DBw+Wc8UAAAAArnZuztx41apV5erqqqysLIf2rKwsBQUFXdA6KlSooObNm2vXrl0lzvfw8JCHh8ffrhUAAADA9cupZ5zc3d3VsmVLpaam2tsKCwuVmpqqyMjIC1pHQUGBtm7dquDg4MtVJgAAAIDrnFPPOElSYmKi4uLi1KpVK7Vp00bjx49Xbm6u4uPjJUmxsbGqUaOGUlJSJEnPP/+8br75ZoWHh+vEiRMaO3as9u/fr0cffdSZuwEAAADgGub04NSrVy8dPXpUI0aMUGZmpiIiIrR06VL7gBEHDhyQi8v/Toz99ttv6tevnzIzMxUQEKCWLVvq22+/VcOGDZ21CwAAAACucTZjjHF2EeUpOztb/v7+OnnypPz8/JxdDoCrTPuJ7Z1dAkqxZsgaZ5cAALjKlCUbXHWj6gEAAABAeSM4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFN2cXAAAAAFxNXnr4XmeXgFIMn73wsq2bM04AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWGI4cuEAHnm/i7BJwHjVHbHV2CQAA4BrGGScAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALV0Rwmjx5ssLCwuTp6am2bdtq3bp1F7Tce++9J5vNph49elzeAgEAAABc15wenObPn6/ExEQlJydr48aNatasmWJiYnTkyJHzLrdv3z498cQT6tChQzlVCgAAAOB65fTgNG7cOPXr10/x8fFq2LChpk6dKm9vb02fPr3UZQoKCtS7d2+NGjVKtWrVKsdqAQAAAFyPnBqc8vPztWHDBkVHR9vbXFxcFB0drbVr15a63PPPP6/q1aurb9++5VEmAAAAgOucmzM3fuzYMRUUFCgwMNChPTAwUNu3by9xmdWrV+v//u//lJaWdkHbyMvLU15enn06Ozv7ousFAAAAcH1y+qV6ZXHq1Ck98sgjeuutt1S1atULWiYlJUX+/v72R0hIyGWuEgAAAMC1xqlnnKpWrSpXV1dlZWU5tGdlZSkoKKhY/927d2vfvn3q3r27va2wsFCS5Obmph07dqh27doOyyQlJSkxMdE+nZ2dTXgCAAAAUCZODU7u7u5q2bKlUlNT7UOKFxYWKjU1VQkJCcX6169fX1u3bnVoe/bZZ3Xq1ClNmDChxEDk4eEhDw+Py1I/AAAAgOuDU4OTJCUmJiouLk6tWrVSmzZtNH78eOXm5io+Pl6SFBsbqxo1aiglJUWenp5q3Lixw/KVKlWSpGLtAAAAAHCpOD049erVS0ePHtWIESOUmZmpiIgILV261D5gxIEDB+TiclXdigUAAADgGuP04CRJCQkJJV6aJ0mrVq0677IzZsy49AUBAAAAwF9wKgcAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMCCm7MLuFq0fPJdZ5eA89gwNtbZJQAAAOAaxhknAAAAALBAcAIAAAAAC1yqBwAAUAaTHv/U2SWgFAmvdXd2CbiGccYJAAAAACyU6YzTli1bLqhf06ZNL6oYAAAAALgSlSk4RUREyGazyRhTbF5Ru81mU0FBwSUrEAAAAACcrUzBae/evZerDgAAAAC4YpUpOIWGhl6uOgAAAADgilWmwSGOHTum/fv3O7T9+OOPio+P1/3336+5c+de0uIAAAAA4EpQpuA0ZMgQ/fe//7VPHzlyRB06dND69euVl5enPn36aNasWZe8SAAAAABwpjIFp++++0533XWXffrdd99V5cqVlZaWpo8//lijR4/W5MmTy1zE5MmTFRYWJk9PT7Vt21br1q0rte+iRYvUqlUrVapUST4+PoqIiCCsAQAAALisyhScMjMzFRYWZp9esWKF7rnnHrm5/Xmr1F133aWMjIwyFTB//nwlJiYqOTlZGzduVLNmzRQTE6MjR46U2L9y5coaPny41q5dqy1btig+Pl7x8fFatmxZmbYLAAAAABeqTMHJz89PJ06csE+vW7dObdu2tU/bbDbl5eWVqYBx48apX79+io+PV8OGDTV16lR5e3tr+vTpJfbv1KmT/vnPf6pBgwaqXbu2hg4dqqZNm2r16tVl2i4AAAAAXKgyBaebb75Z//3vf1VYWKiFCxfq1KlTuvXWW+3zd+7cqZCQkAteX35+vjZs2KDo6Oj/FeTioujoaK1du9ZyeWOMUlNTtWPHDnXs2LHEPnl5ecrOznZ4AAAAAEBZlCk4Pf/88/rkk0/k5eWlXr16adiwYQoICLDPf++99xQVFXXB6zt27JgKCgoUGBjo0B4YGKjMzMxSlzt58qR8fX3l7u6uO+64QxMnTlTnzp1L7JuSkiJ/f3/7oyzBDgAAAACkMv6OU7NmzZSenq41a9YoKCjI4TI9SXrwwQfVoEGDS1pgSSpWrKi0tDTl5OQoNTVViYmJqlWrljp16lSsb1JSkhITE+3T2dnZhCcAAAAAZVKm4LRixQolJCTou+++k5+fn8O8kydP6sknn9TUqVN10003XdD6qlatKldXV2VlZTm0Z2VlKSgoqNTlXFxcFB4eLkmKiIhQenq6UlJSSgxOHh4e8vDwuKB6AAAAAKAkZbpUb/z48erXr1+x0CRJ/v7+GjBggMaNG3fB63N3d1fLli2VmppqbyssLFRqaqoiIyMveD2FhYVlHpQCAAAAAC5UmYLT5s2b1aVLl1Ln33777dqwYUOZCkhMTNRbb72lmTNnKj09XQMHDlRubq7i4+MlSbGxsUpKSrL3T0lJ0Zdffqk9e/YoPT1dr732mmbNmqWHH364TNsFAAAAgAtVpkv1srKyVKFChdJX5uamo0ePlqmAXr166ejRoxoxYoQyMzMVERGhpUuX2geMOHDggFxc/pfvcnNzNWjQIP3888/y8vJS/fr1NXv2bPXq1atM2wUAAACAC1Wm4FSjRg1t27bNfn/RubZs2aLg4OAyF5GQkKCEhIQS561atcph+sUXX9SLL75Y5m0AAAAAwMUq06V63bp103PPPaezZ88Wm3fmzBklJyfrzjvvvGTFAQAAAMCVoExnnJ599lktWrRIdevWVUJCgurVqydJ2r59uyZPnqyCggINHz78shQKAAAAAM5SpuAUGBiob7/9VgMHDlRSUpKMMZIkm82mmJgYTZ48udiP2QIAAADA1a5MwUmSQkND9fnnn+u3337Trl27ZIxRnTp1FBAQcDnqAwDgivJVxyhnl4BSRH39lbNLAHANK3NwKhIQEKDWrVtfyloAAAAA4IpUpsEhAAAAAOB6RHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAtXRHCaPHmywsLC5OnpqbZt22rdunWl9n3rrbfUoUMHBQQEKCAgQNHR0eftDwAAAAB/l9OD0/z585WYmKjk5GRt3LhRzZo1U0xMjI4cOVJi/1WrVunBBx/UypUrtXbtWoWEhOj222/XoUOHyrlyAAAAANcLpwencePGqV+/foqPj1fDhg01depUeXt7a/r06SX2nzNnjgYNGqSIiAjVr19fb7/9tgoLC5WamlrOlQMAAAC4Xjg1OOXn52vDhg2Kjo62t7m4uCg6Olpr1669oHWcPn1av//+uypXrlzi/Ly8PGVnZzs8AAAAAKAsnBqcjh07poKCAgUGBjq0BwYGKjMz84LW8dRTT+mGG25wCF9/lZKSIn9/f/sjJCTkb9cNAAAA4Pri9Ev1/o4xY8bovffe04cffihPT88S+yQlJenkyZP2x8GDB8u5SgAAAABXOzdnbrxq1apydXVVVlaWQ3tWVpaCgoLOu+yrr76qMWPGaPny5WratGmp/Tw8POTh4XFJ6gUAAABwfXLqGSd3d3e1bNnSYWCHooEeIiMjS13ulVde0QsvvKClS5eqVatW5VEqAAAAgOuYU884SVJiYqLi4uLUqlUrtWnTRuPHj1dubq7i4+MlSbGxsapRo4ZSUlIkSS+//LJGjBihuXPnKiwszH4vlK+vr3x9fZ22HwAAAACuXU4PTr169dLRo0c1YsQIZWZmKiIiQkuXLrUPGHHgwAG5uPzvxNiUKVOUn5+ve++912E9ycnJGjlyZHmWDgAAAOA64fTgJEkJCQlKSEgocd6qVascpvft23f5CwIAAACAv7iqR9UDAAAAgPJAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAAC04PTpMnT1ZYWJg8PT3Vtm1brVu3rtS+P/74o3r27KmwsDDZbDaNHz++/AoFAAAAcN1yanCaP3++EhMTlZycrI0bN6pZs2aKiYnRkSNHSux/+vRp1apVS2PGjFFQUFA5VwsAAADgeuXU4DRu3Dj169dP8fHxatiwoaZOnSpvb29Nnz69xP6tW7fW2LFj9cADD8jDw6OcqwUAAABwvXJacMrPz9eGDRsUHR39v2JcXBQdHa21a9c6qywAAAAAKMbNWRs+duyYCgoKFBgY6NAeGBio7du3X7Lt5OXlKS8vzz6dnZ19ydYNAAAA4Prg9MEhLreUlBT5+/vbHyEhIc4uCQAAAMBVxmnBqWrVqnJ1dVVWVpZDe1ZW1iUd+CEpKUknT560Pw4ePHjJ1g0AAADg+uC04OTu7q6WLVsqNTXV3lZYWKjU1FRFRkZesu14eHjIz8/P4QEAAAAAZeG0e5wkKTExUXFxcWrVqpXatGmj8ePHKzc3V/Hx8ZKk2NhY1ahRQykpKZL+HFDip59+sv//oUOHlJaWJl9fX4WHhzttPwAAAABc25wanHr16qWjR49qxIgRyszMVEREhJYuXWofMOLAgQNycfnfSbHDhw+refPm9ulXX31Vr776qqKiorRq1aryLh8AAADAdcKpwUmSEhISlJCQUOK8c8NQWFiYjDHlUBUAAAAA/M81P6oeAAAAAPxdBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALV0Rwmjx5ssLCwuTp6am2bdtq3bp15+2/YMEC1a9fX56enmrSpIk+//zzcqoUAAAAwPXI6cFp/vz5SkxMVHJysjZu3KhmzZopJiZGR44cKbH/t99+qwcffFB9+/bVpk2b1KNHD/Xo0UPbtm0r58oBAAAAXC+cHpzGjRunfv36KT4+Xg0bNtTUqVPl7e2t6dOnl9h/woQJ6tKli5588kk1aNBAL7zwglq0aKFJkyaVc+UAAAAArhduztx4fn6+NmzYoKSkJHubi4uLoqOjtXbt2hKXWbt2rRITEx3aYmJi9NFHH5XYPy8vT3l5efbpkydPSpKys7PLVGtB3pky9Uf5KuvreTFOnS247NvAxSuPY0CS/jjzR7lsB2VXXsdA7h8cA1eq8joGzuSdLpftoOzK6xg4+/vv5bIdlF1Zj4Gi/sYYy75ODU7Hjh1TQUGBAgMDHdoDAwO1ffv2EpfJzMwssX9mZmaJ/VNSUjRq1Khi7SEhIRdZNa5E/hP/5ewS4Gwp/s6uAE7m/xTHwHXPn2PgejdssrMrgLO9+P7FvQ+cOnVK/hbvIU4NTuUhKSnJ4QxVYWGhfv31V1WpUkU2m82JlTlPdna2QkJCdPDgQfn5+Tm7HDgBxwA4BsAxAInjABwDxhidOnVKN9xwg2VfpwanqlWrytXVVVlZWQ7tWVlZCgoKKnGZoKCgMvX38PCQh4eHQ1ulSpUuvuhriJ+f33X5DwT/wzEAjgFwDEDiOMD1fQxYnWkq4tTBIdzd3dWyZUulpqba2woLC5WamqrIyMgSl4mMjHToL0lffvllqf0BAAAA4O9y+qV6iYmJiouLU6tWrdSmTRuNHz9eubm5io+PlyTFxsaqRo0aSklJkSQNHTpUUVFReu2113THHXfovffe0w8//KBp06Y5czcAAAAAXMOcHpx69eqlo0ePasSIEcrMzFRERISWLl1qHwDiwIEDcnH534mxdu3aae7cuXr22Wf1zDPPqE6dOvroo4/UuHFjZ+3CVcfDw0PJycnFLmHE9YNjABwD4BiAxHEAjoGysJkLGXsPAAAAAK5jTv8BXAAAAAC40hGcAAAAAMACwQkAAAAALBCccFFWrVolm82mEydOOLsUAOcxY8YMfrsOQKk6deqkf//7384uA9eIa/37IcHpKnT06FENHDhQNWvWlIeHh4KCghQTE6M1a9Y4uzRcRn369JHNZrM/qlSpoi5dumjLli3OLg2XWJ8+fdSjRw9nl4ErzF/fAypUqKDAwEB17txZ06dPV2FhobPLwyVUnq/1okWL9MILL1zSdeLvy8zM1NChQxUeHi5PT08FBgaqffv2mjJlik6fPu3s8krVrl07/fLLLxf8g7JXG4LTVahnz57atGmTZs6cqZ07d+qTTz5Rp06ddPz4cWeXhsusS5cu+uWXX/TLL78oNTVVbm5uuvPOO51dFoByUvQesG/fPi1ZskS33HKLhg4dqjvvvFN//PGHs8vDJVRer3XlypVVsWLFS7Y+/H179uxR8+bN9cUXX2j06NHatGmT1q5dq2HDhumzzz7T8uXLnV1iqdzd3RUUFCSbzebsUi4LgtNV5sSJE/rmm2/08ssv65ZbblFoaKjatGmjpKQk3XXXXfY+jz76qKpVqyY/Pz/deuut2rx5s8N6pkyZotq1a8vd3V316tXTrFmz7PP27dsnm82mtLQ0h+3abDatWrWqPHYTpSg6wxgUFKSIiAg9/fTTOnjwoI4ePVri6fG0tDTZbDbt27dPkrR//351795dAQEB8vHxUaNGjfT5559LkgoKCtS3b1/ddNNN8vLyUr169TRhwgSH7RedCXn11VcVHBysKlWqaPDgwfr999/L6ym47nTq1EmPPfaYhg0bpsqVKysoKEgjR4506HPixAkNGDBAgYGB8vT0VOPGjfXZZ5859Fm2bJkaNGggX19f+xeyv3r77bfVoEEDeXp6qn79+nrjjTcc5m/dulW33nqrvLy8VKVKFfXv3185OTn2+Rwb5aPoPaBGjRpq0aKFnnnmGX388cdasmSJZsyYIUkaN26cmjRpIh8fH4WEhGjQoEEOr5UkrV69Wh06dJCXl5dCQkL02GOPKTc31z7/l19+0R133CEvLy/ddNNNmjt3rsLCwjR+/HhJfE6Uh/J6rc+9VC8vL09PPPGEatSoIR8fH7Vt27bYa/rBBx+oUaNG8vDwUFhYmF577TWH+WFhYRo9erT+3//7f6pYsaJq1qypadOmXdLn51o2aNAgubm56YcfftD999+vBg0aqFatWrr77ru1ePFide/eXU888YTDH07Hjx8vm82mpUuX2tvCw8P19ttvS5LWr1+vzp07q2rVqvL391dUVJQ2btzosF2bzaa3335b//znP+Xt7a06derok08+cejzySefqE6dOvL09NQtt9yimTNnOnz3OPe7yMiRIxUREeGwjvHjxyssLOzSPFnljOB0lfH19ZWvr68++ugj5eXlldjnvvvu05EjR7RkyRJt2LBBLVq00G233aZff/1VkvThhx9q6NChevzxx7Vt2zYNGDBA8fHxWrlyZXnuCv6mnJwczZ49W+Hh4apSpcoFLTN48GDl5eXp66+/1tatW/Xyyy/L19dXklRYWKgbb7xRCxYs0E8//aQRI0bomWee0fvvv++wjpUrV2r37t1auXKlZs6cqRkzZtg/xHF5zJw5Uz4+Pvr+++/1yiuv6Pnnn9eXX34p6c/XrWvXrlqzZo1mz56tn376SWPGjJGrq6t9+dOnT+vVV1/VrFmz9PXXX+vAgQN64okn7PPnzJmjESNG6KWXXlJ6erpGjx6t5557TjNnzpQk5ebmKiYmRgEBAVq/fr0WLFig5cuXKyEhwaFOjg3nuPXWW9WsWTMtWrRIkuTi4qL//ve/+vHHHzVz5kytWLFCw4YNs/ffvXu3unTpop49e2rLli2aP3++Vq9e7fB6xsbG6vDhw1q1apU++OADTZs2TUeOHCn3fYOjy/FanyshIUFr167Ve++9py1btui+++5Tly5dlJGRIUnasGGD7r//fj3wwAPaunWrRo4cqeeee67Yv/XXXntNrVq10qZNmzRo0CANHDhQO3bsuPRPyjXm+PHj+uKLLzR48GD5+PiU2MdmsykqKkqrV69WQUGBJOmrr75S1apV7SH30KFD2r17tzp16iRJOnXqlOLi4rR69Wp99913qlOnjrp166ZTp045rHvUqFG6//77tWXLFnXr1k29e/e2f3/cu3ev7r33XvXo0UObN2/WgAEDNHz48MvzRFypDK46CxcuNAEBAcbT09O0a9fOJCUlmc2bNxtjjPnmm2+Mn5+fOXv2rMMytWvXNm+++aYxxph27dqZfv36Ocy/7777TLdu3Ywxxuzdu9dIMps2bbLP/+2334wks3LlSmOMMStXrjSSzG+//XZ5dhLFxMXFGVdXV+Pj42N8fHyMJBMcHGw2bNhgjCn5Ndm0aZORZPbu3WuMMaZJkyZm5MiRF7zNwYMHm549ezrUEBoaav744w9723333Wd69er193YODuLi4szdd99tjDEmKirK/OMf/3CY37p1a/PUU08ZY4xZtmyZcXFxMTt27ChxXe+8846RZHbt2mVvmzx5sgkMDLRP165d28ydO9dhuRdeeMFERkYaY4yZNm2aCQgIMDk5Ofb5ixcvNi4uLiYzM9NeM8fG5fXX4+JcvXr1Mg0aNChx3oIFC0yVKlXs03379jX9+/d36PPNN98YFxcXc+bMGZOenm4kmfXr19vnZ2RkGEnm9ddfN8Zc2OcELl55vdbG/PkeM3ToUGOMMfv37zeurq7m0KFDDsvcdtttJikpyRhjzEMPPWQ6d+7sMP/JJ580DRs2tE+Hhoaahx9+2D5dWFhoqlevbqZMmXKevYYxxnz33XdGklm0aJFDe5UqVeyf/8OGDTO//fabcXFxMevXrzeFhYWmcuXKJiUlxbRt29YYY8zs2bNNjRo1St1OQUGBqVixovn000/tbZLMs88+a5/OyckxksySJUuMMcY89dRTpnHjxg7rGT58uMN3j3O/iyQnJ5tmzZo5LPP666+b0NDQsjwtVwzOOF2FevbsqcOHD+uTTz5Rly5dtGrVKrVo0UIzZszQ5s2blZOToypVqtjPTvn6+mrv3r3avXu3JCk9PV3t27d3WGf79u2Vnp7ujN1BGdxyyy1KS0tTWlqa1q1bp5iYGHXt2lX79++/oOUfe+wxvfjii2rfvr2Sk5OLDSwxefJktWzZUtWqVZOvr6+mTZumAwcOOPRp1KiRw9mM4OBg/hJ9mTVt2tRh+q/PeVpamm688UbVrVu31OW9vb1Vu3btEpfPzc3V7t271bdvX4f3jBdffNHhPaNZs2YOf/1s3769CgsLHf6CzLHhPMYY+z0Fy5cv12233aYaNWqoYsWKeuSRR3T8+HH7DeWbN2/WjBkzHF7vmJgYFRYWau/evdqxY4fc3NzUokUL+/rDw8MVEBDglH2Do0v5Wp9r69atKigoUN26dR2W+eqrryy/Q2RkZNjPfkiO71s2m01BQUG8H/wN69atU1pamho1aqS8vDxVqlRJzZo106pVq7R161a5u7urf//+2rRpk3JycvTVV18pKirKvnxWVpb69eunOnXqyN/fX35+fsrJySn2Gf/X183Hx0d+fn72123Hjh1q3bq1Q/82bdpcxr2+8rg5uwBcHE9PT3Xu3FmdO3fWc889p0cffVTJyckaNGiQgoODS7zG/EKHJHZx+TNPG2PsbdyncGXw8fFReHi4ffrtt9+Wv7+/3nrrLd1+++2Szv+6Pfroo4qJidHixYv1xRdfKCUlRa+99pqGDBmi9957T0888YRee+01RUZGqmLFiho7dqy+//57h3VUqFDBYdpmszGi12V2vufcy8vropYvOk6K7od466231LZtW4d+fw1Bf7dOXF7p6em66aabtG/fPt15550aOHCgXnrpJVWuXFmrV69W3759lZ+fL29vb+Xk5GjAgAF67LHHiq2nZs2a2rlzp+X2+Jxwnkv5Wp8rJydHrq6u2rBhQ7F//0WXdV8o3g8uTnh4uGw2W7HLGmvVqiXJ8T2/U6dOWrVqlTw8PBQVFaXKlSurQYMGWr16tb766is9/vjj9r5xcXE6fvy4JkyYoNDQUHl4eCgyMlL5+fkO27nUr5uLi4vD+4R0db9XEJyuEQ0bNtRHH32kFi1aKDMzU25ubqXeeNegQQOtWbNGcXFx9rY1a9aoYcOGkqRq1apJ+vPm4ObNm0uSww3AuHLYbDa5uLjozJkzDq9b0V+GS3rdQkJC9K9//Uv/+te/lJSUpLfeektDhgzRmjVr1K5dOw0aNMjet+gvjLhyNW3aVD///LN27tx53rNOpQkMDNQNN9ygPXv2qHfv3iX2adCggWbMmKHc3Fz7Wac1a9bIxcVF9erV+1v14+9bsWKFtm7dqv/85z/asGGDCgsL9dprr9nDzbn3KbZo0UI//fSTwx9h/qpevXr6448/tGnTJrVs2VKStGvXLv3222/2PnxOOMelfq3P1bx5cxUUFOjIkSPq0KFDiX2KvkP81Zo1a1S3bt0y/7EFxVWpUkWdO3fWpEmTNGTIkFLvc5KkqKgoTZ8+XW5uburSpYukP8PUvHnztHPnTvv9TdKfr9Ebb7yhbt26SZIOHjyoY8eOlam2evXq2QeUKrJ+/frzLlOtWjVlZmY6nCm9mt8ruFTvKnP8+HHdeuutmj17trZs2aK9e/dqwYIFeuWVV3T33XcrOjpakZGR6tGjh7744gvt27dP3377rYYPH64ffvhBkvTkk09qxowZmjJlijIyMjRu3DgtWrTIfrO4l5eXbr75Zo0ZM0bp6en66quv9Oyzzzpzt/H/y8vLU2ZmpjIzM5Wenq4hQ4YoJydH3bt3V3h4uEJCQjRy5EhlZGRo8eLFxUY6+ve//61ly5Zp79692rhxo1auXKkGDRpIkurUqaMffvhBy5Yt086dO/Xcc89ZviHC+aKiotSxY0f17NlTX375pfbu3aslS5Y4jKxkZdSoUUpJSdF///tf7dy5U1u3btU777yjcePGSZJ69+4tT09PxcXFadu2bVq5cqWGDBmiRx55RIGBgZdr11CCoveAQ4cOaePGjRo9erTuvvtu3XnnnYqNjVV4eLh+//13TZw4UXv27NGsWbM0depUh3U89dRT+vbbb5WQkKC0tDRlZGTo448/tg8YUL9+fUVHR6t///5at26dNm3apP79+8vLy8v+xYfPicuvPF7rc9WtW1e9e/dWbGysFi1apL1792rdunVKSUnR4sWLJUmPP/64UlNT9cILL2jnzp2aOXOmJk2a5DDgDP6eN954Q3/88YdatWql+fPnKz09XTt27NDs2bO1fft2e0Dt2LGjTp06pc8++8wekjp16qQ5c+YoODjY4Y9pderU0axZs5Senq7vv/9evXv3vqArFv5qwIAB2r59u5566int3LlT77//vn1QkNKGH+/UqZOOHj2qV155Rbt379bkyZO1ZMmSsj8pVwrn3V6Fi3H27Fnz9NNPmxYtWhh/f3/j7e1t6tWrZ5599llz+vRpY4wx2dnZZsiQIeaGG24wFSpUMCEhIaZ3797mwIED9vW88cYbplatWqZChQqmbt265t1333XYzk8//WQiIyONl5eXiYiIMF988QWDQzhZXFyckWR/VKxY0bRu3dosXLjQ3mf16tWmSZMmxtPT03To0MEsWLDAYXCIhIQEU7t2bePh4WGqVatmHnnkEXPs2DFjzJ/HVp8+fYy/v7+pVKmSGThwoHn66acdbuos6YbloUOHmqioqMu899eXcweHKLpxu8jdd99t4uLi7NPHjx838fHxpkqVKsbT09M0btzYfPbZZ8aYPweH8Pf3d1j+ww8/NOe+/c+ZM8dEREQYd3d3ExAQYDp27Ohwc/KWLVvMLbfcYjw9PU3lypVNv379zKlTp0qsuQjHxqX11/cANzc3U61aNRMdHW2mT59uCgoK7P3GjRtngoODjZeXl4mJiTHvvvtusffrdevWmc6dOxtfX1/j4+NjmjZtal566SX7/MOHD5uuXbsaDw8PExoaaubOnWuqV69upk6dau9j9TmBi1eer/W57zH5+flmxIgRJiwszFSoUMEEBwebf/7zn2bLli32PgsXLjQNGzY0FSpUMDVr1jRjx451qD80NNQ+kEiRZs2ameTk5Evy/FwPDh8+bBISEsxNN91kKlSoYHx9fU2bNm3M2LFjTW5urr1fs2bNTFBQkH36+PHjxmazmQceeMBhfRs3bjStWrUynp6epk6dOmbBggXFXidJ5sMPP3RYzt/f37zzzjv26Y8//tiEh4cbDw8P06lTJzNlyhQjyT7YSEnfD6dMmWJCQkKMj4+PiY2NNS+99NJVOziEzZhzLjwEAAD4i59//lkhISH2wQhw7YiMjNRtt92mF1980dml4Cr00ksvaerUqTp48KCkP38zsGvXrjp79qzc3d2dXN2lxz1OAADAwYoVK5STk6MmTZrol19+0bBhwxQWFqaOHTs6uzRcInl5edq6dat+/PHHEgePAEryxhtvqHXr1qpSpYrWrFmjsWPH2i/9zMrK0scff6w6depck6FJIjgBAIBz/P7773rmmWe0Z88eVaxYUe3atdOcOXOKjbiFq9eSJUsUGxuru+66S/fee6+zy8FVIiMjQy+++KJ+/fVX1axZU48//riSkpIkyf6Dum+88YaTq7x8uFQPAAAAACwwqh4AAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWPj/AJ3jGjFFTxgpAAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "plt.figure(figsize=(10,5))\n", "sns.barplot(x=['Seoul', 'Busan', 'Incheon', 'Daegu', 'Daejeon', 'Gwangju'], y=voting)\n", "plt.title('CSI Score of Voting Model')\n", "plt.ylabel('CSI')\n", "\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.4078715882283252" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.mean(voting)" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[0.3248062015503624,\n", " 0.46608315098458075,\n", " 0.5763157894736463,\n", " 0.2852112676055334,\n", " 0.31884057971011603,\n", " 0.4759725400457121]" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "voting" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[0.5106142349456536,\n", " 0.640202275543952,\n", " 0.709448778435959,\n", " 0.45579515959653394,\n", " 0.453960121993875,\n", " 0.6218724605270242]" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mcc" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[0.9005707762557078,\n", " 0.9721461187214612,\n", " 0.9264840182648402,\n", " 0.9768264840182649,\n", " 0.9141552511415525,\n", " 0.9477168949771689]" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "accuracy" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 2 }