{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "from sklearn.preprocessing import StandardScaler, LabelEncoder\n", "import torch\n", "from torch.utils.data import DataLoader, TensorDataset\n", "import random\n", "\n", "# Python 및 Numpy 시드 고정\n", "seed = 42\n", "random.seed(seed)\n", "np.random.seed(seed)\n", "\n", "# PyTorch 시드 고정\n", "torch.manual_seed(seed)\n", "torch.cuda.manual_seed(seed)\n", "torch.cuda.manual_seed_all(seed) # Multi-GPU 환경에서 동일한 시드 적용\n", "\n", "# PyTorch 연산의 결정적 모드 설정\n", "torch.backends.cudnn.deterministic = True # 실행마다 동일한 결과를 보장\n", "torch.backends.cudnn.benchmark = True # 성능 최적화를 활성화 (가능한 한 빠른 연산 수행)\n", "\n", "# 전처리 함수\n", "def preprocessing(df):\n", " df = df[df.columns].copy()\n", " df.loc[df['wind_dir']=='정온', 'wind_dir'] = \"0\"\n", " df['wind_dir'] = df['wind_dir'].astype('int')\n", " df['lm_cloudcover'] = df['lm_cloudcover'].astype('int')\n", " df['cloudcover'] = df['cloudcover'].astype('int')\n", " return df\n", "\n", "# 데이터셋 준비 함수\n", "def prepare_dataset(region, data_sample='pure', target='multi', fold=3):\n", "\n", " # 데이터 경로 지정\n", " dat_path = f\"../../data/data_for_modeling/{region}_train.csv\"\n", " if data_sample == 'pure':\n", " train_path = dat_path\n", " else:\n", " train_path = f'../../data/data_oversampled/{data_sample}/{data_sample}_{fold}_{region}.csv'\n", " train_path = f'../../data/data_oversampled/{data_sample}/{data_sample}_{fold}_{region}.csv'\n", " test_path = f\"../../data/data_for_modeling/{region}_test.csv\"\n", " drop_col = ['binary_class','multi_class','visi','year']\n", " target_col = f'{target}_class'\n", " \n", " # 데이터 로드\n", " region_dat = preprocessing(pd.read_csv(dat_path, index_col=0))\n", " if data_sample == 'pure':\n", " region_train = region_dat.loc[~region_dat['year'].isin([2021-fold]), :]\n", " else:\n", " region_train = preprocessing(pd.read_csv(train_path))\n", " region_val = region_dat.loc[region_dat['year'].isin([2021-fold]), :]\n", " region_test = preprocessing(pd.read_csv(test_path))\n", "\n", " # 컬럼 정렬 (일관성 유지)\n", " common_columns = region_train.columns.to_list()\n", " train_data = region_train[common_columns]\n", " val_data = region_val[common_columns]\n", " test_data = region_test[common_columns]\n", "\n", " # 설명변수 & 타겟 분리\n", " X_train = train_data.drop(columns=drop_col)\n", " y_train = train_data[target_col]\n", " X_val = val_data.drop(columns=drop_col)\n", " y_val = val_data[target_col]\n", " X_test = test_data.drop(columns=drop_col)\n", " y_test = test_data[target_col]\n", "\n", " # 범주형 & 연속형 변수 분리\n", " categorical_cols = X_train.select_dtypes(include=['object', 'category', 'int64']).columns\n", " numerical_cols = X_train.select_dtypes(include=['float64']).columns\n", "\n", " # 범주형 변수 Label Encoding\n", " label_encoders = {}\n", " for col in categorical_cols:\n", " le = LabelEncoder()\n", " le.fit(X_train[col]) # Train 데이터 기준으로 학습\n", " label_encoders[col] = le\n", "\n", " # 변환 적용\n", " for col in categorical_cols:\n", " X_train[col] = label_encoders[col].transform(X_train[col])\n", " X_val[col] = label_encoders[col].transform(X_val[col])\n", " X_test[col] = label_encoders[col].transform(X_test[col])\n", "\n", " # 연속형 변수 Standard Scaling\n", " scaler = StandardScaler()\n", " scaler.fit(X_train[numerical_cols]) # Train 데이터 기준으로 학습\n", "\n", " # 변환 적용\n", " X_train[numerical_cols] = scaler.transform(X_train[numerical_cols])\n", " X_val[numerical_cols] = scaler.transform(X_val[numerical_cols])\n", " X_test[numerical_cols] = scaler.transform(X_test[numerical_cols])\n", "\n", " return X_train, X_val, X_test, y_train, y_val, y_test, categorical_cols, numerical_cols\n", "\n", "# 데이터 변환 및 dataloader 생성 함수\n", "def prepare_dataloader(region, data_sample='pure', target='multi', fold=3, random_state=None):\n", "\n", " # 데이터 경로 지정\n", " dat_path = f\"../../data/data_for_modeling/{region}_train.csv\"\n", " if data_sample == 'pure':\n", " train_path = dat_path\n", " else:\n", " train_path = f'../../data/data_oversampled/{data_sample}/{data_sample}_{fold}_{region}.csv'\n", " train_path = f'../../data/data_oversampled/{data_sample}/{data_sample}_{fold}_{region}.csv'\n", " test_path = f\"../../data/data_for_modeling/{region}_test.csv\"\n", " drop_col = ['binary_class','multi_class','visi','year']\n", " target_col = f'{target}_class'\n", " \n", " # 데이터 로드\n", " region_dat = preprocessing(pd.read_csv(dat_path, index_col=0))\n", " if data_sample == 'pure':\n", " region_train = region_dat.loc[~region_dat['year'].isin([2021-fold]), :]\n", " else:\n", " region_train = preprocessing(pd.read_csv(train_path))\n", " region_val = region_dat.loc[region_dat['year'].isin([2021-fold]), :]\n", " region_test = preprocessing(pd.read_csv(test_path))\n", "\n", " # 컬럼 정렬 (일관성 유지)\n", " common_columns = region_train.columns.to_list()\n", " train_data = region_train[common_columns]\n", " val_data = region_val[common_columns]\n", " test_data = region_test[common_columns]\n", "\n", " # 설명변수 & 타겟 분리\n", " X_train = train_data.drop(columns=drop_col)\n", " y_train = train_data[target_col]\n", " X_val = val_data.drop(columns=drop_col)\n", " y_val = val_data[target_col]\n", " X_test = test_data.drop(columns=drop_col)\n", " y_test = test_data[target_col]\n", "\n", " # 범주형 & 연속형 변수 분리\n", " categorical_cols = X_train.select_dtypes(include=['object', 'category', 'int64']).columns\n", " numerical_cols = X_train.select_dtypes(include=['float64']).columns\n", "\n", " # 범주형 변수 Label Encoding\n", " label_encoders = {}\n", " for col in categorical_cols:\n", " le = LabelEncoder()\n", " le.fit(X_train[col]) # Train 데이터 기준으로 학습\n", " label_encoders[col] = le\n", "\n", " # 변환 적용\n", " for col in categorical_cols:\n", " X_train[col] = label_encoders[col].transform(X_train[col])\n", " X_val[col] = label_encoders[col].transform(X_val[col])\n", " X_test[col] = label_encoders[col].transform(X_test[col])\n", "\n", " # 연속형 변수 Standard Scaling\n", " scaler = StandardScaler()\n", " scaler.fit(X_train[numerical_cols]) # Train 데이터 기준으로 학습\n", "\n", " # 변환 적용\n", " X_train[numerical_cols] = scaler.transform(X_train[numerical_cols])\n", " X_val[numerical_cols] = scaler.transform(X_val[numerical_cols])\n", " X_test[numerical_cols] = scaler.transform(X_test[numerical_cols])\n", "\n", " # 연속형 변수와 범주형 변수 분리\n", " X_train_num = torch.tensor(X_train[numerical_cols].values, dtype=torch.float32)\n", " X_train_cat = torch.tensor(X_train[categorical_cols].values, dtype=torch.long)\n", "\n", " X_val_num = torch.tensor(X_val[numerical_cols].values, dtype=torch.float32)\n", " X_val_cat = torch.tensor(X_val[categorical_cols].values, dtype=torch.long)\n", "\n", " X_test_num = torch.tensor(X_test[numerical_cols].values, dtype=torch.float32)\n", " X_test_cat = torch.tensor(X_test[categorical_cols].values, dtype=torch.long)\n", "\n", " # 레이블 변환\n", " if target == \"binary\":\n", " y_train_tensor = torch.tensor(y_train.values, dtype=torch.float32) # 이진 분류 → float32\n", " y_val_tensor = torch.tensor(y_val.values, dtype=torch.float32)\n", " y_test_tensor = torch.tensor(y_test.values, dtype=torch.float32)\n", " elif target == \"multi\":\n", " y_train_tensor = torch.tensor(y_train.values, dtype=torch.long) # 다중 분류 → long\n", " y_val_tensor = torch.tensor(y_val.values, dtype=torch.long)\n", " y_test_tensor = torch.tensor(y_test.values, dtype=torch.long)\n", " else:\n", " raise ValueError(\"target must be 'binary' or 'multi'\")\n", "\n", " # TensorDataset 생성\n", " train_dataset = TensorDataset(X_train_num, X_train_cat, y_train_tensor)\n", " val_dataset = TensorDataset(X_val_num, X_val_cat, y_val_tensor)\n", " test_dataset = TensorDataset(X_test_num, X_test_cat, y_test_tensor)\n", "\n", " # DataLoader 생성\n", " if random_state == None:\n", " train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n", " else:\n", " train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True, generator=torch.Generator().manual_seed(random_state))\n", " val_loader = DataLoader(val_dataset, batch_size=64, shuffle=False)\n", " test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)\n", " \n", " return X_train, categorical_cols, numerical_cols, train_loader, val_loader, test_loader" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import sys\n", "sys.path.append('../')\n", "import torch\n", "import torch.nn as nn\n", "import torch.optim as optim\n", "import optuna\n", "import pandas as pd\n", "import numpy as np\n", "import random\n", "from ft_transformer import FTTransformer\n", "from resnet_like import ResNetLike\n", "from deepgbm import DeepGBM\n", "from sklearn.metrics import accuracy_score, f1_score, roc_auc_score, confusion_matrix" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "df_seoul = pd.read_csv(\"../../data/data_for_modeling/seoul_train.csv\")\n", "df_seoul_test = pd.read_csv(\"../../data/data_for_modeling/seoul_test.csv\")\n", "\n", "df_busan = pd.read_csv(\"../../data/data_for_modeling/busan_train.csv\")\n", "df_busan_test = pd.read_csv(\"../../data/data_for_modeling/busan_test.csv\")\n", "\n", "df_daegu = pd.read_csv(\"../../data/data_for_modeling/daegu_train.csv\")\n", "df_daegu_test = pd.read_csv(\"../../data/data_for_modeling/daegu_test.csv\")\n", "\n", "df_daejeon = pd.read_csv(\"../../data/data_for_modeling/daejeon_train.csv\")\n", "df_daejeon_test = pd.read_csv(\"../../data/data_for_modeling/daejeon_test.csv\")\n", "\n", "df_incheon = pd.read_csv(\"../../data/data_for_modeling/incheon_train.csv\")\n", "df_incheon_test = pd.read_csv(\"../../data/data_for_modeling/incheon_test.csv\")\n", "\n", "df_gwangju = pd.read_csv(\"../../data/data_for_modeling/gwangju_train.csv\")\n", "df_gwangju_test = pd.read_csv(\"../../data/data_for_modeling/gwangju_test.csv\")" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import os\n", "import torch\n", "# 디바이스 설정 (CUDA 사용 가능하면 GPU로, 아니면 CPU로)\n", "import glob\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "import warnings\n", "warnings.filterwarnings('ignore')\n", "from sklearn.metrics import matthews_corrcoef\n", "\n", "def calculate_csi(Y_test, pred):\n", "\n", " cm = confusion_matrix(Y_test, pred) # 변수 이름을 cm으로 변경\n", " # 혼동 행렬에서 H, F, M 추출\n", " H = (cm[0, 0] + cm[1, 1])\n", " \n", " F = (cm[1, 0] + cm[2, 0] +\n", " cm[0, 1] + cm[2, 1])\n", "\n", " M = (cm[0, 2] + cm[1, 2])\n", " \n", " # CSI 계산\n", " CSI = H / (H + F + M + 1e-10)\n", " return CSI\n", "\n", "# Soft Voting 앙상블\n", "def get_proba(region, model_choose, data_sample, fold, target='multi'):\n", " _, _, _, _,val_loader , test_loader = prepare_dataloader(region=region, data_sample=data_sample, target=target,fold=fold ,random_state=120)\n", " _ ,_,_ ,_ , y_val,_ ,_ ,_ = prepare_dataset(region=region, data_sample=data_sample, target=target, fold=fold)\n", "\n", " folder_path = f'../save_model/{model_choose}/{data_sample}'\n", " model_paths = [path for path in glob.glob(f'{folder_path}/*.pth') if f'{region}' in path]\n", "\n", " model = torch.load(model_paths[fold-1], weights_only=False).to(device)\n", " model.eval()\n", "\n", " val_preds = []\n", "\n", "\n", " with torch.no_grad():\n", " for x_num_batch, x_cat_batch, _ in val_loader:\n", " output = model(x_num_batch.to(device), x_cat_batch.to(device))\n", " val_proba = torch.softmax(output, dim=1)\n", " val_preds.extend(val_proba.cpu().numpy())\n", " \n", " val_preds = np.array(val_preds)\n", "\n", " return val_preds, y_val\n", "\n", "\n", "# 예시 입력(길이 동일해야 함)\n", "# y_val = 실제 레이블 (배열, 예: [0, 2, 1, 0, 2, ...])\n", "# y_pred = 예측 레이블 (배열, 예: [0, 2, 1, 0, 1, ...])\n", "\n", "def multiclass_mcc(y_val, y_pred):\n", " \"\"\"\n", " 다중 분류에서도 sklearn의 matthews_corrcoef를 그대로 사용할 수 있음.\n", " \"\"\"\n", " return matthews_corrcoef(y_val, y_pred)\n", "\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "deepgbm_pure= dict()\n", "deepgbm_smote = dict()\n", "deepgbm_ctgan20000 = dict()\n", "deepgbm_ctgan10000 = dict()\n", "deepgbm_ctgan7000 = dict()\n", "from scipy.stats import mode # 최다 득표수 계산을 위한 mode 함수" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "df= pd.DataFrame(columns=['region','model','data_sample','CSI','MCC','Accuracy'])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **원데이터**" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.7104578833855647\n", "mean of accuracy : 0.9665059344095949\n", "mean of mcc : 0.8188418354471367\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'seoul'\n", "datasample = 'pure'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.6818642050717673\n", "mean of accuracy : 0.9749078856534504\n", "mean of mcc : 0.8049988566475562\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'busan'\n", "datasample = 'pure'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5671286321477068\n", "mean of accuracy : 0.9090567324566875\n", "mean of mcc : 0.6887072986816571\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'incheon'\n", "datasample = 'pure'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5768403304029271\n", "mean of accuracy : 0.9752356921259908\n", "mean of mcc : 0.7158561147888857\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'daegu'\n", "datasample = 'pure'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.6879666308365847\n", "mean of accuracy : 0.9573174763580109\n", "mean of mcc : 0.797628286801526\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'daejeon'\n", "datasample = 'pure'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.6169039999207967\n", "mean of accuracy : 0.9513605060259002\n", "mean of mcc : 0.7404995782864335\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'gwangju'\n", "datasample = 'pure'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **SMOTE 증강 데이터**" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **서울**" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.6178386591753761\n", "mean of accuracy : 0.9473890885046287\n", "mean of mcc : 0.7437507767516012\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'seoul'\n", "datasample = 'smote'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **부산**" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5771081499427251\n", "mean of accuracy : 0.9590890660478578\n", "mean of mcc : 0.7328290895553372\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'busan'\n", "datasample = 'smote'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **인천**" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.559840116480077\n", "mean of accuracy : 0.9007683126647871\n", "mean of mcc : 0.6829120896700739\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'incheon'\n", "datasample = 'smote'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **대구**" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.6425770288404345\n", "mean of accuracy : 0.9787012085069575\n", "mean of mcc : 0.7686582033950807\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'daegu'\n", "datasample = 'smote'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **대전**" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5569398315371836\n", "mean of accuracy : 0.9299289492726501\n", "mean of mcc : 0.6906810572279379\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'daejeon'\n", "datasample = 'smote'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **광주**" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5526168770566259\n", "mean of accuracy : 0.9341257662333341\n", "mean of mcc : 0.6910361569617187\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'gwangju'\n", "datasample = 'smote'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **GAN 20000**" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **서울**" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(8760,)\n", "mean of csi : 0.7095252362606882\n", "mean of accuracy : 0.9619258968152972\n", "mean of mcc : 0.8058537207489471\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'seoul'\n", "datasample = 'ctgan20000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "print(np.argmax(val_preds, axis=1).shape)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **부산**" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5799669524449778\n", "mean of accuracy : 0.9640024203408438\n", "mean of mcc : 0.7334430424969706\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'busan'\n", "datasample = 'ctgan20000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **인천**" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5289538894298621\n", "mean of accuracy : 0.8935004283421081\n", "mean of mcc : 0.6580659191964314\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'incheon'\n", "datasample = 'ctgan20000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **대구**" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.44609227542135127\n", "mean of accuracy : 0.9682904159492977\n", "mean of mcc : 0.616365605274888\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'daegu'\n", "datasample = 'ctgan20000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **대전**" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5716353901847903\n", "mean of accuracy : 0.9385713751029269\n", "mean of mcc : 0.7095774395951602\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'daejeon'\n", "datasample = 'ctgan20000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **광주**" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.4571493095973662\n", "mean of accuracy : 0.9200951709625637\n", "mean of mcc : 0.620850435638757\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'gwangju'\n", "datasample = 'ctgan20000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **GAN 10000**" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **서울**" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5616336216829804\n", "mean of accuracy : 0.9383062604486363\n", "mean of mcc : 0.700657840332931\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'seoul'\n", "datasample = 'ctgan10000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **부산**" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5879352056252677\n", "mean of accuracy : 0.9672767422711281\n", "mean of mcc : 0.7351657312694786\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'busan'\n", "datasample = 'ctgan10000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[0.4653465346533995, 0.5820895522387269, 0.7163695299836764]" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "csi" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **인천**" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5290269463089048\n", "mean of accuracy : 0.8928201753291581\n", "mean of mcc : 0.6582877751557777\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'incheon'\n", "datasample = 'ctgan10000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **대구**" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5557972102197738\n", "mean of accuracy : 0.9747075421480318\n", "mean of mcc : 0.7032214449405951\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'daegu'\n", "datasample = 'ctgan10000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **대전**" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5413249801262358\n", "mean of accuracy : 0.9286541116683718\n", "mean of mcc : 0.6761149432485588\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'daejeon'\n", "datasample = 'ctgan10000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **광주**" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.41816087724732665\n", "mean of accuracy : 0.9039835816054095\n", "mean of mcc : 0.5735307327535618\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'gwangju'\n", "datasample = 'ctgan10000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **GAN 7000**" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **서울**" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5813133937275518\n", "mean of accuracy : 0.9403897497317663\n", "mean of mcc : 0.7156872321691327\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'seoul'\n", "datasample = 'ctgan7000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **부산**" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.597435747844303\n", "mean of accuracy : 0.9670328367891807\n", "mean of mcc : 0.7454746315945583\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'busan'\n", "datasample = 'ctgan7000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **인천**" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5123361043389418\n", "mean of accuracy : 0.8907633014779882\n", "mean of mcc : 0.646967905975133\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'incheon'\n", "datasample = 'ctgan7000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **대구**" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.6146700981464019\n", "mean of accuracy : 0.9766398640949504\n", "mean of mcc : 0.7399778687018386\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'daegu'\n", "datasample = 'ctgan7000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **대전**" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5581763081057168\n", "mean of accuracy : 0.9308861691244354\n", "mean of mcc : 0.6896808823936648\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'daejeon'\n", "datasample = 'ctgan7000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## **광주**" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean of csi : 0.5798647223742114\n", "mean of accuracy : 0.9421613394216134\n", "mean of mcc : 0.7012366964380711\n" ] } ], "source": [ "csi = []\n", "accuracy = []\n", "mcc = []\n", "\n", "\n", "region = 'gwangju'\n", "datasample = 'ctgan7000'\n", "model= 'deepgbm'\n", "\n", "# 1 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 1)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "# 2 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 2)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "# 3 Fold\n", "val_preds, y_val = get_proba(region, model, datasample, 3)\n", "csi.append(calculate_csi(y_val, np.argmax(val_preds, axis=1)))\n", "accuracy.append(accuracy_score(y_val, np.argmax(val_preds, axis=1)))\n", "mcc.append(multiclass_mcc(y_val, np.argmax(val_preds, axis=1)))\n", "\n", "\n", "\n", "print(\"mean of csi : \", np.mean(csi))\n", "print(\"mean of accuracy : \", np.mean(accuracy))\n", "print(\"mean of mcc : \", np.mean(mcc))\n", "\n", "new_row = pd.DataFrame([{\n", " 'region': region,\n", " 'model': model,\n", " 'data_sample': datasample,\n", " 'CSI': np.mean(csi),\n", " 'MCC': np.mean(mcc),\n", " 'Accuracy': np.mean(accuracy)\n", "\n", "}])\n", "\n", "df = pd.concat([df, new_row], ignore_index=True)\n", "\n" ] }, { "cell_type": "code", "execution_count": 38, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
regionmodeldata_sampleCSIMCCAccuracy
0seouldeepgbmpure0.6442510.7660970.957526
1busandeepgbmpure0.6818640.8049990.974908
2incheondeepgbmpure0.5671290.6887070.909057
3daegudeepgbmpure0.5768400.7158560.975236
4daejeondeepgbmpure0.6879670.7976280.957317
5gwangjudeepgbmpure0.6169040.7405000.951361
6seouldeepgbmsmote0.6178390.7437510.947389
7busandeepgbmsmote0.5771080.7328290.959089
8incheondeepgbmsmote0.5598400.6829120.900768
9daegudeepgbmsmote0.6425770.7686580.978701
10daejeondeepgbmsmote0.5569400.6906810.929929
11gwangjudeepgbmsmote0.5526170.6910360.934126
12seouldeepgbmctgan200000.7095250.8058540.961926
13busandeepgbmctgan200000.5799670.7334430.964002
14incheondeepgbmctgan200000.5289540.6580660.893500
15daegudeepgbmctgan200000.4460920.6163660.968290
16daejeondeepgbmctgan200000.5716350.7095770.938571
17gwangjudeepgbmctgan200000.4571490.6208500.920095
18seouldeepgbmctgan100000.5616340.7006580.938306
19busandeepgbmctgan100000.5879350.7351660.967277
20incheondeepgbmctgan100000.5290270.6582880.892820
21daegudeepgbmctgan100000.5557970.7032210.974708
22daejeondeepgbmctgan100000.5413250.6761150.928654
23gwangjudeepgbmctgan100000.4181610.5735310.903984
24seouldeepgbmctgan70000.5813130.7156870.940390
25busandeepgbmctgan70000.5974360.7454750.967033
26incheondeepgbmctgan70000.5123360.6469680.890763
27daegudeepgbmctgan70000.6146700.7399780.976640
28daejeondeepgbmctgan70000.5581760.6896810.930886
29gwangjudeepgbmctgan70000.5798650.7012370.942161
\n", "
" ], "text/plain": [ " region model data_sample CSI MCC Accuracy\n", "0 seoul deepgbm pure 0.644251 0.766097 0.957526\n", "1 busan deepgbm pure 0.681864 0.804999 0.974908\n", "2 incheon deepgbm pure 0.567129 0.688707 0.909057\n", "3 daegu deepgbm pure 0.576840 0.715856 0.975236\n", "4 daejeon deepgbm pure 0.687967 0.797628 0.957317\n", "5 gwangju deepgbm pure 0.616904 0.740500 0.951361\n", "6 seoul deepgbm smote 0.617839 0.743751 0.947389\n", "7 busan deepgbm smote 0.577108 0.732829 0.959089\n", "8 incheon deepgbm smote 0.559840 0.682912 0.900768\n", "9 daegu deepgbm smote 0.642577 0.768658 0.978701\n", "10 daejeon deepgbm smote 0.556940 0.690681 0.929929\n", "11 gwangju deepgbm smote 0.552617 0.691036 0.934126\n", "12 seoul deepgbm ctgan20000 0.709525 0.805854 0.961926\n", "13 busan deepgbm ctgan20000 0.579967 0.733443 0.964002\n", "14 incheon deepgbm ctgan20000 0.528954 0.658066 0.893500\n", "15 daegu deepgbm ctgan20000 0.446092 0.616366 0.968290\n", "16 daejeon deepgbm ctgan20000 0.571635 0.709577 0.938571\n", "17 gwangju deepgbm ctgan20000 0.457149 0.620850 0.920095\n", "18 seoul deepgbm ctgan10000 0.561634 0.700658 0.938306\n", "19 busan deepgbm ctgan10000 0.587935 0.735166 0.967277\n", "20 incheon deepgbm ctgan10000 0.529027 0.658288 0.892820\n", "21 daegu deepgbm ctgan10000 0.555797 0.703221 0.974708\n", "22 daejeon deepgbm ctgan10000 0.541325 0.676115 0.928654\n", "23 gwangju deepgbm ctgan10000 0.418161 0.573531 0.903984\n", "24 seoul deepgbm ctgan7000 0.581313 0.715687 0.940390\n", "25 busan deepgbm ctgan7000 0.597436 0.745475 0.967033\n", "26 incheon deepgbm ctgan7000 0.512336 0.646968 0.890763\n", "27 daegu deepgbm ctgan7000 0.614670 0.739978 0.976640\n", "28 daejeon deepgbm ctgan7000 0.558176 0.689681 0.930886\n", "29 gwangju deepgbm ctgan7000 0.579865 0.701237 0.942161" ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [], "source": [ "df.to_csv('../model_result/deepgbm_sampled_data_test.csv', index=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "py39", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.18" } }, "nbformat": 4, "nbformat_minor": 2 }