Upload 8 files

Analysis_code/find_reason/wasserstein_distance.ipynb

@@ -0,0 +1,541 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 분석에 필요한 라이브러리 임포트\n",
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "from scipy import stats\n",
+    "from scipy.spatial import distance\n",
+    "from scipy.stats import wasserstein_distance, entropy, ks_2samp\n",
+    "from sklearn.manifold import TSNE\n",
+    "from sklearn.preprocessing import StandardScaler\n",
+    "from sklearn.ensemble import RandomForestRegressor\n",
+    "from sklearn.ensemble import RandomForestClassifier # Added\n",
+    "from sklearn.model_selection import train_test_split # Added\n",
+    "from sklearn.metrics import roc_auc_score # Added\n",
+    "from statsmodels.distributions.empirical_distribution import ECDF # Added\n",
+    "import ot\n",
+    "\n",
+    "\n",
+    "# 한글 폰트 설정\n",
+    "plt.rcParams['font.family'] = 'NanumGothic'\n",
+    "plt.rcParams['axes.unicode_minus'] = False"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "seoul = pd.read_feather(\"../../data/data_for_modeling/df_seoul.feather\")\n",
+    "seoul= seoul[['datetime','hm','PM10','PM25','year','month','hour','multi_class']]\n",
+    "\n",
+    "busan = pd.read_feather(\"../../data/data_for_modeling/df_busan.feather\")\n",
+    "busan= busan[['datetime','hm','PM10','PM25','year','month','hour','multi_class']]\n",
+    "\n",
+    "incheon = pd.read_feather(\"../../data/data_for_modeling/df_incheon.feather\")\n",
+    "incheon= incheon[['datetime','hm','PM10','PM25','year','month','hour','multi_class']]\n",
+    "\n",
+    "daegu = pd.read_feather(\"../../data/data_for_modeling/df_daegu.feather\")\n",
+    "daegu= daegu[['datetime','hm','PM10','PM25','year','month','hour','multi_class']]\n",
+    "\n",
+    "daejeon = pd.read_feather(\"../../data/data_for_modeling/df_daejeon.feather\")\n",
+    "daejeon= daejeon[['datetime','hm','PM10','PM25','year','month','hour','multi_class']]\n",
+    "\n",
+    "gwangju = pd.read_feather(\"../../data/data_for_modeling/df_gwangju.feather\")\n",
+    "gwangju= gwangju[['datetime','hm','PM10','PM25','year','month','hour','multi_class']]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[0.5920662  0.92351786]\n",
+      "[0.60414398 0.9190468 ]\n",
+      "[0.60250035 0.9391276 ]\n",
+      "[0.60112832 0.92493121]\n",
+      "[0.58469137 0.90476229]\n",
+      "[0.617718   0.93503164]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from sklearn.decomposition import PCA\n",
+    "\n",
+    "# 특성 선택 (예: PM10, PM25, hm 등)\n",
+    "features = ['PM10','PM25', 'hm']\n",
+    "# 스케일링\n",
+    "scaler = StandardScaler()\n",
+    "scaled_features = scaler.fit_transform(seoul[features])\n",
+    "pca = PCA(n_components=2)\n",
+    "pca.fit(scaled_features)\n",
+    "print(pca.explained_variance_ratio_.cumsum())\n",
+    "seoul_pca = pca.transform(scaled_features)\n",
+    "seoul.drop(columns=['PM25', 'hm'], inplace=True)\n",
+    "seoul[['pca_x', 'pca_y']] = seoul_pca\n",
+    "\n",
+    "\n",
+    "scaled_features = scaler.fit_transform(busan[features])\n",
+    "pca = PCA(n_components=2)\n",
+    "pca.fit(scaled_features)\n",
+    "print(pca.explained_variance_ratio_.cumsum())\n",
+    "busan_pca = pca.transform(scaled_features)\n",
+    "busan.drop(columns=['PM25', 'hm'], inplace=True)\n",
+    "busan[['pca_x', 'pca_y']] = busan_pca\n",
+    "\n",
+    "scaled_features = scaler.fit_transform(incheon[features])   \n",
+    "pca = PCA(n_components=2)\n",
+    "pca.fit(scaled_features)\n",
+    "print(pca.explained_variance_ratio_.cumsum())\n",
+    "incheon_pca = pca.transform(scaled_features)\n",
+    "incheon.drop(columns=['PM25', 'hm'], inplace=True)\n",
+    "incheon[['pca_x', 'pca_y']] = incheon_pca\n",
+    "\n",
+    "scaled_features = scaler.fit_transform(daegu[features])\n",
+    "pca = PCA(n_components=2)\n",
+    "pca.fit(scaled_features)\n",
+    "print(pca.explained_variance_ratio_.cumsum())\n",
+    "daegu_pca = pca.transform(scaled_features)\n",
+    "daegu.drop(columns=['PM25', 'hm'], inplace=True)\n",
+    "daegu[['pca_x', 'pca_y']] = daegu_pca\n",
+    "\n",
+    "scaled_features = scaler.fit_transform(daejeon[features])\n",
+    "pca = PCA(n_components=2)\n",
+    "pca.fit(scaled_features)\n",
+    "print(pca.explained_variance_ratio_.cumsum())\n",
+    "daejeon_pca = pca.transform(scaled_features)\n",
+    "daejeon.drop(columns=['PM25', 'hm'], inplace=True)\n",
+    "daejeon[['pca_x', 'pca_y']] = daejeon_pca\n",
+    "\n",
+    "scaled_features = scaler.fit_transform(gwangju[features])\n",
+    "pca = PCA(n_components=2)\n",
+    "pca.fit(scaled_features)\n",
+    "print(pca.explained_variance_ratio_.cumsum())\n",
+    "gwangju_pca = pca.transform(scaled_features)\n",
+    "gwangju.drop(columns=['PM25', 'hm'], inplace=True)\n",
+    "gwangju[['pca_x', 'pca_y']] = gwangju_pca\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "seoul_2018 = seoul[seoul['year'] == 2018]\n",
+    "seoul_2019 = seoul[seoul['year'] == 2019]\n",
+    "seoul_2020 = seoul[seoul['year'] == 2020]\n",
+    "seoul_2021 = seoul[seoul['year'] == 2021]\n",
+    "years = [2018, 2019, 2020, 2021]\n",
+    "\n",
+    "\n",
+    "busan_2018 = busan[busan['year'] == 2018]\n",
+    "busan_2019 = busan[busan['year'] == 2019]\n",
+    "busan_2020 = busan[busan['year'] == 2020]\n",
+    "busan_2021 = busan[busan['year'] == 2021]\n",
+    "years = [2018, 2019, 2020, 2021]\n",
+    "\n",
+    "\n",
+    "incheon_2018 = incheon[incheon['year'] == 2018]\n",
+    "incheon_2019 = incheon[incheon['year'] == 2019]\n",
+    "incheon_2020 = incheon[incheon['year'] == 2020]\n",
+    "incheon_2021 = incheon[incheon['year'] == 2021]\n",
+    "years = [2018, 2019, 2020, 2021]\n",
+    "\n",
+    "\n",
+    "daegu_2018 = daegu[daegu['year'] == 2018]\n",
+    "daegu_2019 = daegu[daegu['year'] == 2019]\n",
+    "daegu_2020 = daegu[daegu['year'] == 2020]\n",
+    "daegu_2021 = daegu[daegu['year'] == 2021]\n",
+    "years = [2018, 2019, 2020, 2021]\n",
+    "\n",
+    "\n",
+    "daejeon_2018 = daejeon[daejeon['year'] == 2018]\n",
+    "daejeon_2019 = daejeon[daejeon['year'] == 2019]\n",
+    "daejeon_2020 = daejeon[daejeon['year'] == 2020]\n",
+    "daejeon_2021 = daejeon[daejeon['year'] == 2021]\n",
+    "years = [2018, 2019, 2020, 2021]\n",
+    "\n",
+    "\n",
+    "gwangju_2018 = gwangju[gwangju['year'] == 2018]\n",
+    "gwangju_2019 = gwangju[gwangju['year'] == 2019]\n",
+    "gwangju_2020 = gwangju[gwangju['year'] == 2020]\n",
+    "gwangju_2021 = gwangju[gwangju['year'] == 2021]\n",
+    "years = [2018, 2019, 2020, 2021]\n",
+    "\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "          2018      2019      2020      2021\n",
+      "2018       0.0  0.130217  0.063132  1.081307\n",
+      "2019  0.130217       0.0  0.059051  0.830648\n",
+      "2020  0.063132  0.059051       0.0  0.039927\n",
+      "2021  1.081307  0.830648  0.039927       0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 연도별 데이터 준비\n",
+    "years = [2018, 2019, 2020, 2021]\n",
+    "data_dict = {\n",
+    "    2018: seoul_2018[['pca_x', 'pca_y']].values,\n",
+    "    2019: seoul_2019[['pca_x', 'pca_y']].values,\n",
+    "    2020: seoul_2020[['pca_x', 'pca_y']].values,\n",
+    "    2021: seoul_2021[['pca_x', 'pca_y']].values\n",
+    "}\n",
+    "\n",
+    "\n",
+    "# 결과를 저장할 데이터프레임 생성\n",
+    "result_df = pd.DataFrame(index=years, columns=years)\n",
+    "\n",
+    "for i, year1 in enumerate(years):\n",
+    "    for j, year2 in enumerate(years):\n",
+    "        if year1 == year2:\n",
+    "            result_df.iloc[i, j] = 0.0\n",
+    "        if j < i:\n",
+    "            # 이미 계산된 값 사용\n",
+    "            result_df.iloc[i, j] = result_df.iloc[j, i]\n",
+    "        else:\n",
+    "            X = data_dict[year1]\n",
+    "            Y = data_dict[year2]\n",
+    "            a = np.ones(len(X)) / len(X)\n",
+    "            b = np.ones(len(Y)) / len(Y)\n",
+    "            W = ot.emd2(a, b, ot.dist(X, Y))\n",
+    "            result_df.iloc[i, j] = W\n",
+    "            result_df.iloc[j, i] = W  # 대칭 위치에 동일 값 저장\n",
+    "\n",
+    "# 결과 출력\n",
+    "print(result_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "          2018      2019      2020      2021\n",
+      "2018       0.0  0.116261   0.10445  1.424479\n",
+      "2019  0.116261       0.0   0.09933  1.164067\n",
+      "2020   0.10445   0.09933       0.0  1.075336\n",
+      "2021  1.424479  1.164067  1.075336       0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 연도별 데이터 준비\n",
+    "years = [2018, 2019, 2020, 2021]\n",
+    "data_dict = {\n",
+    "    2018: busan_2018[['pca_x', 'pca_y']].values,\n",
+    "    2019: busan_2019[['pca_x', 'pca_y']].values,\n",
+    "    2020: busan_2020[['pca_x', 'pca_y']].values,\n",
+    "    2021: busan_2021[['pca_x', 'pca_y']].values\n",
+    "}\n",
+    "\n",
+    "\n",
+    "# 결과를 저장할 데이터프레임 생성\n",
+    "result_df = pd.DataFrame(index=years, columns=years)\n",
+    "\n",
+    "for i, year1 in enumerate(years):\n",
+    "    for j, year2 in enumerate(years):\n",
+    "        if year1 == year2:\n",
+    "            result_df.iloc[i, j] = 0.0\n",
+    "        if j < i:\n",
+    "            # 이미 계산된 값 사용\n",
+    "            result_df.iloc[i, j] = result_df.iloc[j, i]\n",
+    "        else:\n",
+    "            X = data_dict[year1]\n",
+    "            Y = data_dict[year2]\n",
+    "            a = np.ones(len(X)) / len(X)\n",
+    "            b = np.ones(len(Y)) / len(Y)\n",
+    "            W = ot.emd2(a, b, ot.dist(X, Y))\n",
+    "            result_df.iloc[i, j] = W\n",
+    "            result_df.iloc[j, i] = W  # 대칭 위치에 동일 값 저장\n",
+    "\n",
+    "# 결과 출력\n",
+    "print(result_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "          2018      2019      2020      2021\n",
+      "2018       0.0  0.080291  0.074071  0.449094\n",
+      "2019  0.080291       0.0  0.060171  0.384189\n",
+      "2020  0.074071  0.060171       0.0   0.04047\n",
+      "2021  0.449094  0.384189   0.04047       0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 연도별 데이터 준비\n",
+    "years = [2018, 2019, 2020, 2021]\n",
+    "data_dict = {\n",
+    "    2018: incheon_2018[['pca_x', 'pca_y']].values,\n",
+    "    2019: incheon_2019[['pca_x', 'pca_y']].values,\n",
+    "    2020: incheon_2020[['pca_x', 'pca_y']].values,\n",
+    "    2021: incheon_2021[['pca_x', 'pca_y']].values\n",
+    "}\n",
+    "\n",
+    "\n",
+    "# 결과를 저장할 데이터프레임 생성\n",
+    "result_df = pd.DataFrame(index=years, columns=years)\n",
+    "\n",
+    "for i, year1 in enumerate(years):\n",
+    "    for j, year2 in enumerate(years):\n",
+    "        if year1 == year2:\n",
+    "            result_df.iloc[i, j] = 0.0\n",
+    "        if j < i:\n",
+    "            # 이미 계산된 값 사용\n",
+    "            result_df.iloc[i, j] = result_df.iloc[j, i]\n",
+    "        else:\n",
+    "            X = data_dict[year1]\n",
+    "            Y = data_dict[year2]\n",
+    "            a = np.ones(len(X)) / len(X)\n",
+    "            b = np.ones(len(Y)) / len(Y)\n",
+    "            W = ot.emd2(a, b, ot.dist(X, Y))\n",
+    "            result_df.iloc[i, j] = W\n",
+    "            result_df.iloc[j, i] = W  # 대칭 위치에 동일 값 저장\n",
+    "\n",
+    "# 결과 출력\n",
+    "print(result_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "          2018      2019      2020      2021\n",
+      "2018       0.0  0.127512  0.112157  0.731476\n",
+      "2019  0.127512       0.0  0.094651  0.647071\n",
+      "2020  0.112157  0.094651       0.0  0.041217\n",
+      "2021  0.731476  0.647071  0.041217       0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 연도별 데이터 준비\n",
+    "years = [2018, 2019, 2020, 2021]\n",
+    "data_dict = {\n",
+    "    2018: daegu_2018[['pca_x', 'pca_y']].values,\n",
+    "    2019: daegu_2019[['pca_x', 'pca_y']].values,\n",
+    "    2020: daegu_2020[['pca_x', 'pca_y']].values,\n",
+    "    2021: daegu_2021[['pca_x', 'pca_y']].values\n",
+    "}\n",
+    "\n",
+    "\n",
+    "# 결과를 저장할 데이터프레임 생성\n",
+    "result_df = pd.DataFrame(index=years, columns=years)\n",
+    "\n",
+    "for i, year1 in enumerate(years):\n",
+    "    for j, year2 in enumerate(years):\n",
+    "        if year1 == year2:\n",
+    "            result_df.iloc[i, j] = 0.0\n",
+    "        if j < i:\n",
+    "            # 이미 계산된 값 사용\n",
+    "            result_df.iloc[i, j] = result_df.iloc[j, i]\n",
+    "        else:\n",
+    "            X = data_dict[year1]\n",
+    "            Y = data_dict[year2]\n",
+    "            a = np.ones(len(X)) / len(X)\n",
+    "            b = np.ones(len(Y)) / len(Y)\n",
+    "            W = ot.emd2(a, b, ot.dist(X, Y))\n",
+    "            result_df.iloc[i, j] = W\n",
+    "            result_df.iloc[j, i] = W  # 대칭 위치에 동일 값 저장\n",
+    "\n",
+    "# 결과 출력\n",
+    "print(result_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "          2018      2019      2020      2021\n",
+      "2018       0.0  0.273013  0.053969  0.877338\n",
+      "2019  0.273013       0.0  0.137817  0.780071\n",
+      "2020  0.053969  0.137817       0.0  0.042294\n",
+      "2021  0.877338  0.780071  0.042294       0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 연도별 데이터 준비\n",
+    "years = [2018, 2019, 2020, 2021]\n",
+    "data_dict = {\n",
+    "    2018: daejeon_2018[['pca_x', 'pca_y']].values,\n",
+    "    2019: daejeon_2019[['pca_x', 'pca_y']].values,\n",
+    "    2020: daejeon_2020[['pca_x', 'pca_y']].values,\n",
+    "    2021: daejeon_2021[['pca_x', 'pca_y']].values\n",
+    "}\n",
+    "\n",
+    "\n",
+    "# 결과를 저장할 데이터프레임 생성\n",
+    "result_df = pd.DataFrame(index=years, columns=years)\n",
+    "\n",
+    "for i, year1 in enumerate(years):\n",
+    "    for j, year2 in enumerate(years):\n",
+    "        if year1 == year2:\n",
+    "            result_df.iloc[i, j] = 0.0\n",
+    "        if j < i:\n",
+    "            # 이미 계산된 값 사용\n",
+    "            result_df.iloc[i, j] = result_df.iloc[j, i]\n",
+    "        else:\n",
+    "            X = data_dict[year1]\n",
+    "            Y = data_dict[year2]\n",
+    "            a = np.ones(len(X)) / len(X)\n",
+    "            b = np.ones(len(Y)) / len(Y)\n",
+    "            W = ot.emd2(a, b, ot.dist(X, Y))\n",
+    "            result_df.iloc[i, j] = W\n",
+    "            result_df.iloc[j, i] = W  # 대칭 위치에 동일 값 저장\n",
+    "\n",
+    "# 결과 출력\n",
+    "print(result_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "          2018      2019      2020      2021\n",
+      "2018       0.0  0.105633   0.08202   1.00155\n",
+      "2019  0.105633       0.0  0.069322  0.892938\n",
+      "2020   0.08202  0.069322       0.0  0.480667\n",
+      "2021   1.00155  0.892938  0.480667       0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 연도별 데이터 준비\n",
+    "years = [2018, 2019, 2020, 2021]\n",
+    "data_dict = {\n",
+    "    2018: gwangju_2018[['pca_x', 'pca_y']].values,\n",
+    "    2019: gwangju_2019[['pca_x', 'pca_y']].values,\n",
+    "    2020: gwangju_2020[['pca_x', 'pca_y']].values,\n",
+    "    2021: gwangju_2021[['pca_x', 'pca_y']].values\n",
+    "}\n",
+    "\n",
+    "\n",
+    "# 결과를 저장할 데이터프레임 생성\n",
+    "result_df = pd.DataFrame(index=years, columns=years)\n",
+    "\n",
+    "for i, year1 in enumerate(years):\n",
+    "    for j, year2 in enumerate(years):\n",
+    "        if year1 == year2:\n",
+    "            result_df.iloc[i, j] = 0.0\n",
+    "        if j < i:\n",
+    "            # 이미 계산된 값 사용\n",
+    "            result_df.iloc[i, j] = result_df.iloc[j, i]\n",
+    "        else:\n",
+    "            X = data_dict[year1]\n",
+    "            Y = data_dict[year2]\n",
+    "            a = np.ones(len(X)) / len(X)\n",
+    "            b = np.ones(len(Y)) / len(Y)\n",
+    "            W = ot.emd2(a, b, ot.dist(X, Y))\n",
+    "            result_df.iloc[i, j] = W\n",
+    "            result_df.iloc[j, i] = W  # 대칭 위치에 동일 값 저장\n",
+    "\n",
+    "# 결과 출력\n",
+    "print(result_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "py39",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}