Hugging Face's logo

bong9513
/
Visibility_Nowcasting

Model card Files
xet
 Community
3
Visibility_Nowcasting
File size: 7,804 Bytes
9e55601
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
import pandas as pd
import numpy as np
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
from imblearn.over_sampling import SMOTENC
import optuna
from ctgan import CTGAN
import torch
import warnings

# ์ง€์—ญ๋ณ„ ๋ฐ์ดํ„ฐ ํŒŒ์ผ ๊ฒฝ๋กœ
regions = ['incheon', 'seoul','busan', 'daegu', 'daejeon', 'gwangju']
file_paths = [f'../../data/data_for_modeling/{region}_train.csv' for region in regions]
output_paths = [f'../../data/data_oversampled/ctgan10000/ctgan10000_2_{region}.csv' for region in regions]

# GPU ์‚ฌ์šฉ ์„ค์ •
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")

# ๊ฒฝ๊ณ  ๋ฌด์‹œ
warnings.filterwarnings("ignore", category=UserWarning, module="optuna.distributions")

# ์ง€์—ญ๋ณ„ ์ฒ˜๋ฆฌ
for file_path, output_path in zip(file_paths, output_paths):
    # ๋ฐ์ดํ„ฐ ๋กœ๋“œ
    data = pd.read_csv(file_path, index_col=0)
    data= data.loc[data['year'].isin([2018,2020]),:]
    data['cloudcover'] = data['cloudcover'].astype('int')
    data['lm_cloudcover'] = data['lm_cloudcover'].astype('int')
    X = data.drop(columns=['multi_class', 'binary_class'])
    y = data['multi_class']

    # ๋ถˆํ•„์š”ํ•œ ์—ด ์ œ๊ฑฐ
    X.drop(columns=['ground_temp - temp_C', 'hour_sin', 'hour_cos', 'month_sin', 'month_cos'], inplace=True)

    # SMOTENC์—์„œ ์‚ฌ์šฉํ•  ๋ฒ”์ฃผํ˜• ๋ณ€์ˆ˜ ์—ด ๋ฒˆํ˜ธ ์„ค์ •
    categorical_features_indices = [i for i, dtype in enumerate(X.dtypes) if dtype != 'float64']

    # sampling_strategy ์„ค์ •
    count_class_0 = (y == 0).sum()
    count_class_1 = (y == 1).sum()
    count_class_2 = (y == 2).sum()
    sampling_strategy = {
        0: 500 if count_class_0 <= 500 else 1000,
        1: int(np.ceil(count_class_1 / 100) * 100),  # ๋ฐฑ์˜ ์ž๋ฆฌ๋กœ ์˜ฌ๋ฆผ
        2: count_class_2
    }

    # SMOTENC ์ ์šฉ
    smotenc = SMOTENC(categorical_features=categorical_features_indices, sampling_strategy=sampling_strategy, random_state=42)
    X_resampled, y_resampled = smotenc.fit_resample(X, y)

    # Resampled ๋ฐ์ดํ„ฐ ์ƒ์„ฑ
    lerp_data = X_resampled.copy()
    lerp_data['multi_class'] = y_resampled

    # CTGAN์—์„œ ์‚ฌ์šฉํ•  ๋ฒ”์ฃผํ˜• ๋ณ€์ˆ˜ ์—ด ์ด๋ฆ„ ์„ค์ •
    categorical_features = [
        col for col, dtype in zip(lerp_data.columns, lerp_data.dtypes) if dtype != 'float64'
    ]

    # Optuna ๋ชฉ์  ํ•จ์ˆ˜ ์ •์˜
    def objective(trial):
        # ํ•˜์ดํผํŒŒ๋ผ๋ฏธํ„ฐ ํƒ์ƒ‰ ๋ฒ”์œ„ ์„ค์ •
        embedding_dim = trial.suggest_int("embedding_dim", 64, 128)
        generator_dim = trial.suggest_categorical("generator_dim", [(64, 64), (128, 128)])
        discriminator_dim = trial.suggest_categorical("discriminator_dim", [(64, 64), (128, 128)])
        pac = trial.suggest_categorical("pac", [4, 8])
        batch_size = trial.suggest_categorical("batch_size", [64, 128, 256])
        discriminator_steps = trial.suggest_int("discriminator_steps", 1, 3)

        # CTGAN ๋ชจ๋ธ ์ƒ์„ฑ
        ctgan = CTGAN(
            embedding_dim=embedding_dim,
            generator_dim=generator_dim,
            discriminator_dim=discriminator_dim,
            batch_size=batch_size,
            discriminator_steps=discriminator_steps,
            pac=pac
        )

        # ๋ฒ”์ฃผ 0 ๋ฐ์ดํ„ฐ ํ•„ํ„ฐ๋ง
        data_0 = lerp_data[lerp_data['multi_class'] == 0]

        # ๋ชจ๋ธ ํ•™์Šต
        ctgan.fit(data_0, discrete_columns=categorical_features)

        # ์ƒ˜ํ”Œ ์ƒ์„ฑ
        generated_data = ctgan.sample(len(data_0) * 2)

        # ํ‰๊ฐ€: ์ƒ˜ํ”Œ์˜ ์—ฐ์†ํ˜• ๋ณ€์ˆ˜ ๋ถ„ํฌ ๋น„๊ต
        real_visi = data_0['visi']
        generated_visi = generated_data['visi']

        # ๋ถ„ํฌ ๊ฐ„ ์ฐจ์ด(MSE) ๊ณ„์‚ฐ
        mse = ((real_visi.mean() - generated_visi.mean())**2 + (real_visi.std() - generated_visi.std())**2)
        return -mse

    # Optuna๋กœ ์ตœ์ ํ™” ์ˆ˜ํ–‰
    study = optuna.create_study(direction="maximize")
    study.optimize(objective, n_trials=50)

    # ์ตœ์  ํ•˜์ดํผํŒŒ๋ผ๋ฏธํ„ฐ ์ถœ๋ ฅ
    best_params = study.best_params

    # ์ตœ์  ํ•˜์ดํผํŒŒ๋ผ๋ฏธํ„ฐ๋กœ CTGAN ํ•™์Šต ๋ฐ ์ƒ˜ํ”Œ ์ƒ์„ฑ
    ctgan = CTGAN(
        embedding_dim=best_params["embedding_dim"],
        generator_dim=best_params["generator_dim"],
        discriminator_dim=best_params["discriminator_dim"],
        batch_size=best_params["batch_size"],
        discriminator_steps=best_params["discriminator_steps"],
        pac=best_params["pac"]
    )

    # ๋ฒ”์ฃผ 0 ๋ฐ์ดํ„ฐ๋กœ ์ตœ์ข… ํ•™์Šต
    ctgan.fit(lerp_data[lerp_data['multi_class'] == 0], discrete_columns=categorical_features)
    generated_0 = ctgan.sample(10000)

    # ๋ฒ”์ฃผ 1 ๋ฐ์ดํ„ฐ ์ตœ์ ํ™” ๋ฐ ์ƒ์„ฑ
    def objective_class1(trial):
        embedding_dim = trial.suggest_int("embedding_dim", 128, 512)
        generator_dim = trial.suggest_categorical("generator_dim", [(128, 128), (256, 256)])
        discriminator_dim = trial.suggest_categorical("discriminator_dim", [(128, 128), (256, 256)])
        pac = trial.suggest_categorical("pac", [4, 8])
        batch_size = trial.suggest_categorical("batch_size", [256, 512, 1024])
        discriminator_steps = trial.suggest_int("discriminator_steps", 1, 5)

        ctgan = CTGAN(
            embedding_dim=embedding_dim,
            generator_dim=generator_dim,
            discriminator_dim=discriminator_dim,
            batch_size=batch_size,
            discriminator_steps=discriminator_steps,
            pac=pac
        )

        data_1 = lerp_data[lerp_data['multi_class'] == 1]
        ctgan.fit(data_1, discrete_columns=categorical_features)
        generated_data = ctgan.sample(len(data_1) * 2)

        real_visi = data_1['visi']
        generated_visi = generated_data['visi']
        mse = ((real_visi.mean() - generated_visi.mean())**2 + (real_visi.std() - generated_visi.std())**2)
        return -mse

    study_class1 = optuna.create_study(direction="maximize")
    study_class1.optimize(objective_class1, n_trials=30)

    best_params_class1 = study_class1.best_params
    ctgan = CTGAN(
        embedding_dim=best_params_class1["embedding_dim"],
        generator_dim=best_params_class1["generator_dim"],
        discriminator_dim=best_params_class1["discriminator_dim"],
        batch_size=best_params_class1["batch_size"],
        discriminator_steps=best_params_class1["discriminator_steps"],
        pac=best_params_class1["pac"]
    )

    ctgan.fit(lerp_data[lerp_data['multi_class'] == 1], discrete_columns=categorical_features)
    generated_1 = ctgan.sample(10000 - int(np.ceil(count_class_1 / 100) * 100))

    # ๋ฐ์ดํ„ฐ ๋ณ‘ํ•ฉ ๋ฐ ์ €์žฅ
    well_generated0 = generated_0[(generated_0['visi'] >= 0) & (generated_0['visi'] < 100)]
    well_generated1 = generated_1[(generated_1['visi'] >= 100) & (generated_1['visi'] < 500)]
    smote_gan_data = pd.concat([lerp_data, well_generated0, well_generated1], axis=0)
    # ์ œ๊ฑฐ๋ณ€์ˆ˜ ๋ณต๊ตฌ
    smote_gan_data['binary_class'] = smote_gan_data['multi_class'].apply(lambda x: 0 if x == 2 else 1)
    smote_gan_data['hour_sin'] = np.sin(2 * np.pi * smote_gan_data['hour'] / 24)
    smote_gan_data['hour_cos'] = np.cos(2 * np.pi * smote_gan_data['hour'] / 24)
    smote_gan_data['month_sin'] = np.sin(2 * np.pi * smote_gan_data['month'] / 12)
    smote_gan_data['month_cos'] = np.cos(2 * np.pi * smote_gan_data['month'] / 12)
    smote_gan_data['ground_temp - temp_C'] = smote_gan_data['groundtemp'] - smote_gan_data['temp_C']

    filtered_data = smote_gan_data[smote_gan_data['multi_class'] != 2]
    original_class2 = data[data['multi_class'] == 2]
    final_data = pd.concat([filtered_data, original_class2], axis=0)
    final_data.reset_index(drop=True, inplace=True)

    # ๊ฒฐ๊ณผ ์ €์žฅ
    final_data.to_csv(output_path, index = False)
    print(len(final_data[final_data['multi_class']==0]),'|',len(final_data[final_data['multi_class']==1]),'|',len(final_data[final_data['multi_class']==2]))