unify freq

src/utils.py

@@ -56,6 +56,29 @@ def format_df(df):
     # make sure the data type is float
     df.iloc[:, 1:] = df.iloc[:, 1:].astype(float)
     return df
+
+def unify_freq(df):
+    # Remove all numeric characters from the 'frequency' column
+    df['frequency'] = df['frequency'].str.replace(r'\d+', '', regex=True)
+    # Remove everything after '-' if present
+    df['frequency'] = df['frequency'].str.split('-').str[0]
+
+    # Define the frequency conversion dictionary
+    freq_conversion = {
+        'T': 'Minutely',
+        'H': 'Hourly',
+        'D': 'Daily',
+        'W': 'Weekly',
+        'M': 'Monthly',
+        'Q': 'Quarterly',
+        'Y': 'Yearly',
+        'A': 'Yearly',
+        'S': 'Secondly'
+    }
+
+    # Map the cleaned 'frequency' values using the dictionary
+    df['frequency'] = df['frequency'].replace(freq_conversion)
+    return df
 def pivot_existed_df(df, tab_name):
     df = df.reset_index()
     if tab_name == 'univariate':
@@ -128,6 +151,8 @@ def get_grouped_dfs(root_dir='results', ds_properties='results/dataset_propertie
             else:
                 df.loc[df['dataset'] == dataset, key] = dataset_properties_dict[dataset][key]
 
+    # unify the frequency
+    df = unify_freq(df)
     # standardize by seasonal naive
     df = standardize_df(df)
     metric_columns = ['eval_metrics/MSE[mean]', 'eval_metrics/MSE[0.5]', 'eval_metrics/MAE[0.5]',
@@ -179,6 +204,13 @@ def standardize_df(df):
     # 6. Create a new df with standardized results
     original_df = df.copy()
     # 1. Get all the unique dataset names
+    dataset_corrections = {
+        "saugeenday": "saugeen",
+        "temperature_rain_with_missing": "temperature_rain",
+        "kdd_cup_2018_with_missing": "kdd_cup_2018",
+        "car_parts_with_missing": "car_parts",
+    }
+    df['dataset'] = df['dataset'].replace(dataset_corrections)
     dataset_names = df['dataset'].unique()
     # 2. For each dataset name, get all the unique frequencies and term lengths
     for dataset in dataset_names:
@@ -191,11 +223,15 @@ def standardize_df(df):
                     (df['dataset'] == dataset) & (df['frequency'] == frequency) & (df['term_length'] == term_length) & (
                                 df['model'] == 'Seasonal_Naive')]
                 for metric in metric_columns:
-                    # 5. For each model name, dataset name, frequency, and term length, divide the model results by the seasonal_naive results
-                    df.loc[(df['dataset'] == dataset) & (df['frequency'] == frequency) & (
-                                df['term_length'] == term_length), metric] = df[(df['dataset'] == dataset) & (
-                                df['frequency'] == frequency) & (df['term_length'] == term_length)][metric] / \
-                                                                             seasonal_naive_results[metric].values[0]
+                    try:
+                        # 5. For each model name, dataset name, frequency, and term length, divide the model results by the seasonal_naive results
+                        df.loc[(df['dataset'] == dataset) & (df['frequency'] == frequency) & (
+                                    df['term_length'] == term_length), metric] = df[(df['dataset'] == dataset) & (
+                                    df['frequency'] == frequency) & (df['term_length'] == term_length)][metric] / \
+                                                                                 seasonal_naive_results[metric].values[0]
+                    except Exception:
+                        print(f"Error: {dataset} {term_length} {frequency} {metric}")
+                        ipdb.set_trace()
     # df[(df['dataset'] == 'bitbrains_fast_storage') & (df['model'] == 'seasonal_naive')]
     return df