new method of reading and interpolating datasets that hopefully generalizes

v2/app.py

@@ -20,8 +20,9 @@ from bess import BatteryParameters, BatteryModel
 # !wget "https://static.renyi.hu/ai-shared/daniel/pq/PL_44527.19-21.csv.gz"
 # !wget "https://static.renyi.hu/ai-shared/daniel/pq/pq_terheles_2021_adatok.tsv"
 
+OLD_DATASET = False
 
-met_2021_data, cons_2021_data = read_datasets()
+met_2021_data, cons_2021_data = read_datasets(old_dataset=OLD_DATASET)
 
 
 # TODO some gui to upload consumption data.
@@ -37,7 +38,19 @@ def recalculate(ui):
         print("Hm, we should do something with this CSV.", ui.consumption_csv.name)
 
     add_production_field(met_2021_data, solar_parameters)
-    all_data = interpolate_and_join(met_2021_data, cons_2021_data)
+
+    if OLD_DATASET:
+        # i've obsoleted this, but it's not a 100% replacement,
+        # it treats daylight savings changes differently.
+        # new version drops repeated hour and interpolates skipped hour.
+        all_data = interpolate_and_join(met_2021_data, cons_2021_data)
+    else:
+        all_data = join_consumption_meteorology(
+            cons_2021_data,
+            met_2021_data,
+            target_freq="5min",
+        )
+
 
     time_interval_min = all_data.index.freq.n
     time_interval_h = time_interval_min / 60

v2/architecture.py

@@ -335,7 +335,7 @@ def main():
 
     solar_parameters = SolarParameters()
 
-    met_2021_data, cons_2021_data = read_datasets()
+    met_2021_data, cons_2021_data = read_datasets(old_dataset=True)
     add_production_field(met_2021_data, solar_parameters)
     all_data = interpolate_and_join(met_2021_data, cons_2021_data)
 

v2/data_processing.py

@@ -10,7 +10,10 @@ START = f"2021-01-01"
 END = f"2022-01-01"
 
 
-def read_datasets(mini=False):
+def read_datasets(mini=False, old_dataset=False):
+    # old_dataset mode is needed if we plug this into interpolate_and_join()
+    # rather than join_consumption_meteorology().
+
     if mini:
         met_filename = 'PL_44527.2101.csv.gz'
         cons_filename = 'pq_terheles_202101_adatok.tsv'
@@ -31,14 +34,58 @@ def read_datasets(mini=False):
     cons_data = cons_data.set_index('Time')
     cons_data['Consumption'] = cons_data['Hatásos teljesítmény [kW]']
 
-    # consumption data is at 14 29 44 59 minutes, we move it by 1 minute
-    # to sync it with production data:
-    cons_data.index = cons_data.index + pd.DateOffset(minutes=1)
+    if old_dataset:
+        # consumption data is at 14 29 44 59 minutes, we move it by 1 minute
+        # to sync it with production data:
+        cons_data.index = cons_data.index + pd.DateOffset(minutes=1)
+
+        met_2021_data = met_data[(met_data.index >= START) & (met_data.index < END)]
+        cons_2021_data = cons_data[(cons_data.index >= START) & (cons_data.index < END)]
+
+        return met_2021_data, cons_2021_data
+    else:
+        return met_data, cons_data
+
+
+
+def interpolate(df, target_idx):
+    return (df                        # 1. start with your data
+       .reindex(target_idx)      # 2. align to the desired timestamps
+       .interpolate(method="time")  # 3. interpolate *within* the range
+       .ffill().bfill()             # 4. forward- and backward-fill anything still missing
+    )
+
+
+def join_consumption_meteorology(
+    cons_data: pd.DataFrame,
+    met_data: pd.DataFrame,
+    target_freq: str = "5min",
+) -> pd.DataFrame:
+    interp_method = "time"
+
+    met = met_data[["Production", "sr", "r", "t", "fs"]]
+    cons = cons_data[["Consumption"]]
+
+    cons.index = cons.index + pd.DateOffset(minutes=1)
+
+    start = max(cons.index.min(), met.index.min())
+    end   = min(cons.index.max(), met.index.max())
+    cons  = cons.loc[start:end].copy()
+    met   = met .loc[start:end].copy()
+
+    # there are dupes because of daylight savings time.
+    cons = cons[~cons.index.duplicated(keep="last")]
+
+    common_idx = pd.date_range(start, end, freq=target_freq)[:-2]
+
+    cons_interp = interpolate(cons, common_idx)
+    met_interp = interpolate(met, common_idx)
 
-    met_2021_data = met_data[(met_data.index >= START) & (met_data.index < END)]
-    cons_2021_data = cons_data[(cons_data.index >= START) & (cons_data.index < END)]
+    #    stitch together
+    # joined = pd.concat([cons_interp["Consumption"], met_interp["Production"]], axis=1)
+    joined = pd.concat([cons_interp, met_interp], axis=1)
 
-    return met_2021_data, cons_2021_data
+    return joined
 
 
 # BESS parameters are now in BatteryModel
@@ -63,6 +110,7 @@ def add_production_field(met_2021_data, parameters):
 
 
 def interpolate_and_join(met_2021_data, cons_2021_data):
+    print("this is obsoleted by join_consumption_meteorology(), do not use")
     applicable = 24*60*365 - 15 + 5
 
     demand_f = interp1d(range(0, 365*24*60, 15), cons_2021_data['Consumption'])