uploadable data files doing something, flaky yet

app.py

@@ -21,9 +21,17 @@ from bess import BatteryParameters, BatteryModel
 # !wget "https://static.renyi.hu/ai-shared/daniel/pq/pq_terheles_2021_adatok.tsv"
 
 OLD_DATASET = False
+MINI = False
 
+if MINI:
+    met_default_filename = 'PL_44527.2101.csv.gz'
+    cons_default_filename = 'pq_terheles_202101_adatok.tsv'
+else:
+    met_default_filename = 'PL_44527.19-21.csv.gz'
+    cons_default_filename = 'pq_terheles_2021_adatok.tsv'
 
-met_2021_data, cons_2021_data = read_datasets(old_dataset=OLD_DATASET)
+
+met_2021_data, cons_2021_data = read_datasets(met_default_filename, cons_default_filename, old_dataset=OLD_DATASET)
 
 
 # TODO some gui to upload consumption data.
@@ -35,20 +43,39 @@ def recalculate(ui):
 
     solar_parameters = SolarParameters(solar_cell_num=ui.solar_cell_num, panel_power_at_NOCT=ui.solar_cell_nominal_capacity)
 
+    met_data = met_2021_data
+    cons_data = cons_2021_data
+
+    changed_datasource = False
+
+    if ui.meteorology_csv is not None:
+        met_filename = ui.meteorology_csv.name
+        changed_datasource = True
+    else:
+        met_filename = met_default_filename
     if ui.consumption_csv is not None:
         print("Hm, we should do something with this CSV.", ui.consumption_csv.name)
+        cons_filename = ui.consumption_csv.name
+        changed_datasource = True
+    else:
+        cons_filename = cons_default_filename
+
+    if changed_datasource:
+        met_data, cons_data = read_datasets(met_filename, cons_filename, old_dataset=OLD_DATASET)
+    else:
+        met_data, cons_data = met_2021_data, cons_2021_data
 
-    add_production_field(met_2021_data, solar_parameters)
+    add_production_field(met_data, solar_parameters)
 
     if OLD_DATASET:
         # i've obsoleted this, but it's not a 100% replacement,
         # it treats daylight savings changes differently.
         # new version drops repeated hour and interpolates skipped hour.
-        all_data = interpolate_and_join(met_2021_data, cons_2021_data)
+        all_data = interpolate_and_join(met_data, cons_data)
     else:
         all_data = join_consumption_meteorology(
-            cons_2021_data,
-            met_2021_data,
+            met_data,
+            cons_data,
             target_freq="5min",
         )
 
@@ -207,8 +234,8 @@ with gr.Blocks() as ui:
                 base_price_HUFpkWh = gr.Number(value=60, label="Base energy price [HUF/kWh]")
 
             with gr.Accordion("Custom consumption/meteorology data", open=False):
-                consumption_csv = gr.File(label="Upload consumption CSV", file_types=[".csv"])
-                meteorology_csv = gr.File(label="Upload meteorology CSV", file_types=[".csv"])
+                consumption_csv = gr.File(label="Upload consumption CSV", file_types=[".csv", ".gz", ".tsv"])
+                meteorology_csv = gr.File(label="Upload meteorology CSV", file_types=[".csv", ".gz", ".tsv"])
 
             run_btn = gr.Button("Run Simulation")
 

data_processing.py

@@ -4,32 +4,22 @@ import pandas as pd
 from scipy.interpolate import interp1d   
 
 
-PATH_PREFIX = "./"
-
 START = f"2021-01-01"
 END = f"2022-01-01"
 
 
-def read_datasets(mini=False, old_dataset=False):
+def read_datasets(met_filename, cons_filename, old_dataset=False):
     # old_dataset mode is needed if we plug this into interpolate_and_join()
     # rather than join_consumption_meteorology().
 
-    if mini:
-        met_filename = 'PL_44527.2101.csv.gz'
-        cons_filename = 'pq_terheles_202101_adatok.tsv'
-    else:
-        met_filename = 'PL_44527.19-21.csv.gz'
-        cons_filename = 'pq_terheles_2021_adatok.tsv'
-
-    #@title ### Preprocessing meteorologic data
-    met_data = pd.read_csv(PATH_PREFIX + met_filename, compression='gzip', sep=';', skipinitialspace=True, na_values='n/a', skiprows=[0, 1, 2, 3, 4])
+    # Preprocessing meteorologic data
+    met_data = pd.read_csv(met_filename, compression='gzip', sep=';', skipinitialspace=True, na_values='n/a', skiprows=[0, 1, 2, 3, 4])
     met_data['Time'] = met_data['Time'].astype(str)
     date_time = met_data['Time'] = pd.to_datetime(met_data['Time'], format='%Y%m%d%H%M')
     met_data = met_data.set_index('Time')
 
-
-    #@title ### Preprocessing consumption data
-    cons_data = pd.read_csv(PATH_PREFIX + cons_filename, sep='\t', skipinitialspace=True, na_values='n/a', decimal=',')
+    # Preprocessing consumption data
+    cons_data = pd.read_csv(cons_filename, sep='\t', skipinitialspace=True, na_values='n/a', decimal=',')
     cons_data['Time'] = pd.to_datetime(cons_data['Korrigált időpont'], format='%m/%d/%y %H:%M')
     cons_data = cons_data.set_index('Time')
     cons_data['Consumption'] = cons_data['Hatásos teljesítmény [kW]']
@@ -57,8 +47,8 @@ def interpolate(df, target_idx):
 
 
 def join_consumption_meteorology(
-    cons_data: pd.DataFrame,
     met_data: pd.DataFrame,
+    cons_data: pd.DataFrame,
     target_freq: str = "5min",
 ) -> pd.DataFrame:
     interp_method = "time"