big code drop with different main_* entry points, tuned evolution, and less logging.

v2/architecture.py

@@ -29,6 +29,14 @@ def add_dummy_predictions(all_data_with_predictions):
     # we predict zero before we have data, no big deal:
     all_data_with_predictions.loc[all_data_with_predictions.index[:cons_shift], 'Consumption_prediction'] = 0
     all_data_with_predictions.loc[all_data_with_predictions.index[:prod_shift], 'Production_prediction'] = 0
+    '''
+    all_data_with_predictions['Consumption'].plot()
+    all_data_with_predictions['Production'].plot()
+    plt.show()
+    all_data_with_predictions['Consumption_prediction'].plot()
+    all_data_with_predictions['Production_prediction'].plot()
+    plt.show()
+    '''
 
 
 # even mock-er class than usual.
@@ -81,7 +89,7 @@ def simulator(battery_model, prod_cons, decider):
 
     consumption_fees_np = prod_cons['Consumption_fees'].to_numpy()
 
-    print("Simulating for", len(demand_np), "time steps. Each step is", step_in_minutes, "minutes.")
+    # print("Simulating for", len(demand_np), "time steps. Each step is", step_in_minutes, "minutes.")
     soc_series = []
     # by convention, we only call end user demand, demand,
     # and we only call end user consumption, consumption.
@@ -113,6 +121,7 @@ def simulator(battery_model, prod_cons, decider):
         consumption_from_network = 0
         discarded_production = 0
         network_used_to_charge = 0
+        bess_from_solar = 0
 
         unsatisfied_demand = demand
         remaining_production = production
@@ -135,7 +144,9 @@ def simulator(battery_model, prod_cons, decider):
             remaining_production = 0
             if decision == Decision.DISCHARGE:
                 # we try to cover the rest from BESS
-                unsatisfied_demand = battery_model.satisfy_demand(unsatisfied_demand)
+                still_unsatisfied_demand = battery_model.satisfy_demand(unsatisfied_demand)
+                consumption_from_bess = unsatisfied_demand - still_unsatisfied_demand
+                unsatisfied_demand = still_unsatisfied_demand
             # we cover the rest from network
             consumption_from_network = unsatisfied_demand
             unsatisfied_demand = 0
@@ -147,6 +158,7 @@ def simulator(battery_model, prod_cons, decider):
             if remaining_production > 0:
                 # exploitable production still remains:
                 discarded_production = battery_model.charge(remaining_production) # remaining_production [kW]
+                bess_from_solar = remaining_production - discarded_production
 
         if decision == Decision.NETWORK_CHARGE:
             # that is some random big number, the actual charge will be
@@ -167,6 +179,9 @@ def simulator(battery_model, prod_cons, decider):
         consumption_from_bess_series.append(consumption_from_bess)
         discarded_production_series.append(discarded_production)
 
+        assert np.isclose(consumption_from_solar + consumption_from_bess + consumption_from_network, demand + consumption_from_network_to_bess)
+        assert np.isclose(consumption_from_solar + discarded_production + bess_from_solar, production)
+
     soc_series = np.array(soc_series)
     consumption_from_solar_series = np.array(consumption_from_solar_series)
     consumption_from_network_series = np.array(consumption_from_network_series)
@@ -189,7 +204,7 @@ def simulator(battery_model, prod_cons, decider):
     fifteen_minute_surdemands_in_kwh = (fifteen_minute_demands_in_kwh - decider.precalculated_supplier.peak_demand).clip(lower=0)
     demand_charges = fifteen_minute_surdemands_in_kwh * decider.precalculated_supplier.surcharge_per_kwh
     total_network_fee = consumption_charge_series.sum() + demand_charges.sum()
-    print(f"Total network fee {total_network_fee / 10 ** 6} MHUF.")
+    # print(f"Total network fee {total_network_fee / 10 ** 6} MHUF.")
 
     if DO_VIS:
         demand_charges.plot()
@@ -209,17 +224,20 @@ def simulator(battery_model, prod_cons, decider):
     return results, total_network_fee
 
 
-
-def optimizer(decider_class, battery_model, all_data_with_predictions, precalculated_supplier):
-    number_of_generations = 1
-    population_size = 10
+def optimizer(decider_class, precalculated_supplier, battery_model, all_data_with_predictions):
+    number_of_generations = 10
+    population_size = 20
     collected_loss_values = []
     def objective_function(params):
-        print("Simulating with parameters", params)
+        # there was an evil numpy view bug that shuffled all results randomly.
+        params = params.copy()
+
+        # print("Simulating with parameters", params)
         decider = decider_class(params, precalculated_supplier)
         t = time.perf_counter()
         results, total_network_fee = simulator(battery_model, all_data_with_predictions, decider)
         collected_loss_values.append((params, total_network_fee))
+        print(params, total_network_fee)
         return total_network_fee
 
     def clipper_function(params):
@@ -246,9 +264,7 @@ def visualize_collected_loss_values(collected_loss_values):
 
     # losses -= losses.min() ; losses /= losses.max()
 
-    plt.scatter(all_params[:, 0], all_params[:, 1], c=range(len(all_params)))
-    plt.show()
-
+    # plt.scatter(all_params[:, 0], all_params[:, 1], c=range(len(all_params))) ; plt.show()
 
     from mpl_toolkits.mplot3d import Axes3D
 
@@ -258,10 +274,52 @@ def visualize_collected_loss_values(collected_loss_values):
     ax = fig.add_subplot(111, projection='3d')
 
     # Scatter plot
-    ax.scatter(all_params[:, 0], all_params[:, 1], losses)
+    ax.scatter(all_params[:, 0], all_params[:, 1], losses, c=range(len(all_params)))
     plt.show()
 
 
+def main_param_grid(precalculated_supplier, battery_model, all_data_with_predictions):
+    N = 11
+    losses = np.zeros((N, N))
+    losses[:, :] = np.nan
+    xlim = [5, 60*24*3 - 1]
+    ylim = [-50, 50]
+    for i, x in enumerate(np.linspace(*xlim, N)):
+        for j, y in enumerate(np.linspace(*ylim, N)):
+            decider = Decider(np.array([x, y]), precalculated_supplier)
+            results, total_network_fee = simulator(battery_model, all_data_with_predictions, decider)
+            # print(x, y, total_network_fee)
+            losses[j, i] = total_network_fee
+    # losses[0, -1] = np.nan
+
+    # + is list extend.
+    # the aspect means square pixels
+    plt.imshow(losses, extent=xlim + ylim, aspect=(xlim[1]-xlim[0])/(ylim[1]-ylim[0]))
+    plt.xlabel('Surdemand lookahead window size [minutes]')
+    plt.ylabel('Lookahead surdemand [kW]')
+    plt.show()
+
+
+def main_inspect_params(precalculated_supplier, battery_model, all_data_with_predictions):
+    def inspect_params(params):
+        decider = Decider(params, precalculated_supplier)
+        results, total_network_fee = simulator(battery_model, all_data_with_predictions, decider)
+        print(params, total_network_fee)
+        print(np.histogram(results['decisions'].to_numpy()))
+        date_range = ("2021-01-15", "2021-02-01")
+        date_range = ("2021-08-15", "2021-09-01")
+
+        plotly_fig = plotly_visualize_simulation(results, date_range=date_range)
+        plotly_fig.update_layout(title=dict(text=f"{total_network_fee/1e6:0.2f} MFt"))
+        plotly_fig.show()
+
+        plotly_fig_2 = plotly_visualize_monthly(results)
+        plotly_fig_2.update_layout(title=dict(text=f"{total_network_fee/1e6:0.2f} MFt"))
+        plotly_fig_2.show()
+
+    inspect_params(np.array([5, 39]))
+    inspect_params(np.array([1400, 50]))
+
 
 def main():
     np.random.seed(1)
@@ -285,7 +343,7 @@ def main():
     time_interval_h = time_interval_min / 60
 
     # for faster testing:
-    DATASET_TRUNCATED_SIZE = None
+    DATASET_TRUNCATED_SIZE = 10000
     if DATASET_TRUNCATED_SIZE is not None:
         print("Truncating dataset to", DATASET_TRUNCATED_SIZE, "datapoints, that is", DATASET_TRUNCATED_SIZE * time_interval_h / 24, "days")
         all_data = all_data.iloc[:DATASET_TRUNCATED_SIZE]
@@ -303,36 +361,22 @@ def main():
 
     battery_model = BatteryModel(capacity_Ah=600, time_interval_h=time_interval_h)
 
-    decider_class = RandomDecider
+    # now that we've finally set everything up, we can do various things, hence several main_*().
 
-    # TODO this is super unfortunate:
-    # Consumption_fees travels via all_data_with_predictions,
-    # peak_demand and surcharge_per_kwh travels via precalculated_supplier of decider.
-    decider_init_mean, decider_init_scale = decider_class.initial_params()
-    decider = decider_class(decider_init_mean, precalculated_supplier)
+    # main_param_grid(precalculated_supplier, battery_model, all_data_with_predictions) ; exit()
 
-    t = time.perf_counter()
-    results, total_network_fee = simulator(battery_model, all_data_with_predictions, decider)
-    print("Simulation runtime", time.perf_counter() - t, "seconds.")
+    # main_inspect_params(precalculated_supplier, battery_model, all_data_with_predictions) ; exit()
 
-    if DO_VIS:
-        results['soc_series'].plot()
-        plt.title('soc_series')
-        plt.show()
 
-    best_params, collected_loss_values = optimizer(decider_class, battery_model, all_data_with_predictions, precalculated_supplier)
-    visualize_collected_loss_values(collected_loss_values)
+    decider_class = Decider
 
-    decider = decider_class(best_params, precalculated_supplier)
-    results, total_network_fee = simulator(battery_model, all_data_with_predictions, decider)
+    # TODO this is super unfortunate:
+    # Consumption_fees travels via all_data_with_predictions,
+    # peak_demand and surcharge_per_kwh travels via precalculated_supplier of decider.
 
-    date_range = ("2021-01-01", "2021-02-01")
-    date_range = ("2021-07-01", "2021-08-01")
-    plotly_fig = plotly_visualize_simulation(results, date_range=date_range)
-    plotly_fig.show()
+    best_params, collected_loss_values = optimizer(decider_class, precalculated_supplier, battery_model, all_data_with_predictions)
+    visualize_collected_loss_values(collected_loss_values)
 
-    plotly_fig_2 = plotly_visualize_monthly(results)
-    plotly_fig_2.show()
 
 if __name__ == '__main__':
     main()

v2/decider.py

@@ -95,13 +95,13 @@ class Decider:
         # it should not be hardwired.
         HARDWIRED_THRESHOLD_FOR_CHEAP_POWER_HUF_PER_KWH = 20 # [HUF/kWh]
         if mean_surdemand_kw > self.lookahead_surdemand_kw and current_fee <= HARDWIRED_THRESHOLD_FOR_CHEAP_POWER_HUF_PER_KWH:
-            return Decision.NETWORK_CHARGE
-
+            decision = Decision.NETWORK_CHARGE
         # peak shaving
-        if deficit_kwh > self.precalculated_supplier.peak_demand:
-            return Decision.DISCHARGE
+        elif deficit_kwh > self.precalculated_supplier.peak_demand:
+            decision = Decision.DISCHARGE
         else:
-            return Decision.PASSIVE
+            decision = Decision.PASSIVE
+        return decision
 
     # this is called by the optimizer so that meaningless parameter settings are not attempted
     # we could vectorize this easily, but it's not a bottleneck, the simulation is.

v2/evolution_strategies.py

@@ -16,7 +16,7 @@ def evolution_strategies_optimizer(objective_function, clipper_function,
                                    number_of_generations,
                                    population_size):
     # Initialize parameters
-    mutation_scale = 0.05
+    mutation_scale = 0.1
     selection_ratio = 0.5
     selected_size = int(population_size * selection_ratio)
 
@@ -27,30 +27,25 @@ def evolution_strategies_optimizer(objective_function, clipper_function,
     for generation in range(number_of_generations):
         # Evaluate fitness
         fitness = np.array([objective_function(individual) for individual in population])
-        
+
         # Select the best individuals
         selected_indices = np.argsort(fitness)[:selected_size]
         selected = population[selected_indices]
-        
+
         # Reproduce (mutate)
         offspring = selected + np.random.normal(loc=0, scale=init_scale * mutation_scale, size=(selected_size, len(init_mean)))
         clip_params(offspring, clipper_function) # in-place
-        
-        # Replacement: Here we simply generate new candidates around the selected ones
-        population[:selected_size] = selected
-        population[selected_size:] = offspring
-        
+
         # Logging
         best_fitness = fitness[selected_indices[0]]
-        best_index = np.argmin(fitness)
-        best_solution = population[best_index]
+        best_solution = selected[0].copy()
         print(f"Generation {generation + 1}: Best Fitness = {best_fitness}", f"Best solution so far: {best_solution}")
 
+        # Replacement: Here we simply generate new candidates around the selected ones
+        population[:selected_size] = selected
+        population[selected_size:] = offspring
 
-    # Best solution
-    best_index = np.argmin(fitness)
-    best_solution = population[best_index]
-    print(f"Best solution found: {best_solution}")
+    print(f"Best solution found: {best_solution} with loss {best_fitness}")
     return best_solution