price prediction v1

.hw_api_key

@@ -0,0 +1 @@
+e76DLTFJzKMcADkC.BEKfBjsL91EXI2qgX9PPOcryLnsqaXWlME4EBWIPrMH8wLLUpdcmN6H3uqZuSO7J

app.py

@@ -0,0 +1,167 @@
+import gradio as gr
+import pandas as pd
+import numpy as np
+import hopsworks
+import joblib
+import os
+import json
+from entsoe import EntsoePandasClient
+from datetime import datetime, timedelta, date
+from pandas import json_normalize
+import tensorflow as tf
+from keras.layers import LSTM
+from urllib.request import urlopen
+from sklearn.preprocessing import LabelEncoder, StandardScaler
+
+
+# from keras.layers import *
+# from keras.models import Sequential
+# from keras.layers import Dense
+# from keras.layers import LSTM
+# from keras.layers import Dropout
+
+project = hopsworks.login()
+fs = project.get_feature_store()
+
+mr = project.get_model_registry()
+model = mr.get_model("SE3_elec_price_model", version=2)
+model_dir = model.download()
+model = joblib.load(model_dir + "/electricity_price.pkl")
+
+def get_price_forecast():
+    today, tomorrow = get_date()
+
+    df_entsoe = get_entsoe_data(today, tomorrow)
+
+    # get timestamps that temp dataset should match on
+    entsoe_earliest = df_entsoe["datetime"].iloc[0]
+    entsoe_latest = df_entsoe["datetime"].iloc[-1]
+    df_temp = get_temp(entsoe_earliest, entsoe_latest)
+    
+    df = df_entsoe.merge(df_temp, how='inner', on='datetime')
+    df.set_index('datetime',inplace=True)
+
+    ## pre-process before predict
+    sc_x=StandardScaler()
+    df_scaled=sc_x.fit_transform(df)
+    sc_y=StandardScaler()
+    sc_y=sc_y.fit(df[['day_ahead_price']])
+
+    step_back=24
+    no_records=len(df_scaled)
+    no_cols=4
+    X_train_shape_pred=[]
+    for i in range(step_back,no_records):
+        X_train_shape_pred.append(df_scaled[i-step_back:i])
+    X_train_shape_pred=np.array(X_train_shape_pred)
+    print(X_train_shape_pred.shape)
+
+    ## predict
+    pred_price = model.predict(X_train_shape_pred)
+    final_pred=sc_y.inverse_transform(pred_price)
+    print(final_pred.shape)
+
+    # append time for prediction
+    predict_time_from = datetime.fromtimestamp(entsoe_latest / 1e3)
+    # calculating timestamps for the next 24 h
+    timestamp_list = [predict_time_from + timedelta(hours=x) for x in range(len(final_pred))]
+    
+    # iterating through timestamp_list
+    # for i, x in enumerate(timestamp_list):
+    #     print(x, final_pred[i])
+    # print(final_pred.shape)
+    df_prediction = pd.DataFrame(
+        {'Datetime': timestamp_list,
+        'Price forecast [EUR/MWh]': final_pred.flatten(),
+        })
+    
+    #df_predictions = pd.DataFrame([timestamp_list, final_pred], columns=["datetime", "Price prediction"])
+    # print(len(final_pred), len(timestamp_list), len(final_pred), len(final_pred[0]))
+
+    return df_prediction
+    #[today, temp, day_ahead_price, pred_price, total_load, total_generation]
+
+# # Returns yesterday and tomorrows date
+def get_date():
+    # yesterday = datetime.today() - timedelta(days=1)
+    # yesterday = yesterday.date().strftime('%Y%m%d')
+    # tomorrow = (datetime.strptime(yesterday, '%Y%m%d') + timedelta(days=2)).strftime('%Y%m%d')
+
+    date_from = datetime.now() - timedelta(days=3)
+    date_from = date_from.date().strftime('%Y%m%d')
+    date_to = (datetime.strptime(date_from, '%Y%m%d') + timedelta(days=4)).strftime('%Y%m%d')
+
+    return date_from, date_to
+
+def get_entsoe_data(date_from, date_to):
+    # Client
+    client = EntsoePandasClient(api_key="cb3a29b2-3276-4a4c-aba3-6507120d99be")
+
+    # Date and country
+    start = pd.Timestamp(date_from, tz='Europe/Stockholm')
+    end = pd.Timestamp(date_to, tz='Europe/Stockholm')
+    country_code = 'SE_3'  
+
+    df_day_price = client.query_day_ahead_prices(country_code, start=start,end=end)
+    df_generation_per_prod = client.query_generation(country_code, start=start,end=end, psr_type=None)    
+    df_load = client.query_load(country_code, start=start,end=end)
+
+    df_entsoe = df_generation_per_prod.join(df_day_price.rename("day_ahead_price"))
+    df_entsoe = df_entsoe.join(df_load)
+
+    df_entsoe_clean = df_entsoe.reset_index()
+    df_entsoe_clean = df_entsoe_clean.rename(columns = {'index':'DateTime'})
+    df_entsoe_clean['DateTime'] = df_entsoe_clean.DateTime.values.astype('int64') // 10 ** 6
+
+    col_list = ["Hydro Water Reservoir", "Nuclear", "Other", "Solar", "Wind Onshore"]
+    df_entsoe_clean['total_generation'] = df_entsoe_clean[list(col_list)].sum(axis=1)
+
+    df_entsoe_clean.drop(col_list + ["Fossil Gas"], axis=1, inplace=True)
+    df_entsoe_clean.rename(columns={"Actual Load": "total_load", "DateTime":"datetime"}, inplace=True)
+
+    return df_entsoe_clean.tail(48)
+
+def get_temp(timeseries_from, timeseries_to):
+
+    url = "https://opendata-download-metobs.smhi.se/api/version/latest/parameter/1/station/71420/period/latest-months/data.json"
+    response = urlopen(url)
+
+    # convert response to json, to dataframe
+    data_json = json.loads(response.read())
+    df_smhi_data = json_normalize(data_json['value']) 
+
+    # extract only the temperature in the time stamp interval
+    df_smhi_data = df_smhi_data.loc[(df_smhi_data['date'] >= timeseries_from) & (df_smhi_data['date'] <= timeseries_to)]
+    df_smhi_data = df_smhi_data.reset_index().rename(columns = {'date':'datetime'})
+
+    df_smhi_data.drop(["index", "quality"], axis=1, inplace=True)
+    df_smhi_data["value"] = df_smhi_data["value"].astype(float)
+    df_smhi_data.rename(columns={"value": "temperature"}, inplace=True)
+
+    return df_smhi_data
+
+demo = gr.Interface(
+    fn = get_price_forecast,
+    title = "SE3 Electricity Day-Ahead Price Prediction",
+    description ="SE3 Electricity Day-Ahead Price Prediction, based on electricity production, generation and temperature",
+    allow_flagging = "never",
+    inputs = [],
+    outputs = [
+        gr.DataFrame(x="datetime", y="Price prediction [EUR/MWh]")
+        # gr.Textbox(label="Date"),
+        # gr.Textbox(label="Temperature Forecast [℃]"),
+        # gr.Textbox(label="Total Load Forecast [MWh]"),
+        # gr.Textbox(label="Total Generation Forecast [MWh]"),
+        # gr.Textbox(label="Predicted Day-Ahead Price [EUR/MWh]"),   
+    ]
+)
+
+demo.launch()
+
+
+# TODO: we have only the demand predictions for two days ago, so we have two options
+#  - skip EIA demand forecast (no comparison)
+#  - show prediction for two days ago
+# TODO: allow custom date/temp input (default to today)?
+# TODO: have done some versioning mess (see reqs file)
+

requirements.txt

@@ -0,0 +1,7 @@
+hopsworks
+joblib
+tensorflow
+pandas
+keras
+entsoe
+scikit-learn==1.0.2