JHA_Solarmon_API

app.py

@@ -3,19 +3,28 @@ import pandas as pd
 import numpy as np
 from datetime import timedelta
 import datetime
-from meteo_functions import get_meteo_data, get_air_quality_data, get_forecast_meteo_data, get_air_quality_forecast
-from predictions import predict 
+from meteo_functions import *
+from predictions import predict, predict_solarmon_history
 import gc
+import os
+
+username = os.environ.get("USERNAME")
+password = os.environ.get("PASSWORD")
 
 def clear_memory():
     gc.collect() 
 
-st.title("Predikce výkonu FVE ABA")
-#t.header("this is a header")
-#sst.subheader("subheader")
-#st.markdown("This is **Markdown**")
-#st.caption("small text")
+def use_history_model(previous_day, date_utc):
+    # st.subheader(f"Predikce výkonu pro: {date_utc}:")
+    df_meteo = get_meteo_data(previous_day, date_utc)
+    df_air_quality = get_air_quality_data(previous_day, date_utc)
+    data = df_meteo.merge(df_air_quality, on="DT", how="inner")
+    st.write("Meteorologická data:")
+    st.dataframe(data)
+    predict(data)
+    clear_memory()
 
+st.title("Predikce výkonu FVE ABA")
 today = datetime.date.today()  
 max_date = today + datetime.timedelta(days=4)
 
@@ -25,30 +34,45 @@ with st.form(key="sample_form"):
 
 if submit_button:   
     previous_day = date_utc - timedelta(days=1) 
-    if date_utc < today - datetime.timedelta(days=1):
-       # st.subheader(f"Predikce výkonu pro: {date_utc}:")
+    if date_utc < today - datetime.timedelta(days=60): # delsi nez dva mesice - data z API k dispozici jen 2 mesice zpetne 
+       use_history_model(previous_day= previous_day, date_utc = date_utc)
+    elif date_utc < today:
+        data_solarmon = load_and_check_data_solarmon(date_utc- timedelta(days=1), date_utc, username, password) # solarmon-api
+        if data_solarmon.isnull().values.any():
+            st.warning("Nepodařilo se načíst data ze systému SOLARMON.")
+            use_history_model(previous_day= previous_day, date_utc = date_utc)
+        else:
+            df_meteo = get_meteo_data(previous_day, date_utc)
+            df_air_quality = get_air_quality_data(previous_day, date_utc)
+            predpoved_meteo = create_time_cycles(df_meteo.merge(df_air_quality, on="DT", how="inner"))
+            
+            # st.write("Data - Solarmon:")
+            # st.dataframe(data_solarmon)
+            
+            # st.write("Předpověď - OpenMeteo:")
+            # st.dataframe(predpoved_meteo)
 
-        df_meteo = get_meteo_data(previous_day, date_utc)
-        df_air_quality = get_air_quality_data(previous_day, date_utc)
-
-        data = df_meteo.merge(df_air_quality, on="DT", how="inner")
-        st.write("Meteorologická data:")
-        st.dataframe(data)
-        predict(data)
-        clear_memory()
+            predpoved_meteo['DT'] = pd.to_datetime(predpoved_meteo['DT']).dt.tz_localize('UTC')
+            data = predpoved_meteo.merge(data_solarmon, on="DT", how="inner")
+            st.write("Dataset:")
+            st.dataframe(data)
 
-    elif previous_day < date_utc <= today + datetime.timedelta(days=4):
-       # st.header(f"Data pro: {date_utc}:")
+            predict_solarmon_history(data)
+            clear_memory()        
+    elif previous_day < date_utc < today + datetime.timedelta(days=5):
+        # st.header(f"Data pro: {date_utc}:")
         df_meteo = get_forecast_meteo_data(previous_day, date_utc)
         df_air_quality = get_air_quality_forecast(previous_day, date_utc)
         data = df_meteo.merge(df_air_quality, on="DT", how="inner")
+
         #st.write("Budouci data:")
         #st.dataframe(df_meteo)
         #st.dataframe(df_air_quality)
-        st.write("Meteorologická data:")
+
+        st.write("Meteorologická předpověď:")
         st.dataframe(data)
         predict(data)
         clear_memory()
 
     else:
-        st.warning("Predikce je dostupná pouze pro následujících 5 dnů.")
+       st.warning("Predikce je dostupná pouze pro následujících 5 dnů.")

input_preprocessor.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c2fcafee3a50a9029417ae11ba929ac8af042978438e334b858dcc2831f01bd
+size 5908

meteo_functions.py

@@ -2,6 +2,9 @@
 import pandas as pd
 import requests
 import numpy as np
+import streamlit as st
+import json
+from datetime import date, timedelta
 
 lat = 49.13114
 lon = 15.18067
@@ -136,7 +139,93 @@ def get_air_quality_forecast(start_date, end_date):
     
     return df_air_quality
 
+def get_data_for_day(day: date, username: str, password: str) -> list[dict]:
+    """
+    Stáhne data z API pomocí POST požadavku pro daný den.
+
+    :param day: Datum typu `datetime.date`
+    :param username: Uživatelské jméno pro API
+    :param password: Heslo pro API
+    :return: Seznam měření (dictů) pro daný den
+    """
+    url = f"https://aba.solarmon.eu/rest-server/?q=getDataPredMod&date={day.strftime('%Y-%m-%d')}"
+    payload = {
+        'username': username,
+        'password': password
+    }
+
+    try:
+        response = requests.post(url, data=payload)
+        response.raise_for_status()
+        data = response.json()
+        data = json.loads(data)
+        return data.get("data", [])
+    except (requests.RequestException, ValueError) as e:
+        st.error(f"Chyba při načítání dat pro {day}: {e}")
+        return []
+
+def load_and_check_data_solarmon(start_date: date, end_date: date, username: str, password: str) -> pd.DataFrame:
+    current_date = start_date
+    all_data = []
+
+    while current_date <= end_date + timedelta(days=1):# pridame dalsi den kvuli poslednim 2 hodinam
+        daily_data = pd.DataFrame(get_data_for_day(current_date, username, password))
+        all_data.append(daily_data)
+        current_date += timedelta(days=1)
+
+    solarmon_meteo = pd.concat(all_data).sort_index()  
+
+    solarmon_meteo = pd.DataFrame(solarmon_meteo)
+    solarmon_meteo["time"] = pd.to_datetime(solarmon_meteo["time"])  # + pd.Timedelta(minutes=5) # experimentalne zjistit zda je potreba
+    solarmon_meteo["time"] = solarmon_meteo["time"].dt.tz_localize("Europe/Prague").dt.tz_convert("UTC")
+    solarmon_meteo = solarmon_meteo.set_index("time")
+
+    # převody typů a kontrola
+    solarmon_meteo["int_sol_irr"] = pd.to_numeric(solarmon_meteo["int_sol_irr"], errors='coerce')
+    solarmon_meteo["tmp_module"] = pd.to_numeric(solarmon_meteo["tmp_module"], errors='coerce')
+    solarmon_meteo["energy"] = pd.to_numeric(solarmon_meteo["energy"], errors='coerce')
+
+    # Resample na hodinové intervaly
+    solarmon_meteo = solarmon_meteo.resample('1h').agg({
+        'int_sol_irr': 'mean',
+        'wind_vel': 'mean',
+        'tmp_amb': 'mean',
+        'tmp_module': 'mean',
+        'energy': 'sum'
+    })
+    solarmon_meteo['energy'] = solarmon_meteo['energy'] / 1000 
+
+    solarmon_meteo.rename_axis("DT", inplace=True)
+    solarmon_meteo.rename(
+    columns={
+        'int_sol_irr': 'Me',
+        'tmp_module': 'tp',
+        'energy': 'output',
+        'tmp_amb': 'to'
+    },
+    inplace=True
+    )
+    solarmon_meteo.drop(['wind_vel'], axis=1, inplace=True)
+
+    solarmon_meteo.index = pd.to_datetime(solarmon_meteo.index)
+    start_datetime = pd.to_datetime(start_date).tz_localize('UTC')
+    end_datetime = pd.to_datetime(end_date).tz_localize('UTC') + pd.Timedelta(days=1)
+    solarmon_meteo = solarmon_meteo[(solarmon_meteo.index >= start_datetime) & (solarmon_meteo.index < end_datetime)]
+
+    # solarmon_meteo.isnull().values.any() kontroluje zda neni NaN hodnota
+    solarmon_meteo = solarmon_meteo.reset_index(drop=False) # resetovani indexu
+
+    # df_meteo["DT"] = df_meteo["DT"].dt.tz_localize("UTC")
+    #df_meteo = get_meteo_data(previous_date, selected_date)
+    #df_air_quality = get_air_quality_data(previous_date, selected_date)
+    #df_meteo = df_meteo.merge(df_air_quality, on="DT", how="inner")
 
+    
+    
+    #df = df.merge(df_meteo, on="DT", how="inner")
+    #df = create_time_cycles(df)
+    return solarmon_meteo
+    # return df[2:26]
 
 
     

model_hist/input_preprocessor.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bd687f252949c2015c68b81ae16b337d61ead28e6ec1f7bbb934fedf1f1bbfe
+size 5756

model_hist/output_preprocessor.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a21576128cba609962c3396a6e7397f4a2765ede43a3e4c89d7481b6269ed1e5
+size 1705

output_preprocessor.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a21576128cba609962c3396a6e7397f4a2765ede43a3e4c89d7481b6269ed1e5
+size 1705

predictions.py

@@ -3,7 +3,7 @@ import pandas as pd
 import numpy as np
 from meteo_functions import create_time_cycles
 import joblib
-from preprocessing_functions import create_sequences, plot_solar_power_prediction, load_model, load_transformers
+from preprocessing_functions import create_sequences, create_sequences_solarmon, plot_solar_power_prediction, load_model, load_model_solarmon, load_transformers, load_transformers_solarmon
 from tensorflow import keras
 import gc
 
@@ -58,3 +58,56 @@ def predict(data):
     gc.collect()  
 
     return None
+
+def predict_solarmon_history(data: pd.DataFrame):
+    """
+    Jedná se o přesnější predikce pro aktuální a minulé dny s využitím API od Solarmonu
+    data:: dataframe s daty za minuly den
+    predpoved:: dataframe s predpovedi pro aktualni den
+    """
+    st.divider()
+    # Načtění transformátorů
+    input_preprocessor, output_scaler = load_transformers_solarmon()
+
+    # transformace vstupnich prom
+    X_dataset = input_preprocessor.transform(data)
+    X_dataset = pd.DataFrame(X_dataset, columns=input_preprocessor.get_feature_names_out(), index=data.index)
+    # st.write(data.keys())
+    # st.write(input_preprocessor.get_feature_names_out())
+    rename_map = {
+        "yeo_minmax__RAD": "RAD",
+        "yeo_minmax__Relative_Humidity_2m": "Relative_Humidity_2m",
+        "yeo_minmax__PM10": "PM10",
+        "yeo_minmax__output": "output",
+        "yeo_minmax__Me": "Me",
+        "yeo_standard__Cloud_Cover": "Cloud_Cover",
+        "minmax__Temperature_2m": "Temperature_2m",
+        "minmax__wind_u": "wind_u",
+        "minmax__wind_v": "wind_v",
+        "minmax__Surface_Pressure": "Surface_Pressure",
+        "minmax__Ozone": "Ozone",
+        "minmax__tp": "tp",
+        "remainder__sin_hour": "sin_hour",
+        "remainder__cos_hour": "cos_hour",
+        "remainder__cos_day_of_year": "cos_day_of_year",
+        "remainder__sin_day_of_year": "sin_day_of_year"
+    }
+    X_dataset = X_dataset.rename(columns=rename_map)
+    
+  # pro open meteo
+    features = ["Me", "output","tp","Relative_Humidity_2m", "Surface_Pressure", 'Cloud_Cover',
+              'sin_hour', 'cos_hour', 'cos_day_of_year', "sin_day_of_year", "wind_u", "wind_v", "Ozone", "PM10"]
+    future_features = ["Me", "tp", "Relative_Humidity_2m", "Surface_Pressure", 'Cloud_Cover', "wind_u", "wind_v", "Ozone", "PM10", 'sin_hour', 'cos_hour', 'cos_day_of_year', "sin_day_of_year"]
+    x = create_sequences_solarmon(X_dataset, window=24, horizon=24, past_features=features, future_features=future_features)
+    model = load_model_solarmon()
+    y_pred = model.predict(x) # predikce
+    y_pred_trans = np.array([output_scaler.inverse_transform(y_pred[:, i].reshape(-1, 1)).flatten() for i in range(y_pred.shape[1])]).T
+
+    y_pred_trans[y_pred_trans < 1] = 0
+    st.write("**Predikované hodnoty výkonu (kW) pro jednotlivé hodiny:**")
+    st.write(y_pred_trans, use_container_width=True)
+
+    my_plt = plot_solar_power_prediction(y_pred_trans, df_true=data[['output']])
+    st.pyplot(my_plt)
+    
+    return None

preprocessing_functions.py

@@ -8,12 +8,24 @@ import joblib
 from pathlib import Path
 
 MODEL_PATH = "output_predictions_to_meteo_smape_25.keras"
+MODEL_PATH_SOLARMON = "model_24h_base_on_Me_open_meteo_weather_predictions.keras"
+
+MODEL_PATH_SOLARMON_HIST = "model_hist\model_24h_predictions_hist.keras"
+
+TRANSFORMER_INPUT_PATH_HIST = "model_hist\input_preprocessor.pkl"
+TRANSFORMER_OUTPUT_PATH_HIST = "model_hist\output_preprocessor.pkl"
 
 @st.cache_resource
 def load_model(): 
     model = tf.keras.models.load_model(MODEL_PATH)
     return model
 
+@st.cache_resource
+def load_model_solarmon(): 
+   # model = tf.keras.models.load_model(MODEL_PATH_SOLARMON)
+    model = tf.keras.models.load_model(MODEL_PATH_SOLARMON_HIST)
+    return model
+
 @st.cache_resource 
 def load_transformers():
     input_preprocessor = joblib.load('input_preprocessor_meteo_to_smape25.pkl')
@@ -21,6 +33,63 @@ def load_transformers():
 
     return input_preprocessor, output_scaler
 
+@st.cache_resource 
+def load_transformers_solarmon():
+    #input_preprocessor = joblib.load('input_preprocessor.pkl')
+    #output_scaler = joblib.load('output_preprocessor.pkl')
+    
+    input_preprocessor = joblib.load(TRANSFORMER_INPUT_PATH_HIST)
+    output_scaler = joblib.load(TRANSFORMER_OUTPUT_PATH_HIST)    
+    return input_preprocessor, output_scaler
+
+import numpy as np
+
+def create_sequences_solarmon(data, window, horizon, past_features, future_features):
+    """
+    Vytvoří sekvence vstupních dat a odpovídající cílové hodnoty pro trénování LSTM modelu.
+
+    Parametry:
+    ----------
+    data : pandas.DataFrame
+        DataFrame obsahující časové řady.
+    window : int
+        Počet časových kroků v minulosti.
+    horizon : int
+        Počet časových kroků do budoucnosti.
+    past_features : list
+        Seznam sloupců, které budou použity jako vstupní vlastnosti v minulosti.
+    future_features : list
+        Seznam sloupců, které budou použity jako vstupní vlastnosti v budoucnosti.
+    target : str
+        Název sloupce, který bude použit jako cílová hodnota.
+
+    Návratové hodnoty:
+    -------------------
+    X : numpy.ndarray
+        Pole tvaru (vzorky, window, past_features + future_features), obsahující sekvence vstupních dat.
+    y : numpy.ndarray
+        Pole tvaru (vzorky, horizon), obsahující odpovídající cílové hodnoty.
+    """
+
+    # Vytvoření historických sekvencí (past_features)
+    X_past = np.lib.stride_tricks.sliding_window_view(
+        data[past_features].values, (window, len(past_features))
+    )[:-horizon, :, :]
+
+    X_past = np.squeeze(X_past, axis=1)  # Výstup: (vzorky, window, len(past_features))
+
+    # Vytvoření budoucích sekvencí (future_features)
+    X_future = np.lib.stride_tricks.sliding_window_view(
+        data[future_features].values, (window, len(future_features))
+    )[horizon-1 : len(X_past) + horizon-1, :, :]
+
+    X_future = np.squeeze(X_future, axis=1)  # Výstup: (vzorky, window, len(future_features))
+
+    # Spojení historických a budoucích proměnných do jednoho pole
+    X = np.concatenate([X_past, X_future], axis=2)  # (vzorky, window, past_features + future_features)
+
+    return X
+
 def create_sequences(data, window, horizon, past_features, future_features):
     """
     Vytvoří sekvence vstupních dat a odpovídající cílové hodnoty pro trénování LSTM modelu.
@@ -64,26 +133,36 @@ def create_sequences(data, window, horizon, past_features, future_features):
 
     return X
 
-def plot_solar_power_prediction(y_pred_trans):
+import numpy as np
+import matplotlib.pyplot as plt
+
+def plot_solar_power_prediction(y_pred_trans, df_true=None):
     """
     Vytvoří graf predikce výkonu fotovoltaické elektrárny v průběhu dne.
     
     Args:
         y_pred_trans (numpy.ndarray): Pole s predikovanými hodnotami výkonu (kW) ve tvaru (1, 24).
-    
+        df_true (pd.DataFrame, optional): DataFrame se skutečnými hodnotami výkonu. 
+                                          Musí obsahovat sloupec 'output' s 24 hodnotami.
+
     Returns:
         plt.Figure: Graf pro zobrazení.
     """
     hours = np.arange(24)
     
     plt.figure(figsize=(10, 6))
-    plt.plot(hours, y_pred_trans.flatten(), marker='o', label='Výkon (kW)')
+    plt.plot(hours, y_pred_trans.flatten(), marker='o', label='Predikce (kW)', color='tab:blue')
+    
+    if df_true is not None and 'output' in df_true.columns and len(df_true) >= 24:
+        plt.plot(hours, df_true['output'].values[23:-1], marker='x', label='Skutečný výkon (kW)', color='tab:orange')
+    
     plt.xlabel('Hodiny (UTC)')
     plt.ylabel('Výkon (kW)')
-    plt.title('Predikce výkonu fotovoltaické elektrárny v průběhu dne')
-    plt.xticks(hours) 
+    plt.title('Predikce vs. skutečný výkon FVE v průběhu dne')
+    plt.xticks(hours)
     plt.grid(True)
     plt.legend()
     
     return plt
 
+