Test of no rte
Browse files- .gitignore +2 -1
- app.py +195 -248
.gitignore
CHANGED
|
@@ -1 +1,2 @@
|
|
| 1 |
-
/app_with_api.py
|
|
|
|
|
|
| 1 |
+
/app_with_api.py
|
| 2 |
+
/app_with_rte.py
|
app.py
CHANGED
|
@@ -24,11 +24,12 @@ def mongo_unavs_call(user_input_start_date, user_input_end_date, user_input_past
|
|
| 24 |
passw = "tN9XpCCQM2MtYDme"
|
| 25 |
host = "nucmonitordata.xxcwx9k.mongodb.net"
|
| 26 |
client = pymongo.MongoClient(
|
| 27 |
-
f"mongodb+srv://{user}:{passw}@{host}/?retryWrites=true&w=majority"
|
| 28 |
)
|
| 29 |
|
| 30 |
db = client["data"]
|
| 31 |
-
|
|
|
|
| 32 |
|
| 33 |
start_date = f"{user_input_start_date}T00:00:00"
|
| 34 |
end_date = f"{user_input_end_date}T23:59:59"
|
|
@@ -56,9 +57,13 @@ def mongo_unavs_call(user_input_start_date, user_input_end_date, user_input_past
|
|
| 56 |
}
|
| 57 |
]
|
| 58 |
|
| 59 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
|
| 61 |
-
return
|
| 62 |
|
| 63 |
# --------------------------------------------------------------------------------------- #
|
| 64 |
|
|
@@ -92,120 +97,7 @@ def add_total(data):
|
|
| 92 |
|
| 93 |
# --------------------------------------------------------------------------------------- #
|
| 94 |
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
# Function to create an authentication token. This token is then used in the HTTP requests to the API for authentication.
|
| 98 |
-
# It is necessary to receive data from RTE.
|
| 99 |
-
def get_oauth():
|
| 100 |
-
# ID from the user. This is encoded to base64 and sent in an HTTP request to receive the oauth token.
|
| 101 |
-
# This ID is from my account (RMP). However, another account can be created in the RTE API portal and get another ID.
|
| 102 |
-
joined_ID = '057e2984-edb3-4706-984b-9ea0176e74db:dc9df9f7-9f91-4c7a-910c-15c4832fb7bc'
|
| 103 |
-
b64_ID = base64.b64encode(joined_ID.encode('utf-8'))
|
| 104 |
-
b64_ID_decoded = b64_ID.decode('utf-8')
|
| 105 |
-
|
| 106 |
-
# Headers for the HTTP request
|
| 107 |
-
headers = {'Content-Type': 'application/x-www-form-urlencoded',
|
| 108 |
-
'Authorization': f'Basic {b64_ID_decoded}'}
|
| 109 |
-
api_url = 'https://digital.iservices.rte-france.com/token/oauth/'
|
| 110 |
-
# Call to the API and if successful, the response will be 200.
|
| 111 |
-
response = requests.post(api_url, headers=headers)
|
| 112 |
-
|
| 113 |
-
# When positive response, the token is retrieved
|
| 114 |
-
data = response.json()
|
| 115 |
-
oauth = data['access_token']
|
| 116 |
-
|
| 117 |
-
return(oauth)
|
| 118 |
-
|
| 119 |
-
# --------------------------------------------------------------------------------------- #
|
| 120 |
-
|
| 121 |
-
# This function does severall calls to the RTE API (because maximum time between start_date and end_date is 1 month)
|
| 122 |
-
# the argument past_photo is a boolean (True, False) that indicates if we want to make a photo from the past or not
|
| 123 |
-
# However, the past_photo part and past_date is not yet implemented.
|
| 124 |
-
def get_unavailabilities(usr_start_date, usr_end_date):
|
| 125 |
-
oauth = get_oauth()
|
| 126 |
-
print("Get Oauth done")
|
| 127 |
-
date_type = 'APPLICATION_DATE'
|
| 128 |
-
|
| 129 |
-
# Current year/month/day/hour/minute/second is calculated for the last call to the API. For instance, if today is 05/05/2023,
|
| 130 |
-
# the last call of the API will be from 01/05/2023 to 05/05/2023 (+current hour,minute,second).
|
| 131 |
-
current_datetime = datetime.datetime.now()
|
| 132 |
-
# current_year = current_datetime.strftime('%Y')
|
| 133 |
-
# current_month = current_datetime.strftime('%m')
|
| 134 |
-
# current_day = current_datetime.strftime('%d')
|
| 135 |
-
# current_hour = current_datetime.strftime('%H')
|
| 136 |
-
# current_minute = current_datetime.strftime('%M')
|
| 137 |
-
# current_second = current_datetime.strftime('%S')
|
| 138 |
-
|
| 139 |
-
# Headers for the HTTP request
|
| 140 |
-
headers = {'Host': 'digital.iservices.rte-france.com',
|
| 141 |
-
'Authorization': f'Bearer {oauth}'
|
| 142 |
-
}
|
| 143 |
-
|
| 144 |
-
# the responses object is where we are going to store all the responses from the API.
|
| 145 |
-
# Initially, current_datetime is included to know when we have called the API and all the
|
| 146 |
-
# individual results of the API (because each call is Maz 1 month) are stored in responses["results"]
|
| 147 |
-
responses = {"current_datetime": current_datetime.strftime("%m/%d/%Y, %H:%M:%S"),
|
| 148 |
-
"results":[]
|
| 149 |
-
}
|
| 150 |
-
|
| 151 |
-
# --------------------------- HERE HAVE TO GET THE RANGE OF DATES FROM START AND END AND PUT THEM INTO LIST --------------------------- #
|
| 152 |
-
# Convert start_date and end_date to datetime objects
|
| 153 |
-
usr_start_date = str(usr_start_date)
|
| 154 |
-
usr_end_date = str(usr_end_date)
|
| 155 |
-
start_date_obj = datetime.datetime.strptime(usr_start_date, "%Y-%m-%d").date()
|
| 156 |
-
end_date_obj = datetime.datetime.strptime(usr_end_date, "%Y-%m-%d").date()
|
| 157 |
-
# start_date_obj = usr_start_date
|
| 158 |
-
# end_date_obj = usr_end_date
|
| 159 |
-
# Initialize lists to store years and months
|
| 160 |
-
years = []
|
| 161 |
-
months = []
|
| 162 |
-
|
| 163 |
-
# Generate the range of years and months
|
| 164 |
-
current_date = start_date_obj
|
| 165 |
-
while current_date <= end_date_obj:
|
| 166 |
-
years.append(current_date.year)
|
| 167 |
-
months.append(current_date.month)
|
| 168 |
-
current_date += datetime.timedelta(days=1)
|
| 169 |
-
|
| 170 |
-
# Remove duplicates from the lists
|
| 171 |
-
years = list(set(years))
|
| 172 |
-
months = list(set(months))
|
| 173 |
-
years.sort()
|
| 174 |
-
months.sort()
|
| 175 |
-
print(years)
|
| 176 |
-
print(months)
|
| 177 |
-
# --------------------------- HERE HAVE TO GET THE RANGE OF DATES FROM START AND END AND PUT THEM INTO LIST --------------------------- #
|
| 178 |
-
|
| 179 |
-
# Loop to call the API all the necessary times.
|
| 180 |
-
for i in range(len(years)):
|
| 181 |
-
for j in range(len(months)):
|
| 182 |
-
# start_year and start_month of the current call to the API
|
| 183 |
-
start_year = years[i]
|
| 184 |
-
start_month = months[j]
|
| 185 |
-
# start_date is constructed. Now we only need to construct the end_date.
|
| 186 |
-
start_date = f'{start_year}-{start_month}-01T00:00:00%2B02:00'
|
| 187 |
-
|
| 188 |
-
if True:
|
| 189 |
-
# Calculate the number of days in the current month
|
| 190 |
-
_, num_days = monthrange(int(start_year), int(start_month))
|
| 191 |
-
end_date = f'{start_year}-{start_month}-{num_days}T23:59:59%2B02:00'
|
| 192 |
-
|
| 193 |
-
print(f'start date is {start_date}')
|
| 194 |
-
print(f'end date is {end_date}')
|
| 195 |
-
|
| 196 |
-
# Call to the API
|
| 197 |
-
api_url = f'https://digital.iservices.rte-france.com/open_api/unavailability_additional_information/v4/generation_unavailabilities?date_type={date_type}&start_date={start_date}&end_date={end_date}'
|
| 198 |
-
|
| 199 |
-
response = requests.get(api_url, headers=headers)
|
| 200 |
-
json_response = response.json()
|
| 201 |
-
responses["results"].append(json_response)
|
| 202 |
-
# print(responses)
|
| 203 |
-
return responses
|
| 204 |
-
|
| 205 |
-
# --------------------------------------------------------------------------------------- #
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
def nuc_monitor(usr_start_date, usr_end_date, past_date, mongo_db_data, rte_data):
|
| 209 |
# # Slightly changed metadata to fit the data from the RTE API: ST-LAURENT B 2 --> ST LAURENT 2, ....
|
| 210 |
|
| 211 |
plants_metadata = {"BELLEVILLE 1": 1310.0, "BELLEVILLE 2": 1310.0, "BLAYAIS 1": 910.0, "BLAYAIS 2": 910.0,
|
|
@@ -223,44 +115,6 @@ def nuc_monitor(usr_start_date, usr_end_date, past_date, mongo_db_data, rte_data
|
|
| 223 |
"ST LAURENT 2": 915.0, "TRICASTIN 1": 915.0, "TRICASTIN 2": 915.0, "TRICASTIN 3": 915.0,
|
| 224 |
"TRICASTIN 4": 915.0, "FESSENHEIM 1": 880.0, "FESSENHEIM 2": 880.0}
|
| 225 |
|
| 226 |
-
# --------------------- INITIAL DATA CLEANING FOR RTE DATA ------------------------ #
|
| 227 |
-
# unav_API = rte_data.json()
|
| 228 |
-
# rte_stuff = get_unavailabilities(usr_start_date, usr_end_date)
|
| 229 |
-
# rte_stuff = get_rte_data(usr_start_date, usr_end_date)
|
| 230 |
-
unav_API = rte_data
|
| 231 |
-
# print(unav_API)
|
| 232 |
-
# Store the unavailabilities in a list
|
| 233 |
-
unavailabilities = []
|
| 234 |
-
print("Unav")
|
| 235 |
-
for unavailabilities_API in unav_API['results']:
|
| 236 |
-
try:
|
| 237 |
-
unavailabilities.extend(unavailabilities_API.get('generation_unavailabilities', []))
|
| 238 |
-
except:
|
| 239 |
-
print('There was an error')
|
| 240 |
-
# print(unavailabilities_API)
|
| 241 |
-
rte_df = pd.DataFrame(unavailabilities)
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
def unpack_values(row):
|
| 245 |
-
if isinstance(row["values"], list):
|
| 246 |
-
for key, value in row["values"][0].items():
|
| 247 |
-
row[key] = value
|
| 248 |
-
return row
|
| 249 |
-
# Apply the function to each row in the DataFrame
|
| 250 |
-
rte_df = rte_df.apply(unpack_values, axis=1)
|
| 251 |
-
|
| 252 |
-
# Drop the original "values" column
|
| 253 |
-
rte_df.drop("values", axis=1, inplace=True)
|
| 254 |
-
|
| 255 |
-
# Unpack the unit column
|
| 256 |
-
rte_df2 = pd.concat([rte_df, pd.json_normalize(rte_df['unit'])], axis=1)
|
| 257 |
-
rte_df2.drop('unit', axis=1, inplace=True)
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
rte_nuclear_unav = rte_df2[(rte_df2["production_type"] == "NUCLEAR")]
|
| 261 |
-
|
| 262 |
-
# --------------------- INITIAL DATA CLEANING FOR RTE DATA ------------------------ #
|
| 263 |
-
|
| 264 |
|
| 265 |
# --------------------- INITIAL DATA CLEANING FOR MONGO DATA ------------------------ #
|
| 266 |
|
|
@@ -291,21 +145,12 @@ def nuc_monitor(usr_start_date, usr_end_date, past_date, mongo_db_data, rte_data
|
|
| 291 |
mongo_df2 = mongo_df_result
|
| 292 |
mongo_df2.rename(columns=lambda col: col.replace('unit.', ''), inplace=True)
|
| 293 |
|
| 294 |
-
|
| 295 |
-
|
| 296 |
# --------------------- INITIAL DATA CLEANING FOR MONGO DATA ------------------------ #
|
| 297 |
|
| 298 |
# Make the two dataframes have the same columns
|
| 299 |
mongo_unavs = mongo_df2.copy()
|
| 300 |
mongo_unavs.drop(columns="type", inplace=True)
|
| 301 |
|
| 302 |
-
rte_unavs = rte_nuclear_unav.copy()
|
| 303 |
-
rte_unavs.drop(columns="type", inplace=True)
|
| 304 |
-
|
| 305 |
-
# Merge dataframes
|
| 306 |
-
column_order = mongo_unavs.columns
|
| 307 |
-
# print(column_order)
|
| 308 |
-
merged_df = pd.concat([mongo_unavs[column_order], rte_unavs[column_order]], ignore_index=True)
|
| 309 |
# merged_df['updated_date'] = merged_df['updated_date'].astype(str)
|
| 310 |
|
| 311 |
# --------------------------- HERE IS THE CHANGE TO GET ONLY ACTIVE OR ACTIVE AND INACTIVE --------------------------- #
|
|
@@ -317,7 +162,7 @@ def nuc_monitor(usr_start_date, usr_end_date, past_date, mongo_db_data, rte_data
|
|
| 317 |
past_date_str = str(past_date)
|
| 318 |
current_datetime_str = current_datetime.strftime("%Y-%m-%d")
|
| 319 |
|
| 320 |
-
nuclear_unav =
|
| 321 |
|
| 322 |
# if photo_date == True:
|
| 323 |
# nuclear_unav = merged_df.copy()[(merged_df.copy()["production_type"] == "NUCLEAR") & (merged_df.copy()["updated_date"] <= past_date_str)]
|
|
@@ -511,11 +356,7 @@ def nuc_monitor(usr_start_date, usr_end_date, past_date, mongo_db_data, rte_data
|
|
| 511 |
return json_data
|
| 512 |
# -------------------------------------------------
|
| 513 |
|
| 514 |
-
|
| 515 |
-
def get_rte_data(start_date, end_date):
|
| 516 |
-
rte_data = get_unavailabilities(start_date, end_date)
|
| 517 |
-
print(rte_data)
|
| 518 |
-
return rte_data
|
| 519 |
@st.cache_data
|
| 520 |
def get_mongodb_data(start_date, end_date, past_date):
|
| 521 |
database_data = mongo_unavs_call(start_date, end_date, past_date)
|
|
@@ -524,8 +365,7 @@ def get_mongodb_data(start_date, end_date, past_date):
|
|
| 524 |
@st.cache_data
|
| 525 |
def get_nucmonitor_data(start_date, end_date, past_date):
|
| 526 |
mongo = get_mongodb_data(start_date, end_date, past_date)
|
| 527 |
-
|
| 528 |
-
response_nucmonitor = nuc_monitor(start_date, end_date, past_date, mongo, rte)
|
| 529 |
# nucmonitor_data = response_nucmonitor.json()
|
| 530 |
# nucmonitor_json = json.loads(nucmonitor_data)
|
| 531 |
print(response_nucmonitor)
|
|
@@ -540,18 +380,21 @@ def run_app():
|
|
| 540 |
start_date = st.date_input("Start Date")
|
| 541 |
end_date = st.date_input("End Date")
|
| 542 |
past_date = st.date_input("Cutoff Date")
|
| 543 |
-
|
| 544 |
|
|
|
|
|
|
|
| 545 |
with st.form("nucmonitor_form"):
|
| 546 |
submitted = st.form_submit_button("Get Nucmonitor")
|
| 547 |
|
| 548 |
if not submitted:
|
| 549 |
st.write("Form not submitted")
|
| 550 |
-
|
| 551 |
else:
|
| 552 |
st.write("Data received from Flask:")
|
| 553 |
df_nucmonitor = get_nucmonitor_data(start_date, end_date, current_date)
|
| 554 |
df_photo_date = get_nucmonitor_data(start_date, end_date, past_date)
|
|
|
|
| 555 |
current_date_str = str(current_date.strftime('%Y-%m-%d'))
|
| 556 |
past_date_str = str(past_date.strftime('%Y-%m-%d'))
|
| 557 |
st.write("Nucmonitor")
|
|
@@ -584,81 +427,185 @@ def run_app():
|
|
| 584 |
|
| 585 |
st.write(df_photo_date_2)
|
| 586 |
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
#
|
| 591 |
-
#
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
#
|
| 598 |
-
|
| 599 |
-
#
|
| 600 |
-
#
|
| 601 |
-
#
|
| 602 |
-
|
| 603 |
-
#
|
| 604 |
-
#
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
#
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
#
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
|
| 635 |
-
|
| 636 |
-
|
| 637 |
-
|
| 638 |
-
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
|
| 644 |
-
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 654 |
|
| 655 |
-
#
|
| 656 |
-
#
|
| 657 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 658 |
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 662 |
|
| 663 |
# # Set Nucmonitor as a dotted line until the current date
|
| 664 |
|
|
|
|
| 24 |
passw = "tN9XpCCQM2MtYDme"
|
| 25 |
host = "nucmonitordata.xxcwx9k.mongodb.net"
|
| 26 |
client = pymongo.MongoClient(
|
| 27 |
+
f"mongodb+srv://{user}:{passw}@{host}/?retryWrites=true&w=majority&connectTimeoutMS=5000"
|
| 28 |
)
|
| 29 |
|
| 30 |
db = client["data"]
|
| 31 |
+
collection_past_unavs = db["unavs"]
|
| 32 |
+
collection_unavs =db["unavs_update"]
|
| 33 |
|
| 34 |
start_date = f"{user_input_start_date}T00:00:00"
|
| 35 |
end_date = f"{user_input_end_date}T23:59:59"
|
|
|
|
| 57 |
}
|
| 58 |
]
|
| 59 |
|
| 60 |
+
result1 = list(collection_past_unavs.aggregate(pipeline))
|
| 61 |
+
result2 = list(collection_unavs.aggregate(pipeline))
|
| 62 |
+
|
| 63 |
+
# Merge the two lists of JSON results
|
| 64 |
+
merged_result = result1 + result2
|
| 65 |
|
| 66 |
+
return merged_result
|
| 67 |
|
| 68 |
# --------------------------------------------------------------------------------------- #
|
| 69 |
|
|
|
|
| 97 |
|
| 98 |
# --------------------------------------------------------------------------------------- #
|
| 99 |
|
| 100 |
+
def nuc_monitor(usr_start_date, usr_end_date, past_date, mongo_db_data):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
# # Slightly changed metadata to fit the data from the RTE API: ST-LAURENT B 2 --> ST LAURENT 2, ....
|
| 102 |
|
| 103 |
plants_metadata = {"BELLEVILLE 1": 1310.0, "BELLEVILLE 2": 1310.0, "BLAYAIS 1": 910.0, "BLAYAIS 2": 910.0,
|
|
|
|
| 115 |
"ST LAURENT 2": 915.0, "TRICASTIN 1": 915.0, "TRICASTIN 2": 915.0, "TRICASTIN 3": 915.0,
|
| 116 |
"TRICASTIN 4": 915.0, "FESSENHEIM 1": 880.0, "FESSENHEIM 2": 880.0}
|
| 117 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
|
| 119 |
# --------------------- INITIAL DATA CLEANING FOR MONGO DATA ------------------------ #
|
| 120 |
|
|
|
|
| 145 |
mongo_df2 = mongo_df_result
|
| 146 |
mongo_df2.rename(columns=lambda col: col.replace('unit.', ''), inplace=True)
|
| 147 |
|
|
|
|
|
|
|
| 148 |
# --------------------- INITIAL DATA CLEANING FOR MONGO DATA ------------------------ #
|
| 149 |
|
| 150 |
# Make the two dataframes have the same columns
|
| 151 |
mongo_unavs = mongo_df2.copy()
|
| 152 |
mongo_unavs.drop(columns="type", inplace=True)
|
| 153 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
# merged_df['updated_date'] = merged_df['updated_date'].astype(str)
|
| 155 |
|
| 156 |
# --------------------------- HERE IS THE CHANGE TO GET ONLY ACTIVE OR ACTIVE AND INACTIVE --------------------------- #
|
|
|
|
| 162 |
past_date_str = str(past_date)
|
| 163 |
current_datetime_str = current_datetime.strftime("%Y-%m-%d")
|
| 164 |
|
| 165 |
+
nuclear_unav = mongo_unavs.copy()[(mongo_unavs.copy()["production_type"] == "NUCLEAR") & (mongo_unavs.copy()["updated_date"] <= past_date_str)]
|
| 166 |
|
| 167 |
# if photo_date == True:
|
| 168 |
# nuclear_unav = merged_df.copy()[(merged_df.copy()["production_type"] == "NUCLEAR") & (merged_df.copy()["updated_date"] <= past_date_str)]
|
|
|
|
| 356 |
return json_data
|
| 357 |
# -------------------------------------------------
|
| 358 |
|
| 359 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
@st.cache_data
|
| 361 |
def get_mongodb_data(start_date, end_date, past_date):
|
| 362 |
database_data = mongo_unavs_call(start_date, end_date, past_date)
|
|
|
|
| 365 |
@st.cache_data
|
| 366 |
def get_nucmonitor_data(start_date, end_date, past_date):
|
| 367 |
mongo = get_mongodb_data(start_date, end_date, past_date)
|
| 368 |
+
response_nucmonitor = nuc_monitor(start_date, end_date, past_date, mongo)
|
|
|
|
| 369 |
# nucmonitor_data = response_nucmonitor.json()
|
| 370 |
# nucmonitor_json = json.loads(nucmonitor_data)
|
| 371 |
print(response_nucmonitor)
|
|
|
|
| 380 |
start_date = st.date_input("Start Date")
|
| 381 |
end_date = st.date_input("End Date")
|
| 382 |
past_date = st.date_input("Cutoff Date")
|
| 383 |
+
# winter_date = st.date_input("Winter Cutoff Date")
|
| 384 |
|
| 385 |
+
current_date = datetime.datetime.now()
|
| 386 |
+
|
| 387 |
with st.form("nucmonitor_form"):
|
| 388 |
submitted = st.form_submit_button("Get Nucmonitor")
|
| 389 |
|
| 390 |
if not submitted:
|
| 391 |
st.write("Form not submitted")
|
| 392 |
+
|
| 393 |
else:
|
| 394 |
st.write("Data received from Flask:")
|
| 395 |
df_nucmonitor = get_nucmonitor_data(start_date, end_date, current_date)
|
| 396 |
df_photo_date = get_nucmonitor_data(start_date, end_date, past_date)
|
| 397 |
+
# df_winter_date = get_nucmonitor_data(start_date, end_date, winter_date)
|
| 398 |
current_date_str = str(current_date.strftime('%Y-%m-%d'))
|
| 399 |
past_date_str = str(past_date.strftime('%Y-%m-%d'))
|
| 400 |
st.write("Nucmonitor")
|
|
|
|
| 427 |
|
| 428 |
st.write(df_photo_date_2)
|
| 429 |
|
| 430 |
+
# --------------------------------- AVERAGE EXPECTED AVAILABILITY M-1 M M+1 M+2 PIPELINE --------------------------------- #
|
| 431 |
+
|
| 432 |
+
# Create a Table that displays the forecast of each dataframe total for two months before date and two months after
|
| 433 |
+
# Filter dates for two months before and after the current date
|
| 434 |
+
# Define date ranges
|
| 435 |
+
two_months_before = (current_date - pd.DateOffset(months=2)).strftime('%Y-%m')
|
| 436 |
+
one_month_before = (current_date - pd.DateOffset(months=1)).strftime('%Y-%m')
|
| 437 |
+
one_month_after = (current_date + pd.DateOffset(months=1)).strftime('%Y-%m')
|
| 438 |
+
two_months_after = (current_date + pd.DateOffset(months=2)).strftime('%Y-%m')
|
| 439 |
+
|
| 440 |
+
# Assuming df is the DataFrame containing the date index and the 'Total' column
|
| 441 |
+
|
| 442 |
+
# # Convert the index to datetime if it's not already
|
| 443 |
+
# df_nucmonitor_2.index = pd.to_datetime(df_nucmonitor_2.index)
|
| 444 |
+
# df_photo_date_2.index = pd.to_datetime(df_photo_date_2.index)
|
| 445 |
+
|
| 446 |
+
# # Calculate monthly averages with date in yyyy-mm format
|
| 447 |
+
# monthly_average_nucmonitor = df_nucmonitor_2.resample('M').mean()
|
| 448 |
+
# monthly_average_photo_date = df_photo_date_2.resample('M').mean()
|
| 449 |
+
|
| 450 |
+
# Convert the index to datetime if it's not already
|
| 451 |
+
df_nucmonitor_2.index = pd.to_datetime(df_nucmonitor_2.index)
|
| 452 |
+
df_photo_date_2.index = pd.to_datetime(df_photo_date_2.index)
|
| 453 |
+
|
| 454 |
+
# Calculate monthly averages with date in yyyy-mm format
|
| 455 |
+
monthly_average_nucmonitor = df_nucmonitor_2.resample('M').mean()
|
| 456 |
+
monthly_average_nucmonitor.index = monthly_average_nucmonitor.index.strftime('%Y-%m')
|
| 457 |
+
|
| 458 |
+
monthly_average_photo_date = df_photo_date_2.resample('M').mean()
|
| 459 |
+
monthly_average_photo_date.index = monthly_average_photo_date.index.strftime('%Y-%m')
|
| 460 |
+
|
| 461 |
+
|
| 462 |
+
print(monthly_average_nucmonitor)
|
| 463 |
+
print(monthly_average_nucmonitor.index)
|
| 464 |
+
print(len(monthly_average_nucmonitor.index) < 5)
|
| 465 |
+
if (len(monthly_average_nucmonitor.index) < 5) or (two_months_before not in monthly_average_nucmonitor.index or two_months_after not in monthly_average_nucmonitor.index):
|
| 466 |
+
df_display_normal_bool = False
|
| 467 |
+
|
| 468 |
+
else:
|
| 469 |
+
print(two_months_before, one_month_before, current_date.strftime('%Y-%m'), one_month_after, two_months_after)
|
| 470 |
+
# Filter DataFrames based on date ranges
|
| 471 |
+
df_nucmonitor_filtered = monthly_average_nucmonitor[
|
| 472 |
+
(monthly_average_nucmonitor.index == two_months_before) |
|
| 473 |
+
(monthly_average_nucmonitor.index == one_month_before) |
|
| 474 |
+
(monthly_average_nucmonitor.index == current_date.strftime('%Y-%m')) |
|
| 475 |
+
(monthly_average_nucmonitor.index == one_month_after) |
|
| 476 |
+
(monthly_average_nucmonitor.index == two_months_after)
|
| 477 |
+
]
|
| 478 |
+
|
| 479 |
+
df_photo_date_filtered = monthly_average_photo_date[
|
| 480 |
+
(monthly_average_photo_date.index == two_months_before) |
|
| 481 |
+
(monthly_average_photo_date.index == one_month_before) |
|
| 482 |
+
(monthly_average_photo_date.index == current_date.strftime('%Y-%m')) |
|
| 483 |
+
(monthly_average_photo_date.index == one_month_after) |
|
| 484 |
+
(monthly_average_photo_date.index == two_months_after)
|
| 485 |
+
]
|
| 486 |
+
|
| 487 |
+
# Display the filtered DataFrames
|
| 488 |
+
st.write(f"Forecast update {current_date_str}")
|
| 489 |
+
st.write(df_nucmonitor_filtered)
|
| 490 |
+
st.write(f"Forecast update {past_date_str}")
|
| 491 |
+
st.write(df_photo_date_filtered)
|
| 492 |
+
|
| 493 |
+
current_forecast_update = df_nucmonitor_filtered.tolist()
|
| 494 |
+
past_forecast_update = df_photo_date_filtered.tolist()
|
| 495 |
+
delta = [current - past for current, past in zip(current_forecast_update, past_forecast_update)]
|
| 496 |
+
|
| 497 |
+
print('Dates:', [two_months_before, one_month_before, current_date.strftime('%Y-%m'), one_month_after, two_months_after])
|
| 498 |
+
print(f"Forecast update {current_date_str}", current_forecast_update)
|
| 499 |
+
print(f"Forecast update {past_date_str}", past_forecast_update,)
|
| 500 |
+
print('Delta', delta)
|
| 501 |
+
|
| 502 |
+
# Create a DataFrame for display
|
| 503 |
+
data_avg_expected_normal = {
|
| 504 |
+
'Dates': [two_months_before, one_month_before, current_date.strftime('%Y-%m'), one_month_after, two_months_after],
|
| 505 |
+
f"Forecast update {current_date_str}": current_forecast_update,
|
| 506 |
+
f"Forecast update {past_date_str}": past_forecast_update,
|
| 507 |
+
'Delta': delta
|
| 508 |
+
}
|
| 509 |
+
df_display_normal_bool = True
|
| 510 |
+
|
| 511 |
+
# --------------------------------- AVERAGE EXPECTED AVAILABILITY M-1 M M+1 M+2 PIPELINE --------------------------------- #
|
| 512 |
+
|
| 513 |
+
# --------------------------------- AVERAGE EXPECTED AVAILABILITY WINTER PIPELINE --------------------------------- #
|
| 514 |
+
# Create a Table that displays the forecast of each dataframe for the Winter months (Nov, Dec, Jan, Feb, Mar)
|
| 515 |
+
|
| 516 |
+
# Create a table that gets the forecast for winter. This involves creating a new dataframe with
|
| 517 |
+
# only the winter months with the total of each day, and another dataframe with the average of each month. Each month
|
| 518 |
+
# included will only be 20xx-11, 12, and 20xx+1-01, 02, 03
|
| 519 |
|
| 520 |
+
# Define date ranges for winter months
|
| 521 |
+
# winter_start_date = current_date.replace(month=11, day=1)
|
| 522 |
+
# winter_end_date = (current_date.replace(year=current_date.year+1, month=3, day=31))
|
| 523 |
+
winter_start = f"{current_date.year}-11"
|
| 524 |
+
winter_end = f"{current_date.year+1}-03"
|
| 525 |
+
winter_start_str = str(winter_start)
|
| 526 |
+
winter_end_str = str(winter_end)
|
| 527 |
+
print("winter_start_str", winter_start)
|
| 528 |
+
print("winter_end_str", winter_end)
|
| 529 |
+
print("monthly_average_nucmonitor.index", monthly_average_nucmonitor.index)
|
| 530 |
+
print(monthly_average_nucmonitor.index == winter_start)
|
| 531 |
+
print(monthly_average_nucmonitor.index == winter_end)
|
| 532 |
+
if monthly_average_nucmonitor.index.any() != winter_start or monthly_average_nucmonitor.index.any() != winter_end:
|
| 533 |
+
df_display_winter_bool = False
|
| 534 |
+
|
| 535 |
+
else:
|
| 536 |
+
# Filter DataFrames based on winter date range
|
| 537 |
+
df_nucmonitor_winter = monthly_average_nucmonitor[(monthly_average_nucmonitor.index >= winter_start_str) & (monthly_average_nucmonitor.index <= winter_end_str)]
|
| 538 |
+
|
| 539 |
+
df_photo_date_winter = monthly_average_photo_date[(monthly_average_photo_date.index >= winter_start_str) & (monthly_average_photo_date.index <= winter_end_str)]
|
| 540 |
+
|
| 541 |
+
# Display the forecast DataFrames for winter
|
| 542 |
+
st.title("Forecast for Winter Months")
|
| 543 |
+
st.write(f"Forecast for {current_date.year}-{current_date.year+1} (Nov, Dec, Jan, Feb, Mar)")
|
| 544 |
+
st.write("Nucmonitor Forecast:")
|
| 545 |
+
st.write(df_nucmonitor_winter)
|
| 546 |
+
st.write("Photo Date Forecast:")
|
| 547 |
+
st.write(df_photo_date_winter)
|
| 548 |
+
|
| 549 |
+
current_winter_forecast_update = df_nucmonitor_winter.tolist()
|
| 550 |
+
past_winter_forecast_update = df_photo_date_winter.tolist()
|
| 551 |
+
winter_delta = [current - past for current, past in zip(current_winter_forecast_update, past_winter_forecast_update)]
|
| 552 |
+
print("current_winter_forecast_update:", current_winter_forecast_update)
|
| 553 |
+
print("past_winter_forecast_update:", past_winter_forecast_update)
|
| 554 |
+
|
| 555 |
+
# Create a DataFrame for display
|
| 556 |
+
data_avg_expected_winter = {
|
| 557 |
+
'Dates': [f'Nov-{current_date.year}', f'Dec-{current_date.year}', f'Jan-{current_date.year+1}', f'Feb-{current_date.year+1}', f'Mar-{current_date.year+1}'],
|
| 558 |
+
f"Forecast update {current_date_str}": current_winter_forecast_update,
|
| 559 |
+
f"Forecast update {past_date_str}": past_winter_forecast_update,
|
| 560 |
+
'Delta': winter_delta
|
| 561 |
+
}
|
| 562 |
+
print(data_avg_expected_winter)
|
| 563 |
+
df_display_winter_bool = True
|
| 564 |
|
| 565 |
+
# --------------------------------- AVERAGE EXPECTED AVAILABILITY WINTER PIPELINE --------------------------------- #
|
| 566 |
+
|
| 567 |
+
# --------------------------------- VISUALIZE --------------------------------- #
|
| 568 |
+
if df_display_normal_bool:
|
| 569 |
+
df_display_normal = pd.DataFrame(data_avg_expected_normal)
|
| 570 |
+
# Display the DataFrame as a horizontal table
|
| 571 |
+
st.write("Table 1. Average expected availability on the French nuclear fleet (MW) - M-1, M, M+1, M+2, M+3")
|
| 572 |
+
st.table(df_display_normal)
|
| 573 |
+
|
| 574 |
+
if df_display_winter_bool:
|
| 575 |
+
df_display_winter = pd.DataFrame(data_avg_expected_winter)
|
| 576 |
+
st.write(f"Table 2. Average expected availability on the French nuclear fleet (MW) - Winter {winter_start}/{winter_end}")
|
| 577 |
+
st.table(df_display_winter)
|
| 578 |
+
|
| 579 |
+
# Line charts of the forecasts (need to combine them so they appear in the same chart)
|
| 580 |
+
st.write("Current forecast")
|
| 581 |
+
st.line_chart(df_nucmonitor_2)
|
| 582 |
+
|
| 583 |
+
st.write("Previous forecast")
|
| 584 |
+
st.line_chart(df_photo_date_2)
|
| 585 |
+
# Create a new dataframe out of df_nucmonitor_2 call real_forecast that contains df_nucmonitor_2 up until current_date
|
| 586 |
+
|
| 587 |
+
# Slice the DataFrame to include data up until current_date
|
| 588 |
+
real_forecast = df_nucmonitor_2.loc[df_nucmonitor_2.index <= current_date_str]
|
| 589 |
+
|
| 590 |
+
# Winter forecast still not the correct one, this is just a placeholder
|
| 591 |
+
# winter_forecast = df_nucmonitor_2.loc[(df_nucmonitor_2.index >= winter_start_date) & (df_nucmonitor_2.index <= winter_end_date)]
|
| 592 |
+
|
| 593 |
+
# Optionally, if you want to reset the index
|
| 594 |
+
# real_forecast = real_forecast.reset_index()
|
| 595 |
+
print(real_forecast)
|
| 596 |
+
st.write("Real forecast")
|
| 597 |
+
st.line_chart(real_forecast)
|
| 598 |
+
|
| 599 |
+
# Combine dataframes
|
| 600 |
+
# combined_df = pd.concat([df_nucmonitor_2, df_photo_date_2, real_forecast, winter_forecast], axis=1)
|
| 601 |
+
combined_df = pd.concat([df_nucmonitor_2, df_photo_date_2, real_forecast], axis=1)
|
| 602 |
+
|
| 603 |
+
# combined_df.columns = [f'Forecast {current_date_str}', f'Forecast {past_date_str}', 'Real Forecast', f'Winter forecast {winter_start}/{winter_end}']
|
| 604 |
+
combined_df.columns = [f'Forecast {current_date_str}', f'Forecast {past_date_str}', 'Real Forecast']
|
| 605 |
+
|
| 606 |
+
print(combined_df)
|
| 607 |
+
st.write(f"Graph 1. {start_date} to {end_date}")
|
| 608 |
+
st.line_chart(combined_df)
|
| 609 |
|
| 610 |
# # Set Nucmonitor as a dotted line until the current date
|
| 611 |
|