# %% import gradio as gr import numpy as np import requests import pandas as pd import hopsworks import joblib import torch from torch import nn import os import httpx import datetime import json from urllib.request import Request, urlopen import random from datetime import datetime from sklearn.preprocessing import StandardScaler # %% project = hopsworks.login() fs = project.get_feature_store() # %% #Load api keys weather_api_key = os.getenv("WEATHER_API_KEY") pressure_api_key = os.getenv("PRESSURE_API_KEY") flight_api_key = os.getenv("FLIGHT_API_KEY") # %% #Mappings icao_to_iata_map = { "KDTW": "DTW", "KLAS": "LAS", "KPHL": "PHL", "KDEN": "DEN", "KCLT": "CLT", "KSEA": "SEA", "KMCO": "MCO", "KFLL": "FLL", "KIAD": "IAD", "KIAH": "IAH", "KSFO": "SFO", "KEWR": "EWR", "KMIA": "MIA", "KJFK": "JFK", "KLAX": "LAX", "KORD": "ORD", "KATL": "ATL", } iata_to_icao_map = {v: k for k, v in icao_to_iata_map.items()} wac_map = { "BOS": 13, "CLT": 36, "DEN": 82, "DTW": 43, "EWR": 21, "FLL": 33, "IAD": 38, "IAH": 74, "JFK": 22, "LAS": 85, "LAX": 91, "MCO": 33, "MIA": 33, "ORD": 41, "PHL": 23, "SEA": 93, "SFO": 91, "ATL": 34, } weather_features = [ ("dewpoint", "value"), "relative_humidity", ("remarks_info", "precip_hourly", "value"), ("remarks_info", "temperature_decimal", "value"), ("visibility", "value"), ("wind_direction", "value"), ("wind_gust", "value"), ("wind_speed", "value"), ] pressure_features = [("pressure", "hpa")] flight_features = [ "flight_date", ("departure", "iata"), ("departure", "delay"), ("departure", "scheduled"), ("arrival", "iata"), ("arrival", "delay"), ("arrival", "scheduled"), ] airport_id_map={ "CLT": 11057, "DEN": 11292, "DTW": 11433, "EWR": 11618, "FLL": 11697, "IAD": 12264, "IAH": 12266, "JFK": 12478, "LAS": 12889, "LAX": 12892, "MCO": 13204, "MIA": 13303, "ORD": 13930, "PHL": 14100, "SEA": 14747, "SFO": 14771, "ATL": 10397, } label_tranformed_airport_id_map={'ATL': 0, 'CLT': 1, 'DEN': 2, 'DTW': 3, 'EWR': 4, 'FLL': 5, 'IAD': 6, 'IAH': 7, 'JFK': 8, 'LAS': 9, 'LAX': 10, 'MCO': 11, 'MIA': 12, 'ORD': 13, 'PHL': 14, 'SEA': 15, 'SFO': 16} # Create predefined lists for origin and destination airport codes airports = [ "PHL - PHILADELPHIA INTERNATIONAL AIRPORT, PA US", "SEA - SEATTLE TACOMA AIRPORT, WA US", "JFK - JFK INTERNATIONAL AIRPORT, NY US", "DEN - DENVER INTERNATIONAL AIRPORT, CO US", "EWR - NEWARK LIBERTY INTERNATIONAL AIRPORT, NJ US", "LAS - MCCARRAN INTERNATIONAL AIRPORT, NV US", "MCO - ORLANDO INTERNATIONAL AIRPORT, FL US", "ATL - ATLANTA HARTSFIELD JACKSON INTERNATIONAL AIRPORT, GA US", "FLL - FORT LAUDERDALE INTERNATIONAL AIRPORT, FL US", "DTW - DETROIT METRO AIRPORT, MI US", "IAD - WASHINGTON DULLES INTERNATIONAL AIRPORT, VA US", "ORD - CHICAGO OHARE INTERNATIONAL AIRPORT, IL US", "LAX - LOS ANGELES INTERNATIONAL AIRPORT, CA US", "CLT - CHARLOTTE DOUGLAS AIRPORT, NC US", "MIA - MIAMI INTERNATIONAL AIRPORT, FL US", "IAH - HOUSTON INTERCONTINENTAL AIRPORT, TX US", "SFO - SAN FRANCISCO INTERNATIONAL AIRPORT, CA US"] # %% #Class definition needed due to the way pytorch neural networks are saved and loaded by python # A solution, if needed, would be to save the state dict of the NN and load the model via load_state_dict class NeuralNetwork(nn.Module): def __init__(self, input_size): super(NeuralNetwork, self).__init__() self.fc1 = nn.Linear(input_size, 128) self.relu = nn.ReLU() self.dropout = nn.Dropout(0.2) self.fc2 = nn.Linear(128, 64) self.output = nn.Linear(64, 1) def forward(self, x): x = self.fc1(x) x = self.relu(x) x = self.dropout(x) x = self.fc2(x) x = self.relu(x) x = self.output(x) return x #Load model from model registry mr = project.get_model_registry() model = mr.get_model("flight_delay_model", version=2) model_dir = model.download() model = joblib.load(model_dir + "/flight_delay_model.pkl") # get the original train test splits used for training the model and use it for fitting scaler feature_view = fs.get_feature_view(name="flight_data_v3",version=1) X_train, X_test, y_train, y_test = feature_view.get_train_test_split(training_dataset_version=3) #fit scaler the same way it was used for training scaler = StandardScaler() X_train_tensor = torch.tensor(X_train.values, dtype=torch.float32) X_train_scaled = scaler.fit_transform(X_train_tensor) X_train_tensor = torch.tensor(X_train_scaled, dtype=torch.float32) results = pd.DataFrame(columns=["Origin Airport", "Destination Airport", "Scheduled Departure", "Scheduled Arrival", "Predicted Departure Delay"]) # %% def get_weather_data(selected_airports_iata): # Input: list of selected airports in IATA code # Make API call to fetch weather data for the airport # Process and return weather data responses = {} for airport in selected_airports_iata: print(f"Getting weather for {airport}") request = Request( f"https://avwx.rest/api/metar/{iata_to_icao_map[airport]}", headers={"Authorization": weather_api_key}, ) response_body = urlopen(request).read() response_json = json.loads(response_body) responses[airport] = response_json weather_data = [] for airport in selected_airports_iata: response_json = responses[airport] data = {"airport": airport} data["HourlyDewPointTemperature"] = response_json["remarks_info"][ "dewpoint_decimal" ]["value"] data["HourlyRelativeHumidity"] = response_json["relative_humidity"] if response_json["remarks_info"]["precip_hourly"] is not None: data["HourlyPrecipitation"] = response_json["remarks_info"]["precip_hourly"][ "value" ] else: data["HourlyPrecipitation"] = 0 data["HourlyDryBulbTemperature"] = response_json["remarks_info"][ "temperature_decimal" ]["value"] data["HourlyVisibility"] = response_json["visibility"]["value"] data["HourlyWindDirection"] = response_json["wind_direction"]["value"] if response_json["wind_gust"] is not None: data["HourlyWindGustSpeed"] = response_json["wind_gust"]["value"] else: data["HourlyWindGustSpeed"] = 0 data["HourlyWindSpeed"] = response_json["wind_speed"]["value"] weather_data.append(data) weather_data = pd.DataFrame(weather_data) #weather_data.info() return weather_data # %% def get_pressure_data(selected_airports_iata): # Input: list of selected airports in IATA code responses={} url = "https://api.checkwx.com/metar/KJFK/decoded" #response = requests.request("GET", url, headers={"X-API-Key": pressure_api_key}) for airport in selected_airports_iata: print(f"Getting pressure for {airport}") request = Request( f"https://api.checkwx.com/metar/{iata_to_icao_map[airport]}/decoded", headers={"X-API-Key": pressure_api_key}, ) response_body = urlopen(request).read() response_json = json.loads(response_body) responses[airport] = response_json pressure_data = [] for airport in selected_airports_iata: response_json = responses[airport] data = {"airport": airport} data["HourlyStationPressure"] = response_json["data"][0]["barometer"]["hpa"] pressure_data.append(data) pressure_data = pd.DataFrame(pressure_data) #pressure_data.info() return pressure_data # %% def get_flight_data(origin, destination,scheduled_dep_time, scheduled_arr_time): # Input: origin airport IATA code, destination airport IATA code, # and dep and arr time in HH:MM 24 hour format current_datetime = datetime.now() # Extract different date-related information day_of_week = current_datetime.weekday() day_of_month = current_datetime.day year = current_datetime.year month = current_datetime.month origin_wac = wac_map[origin] origin_airport_id = label_tranformed_airport_id_map[origin] # Mapping destination to dest_WAC and dest_airport_id dest_wac = wac_map[destination] dest_airport_id = label_tranformed_airport_id_map[destination] # Create a DataFrame for the given airport codes airport_df = pd.DataFrame({ #"Year":[year], "month":[month], "Day_of_month":[day_of_month], "Day_of_week":[day_of_week], "origin": [origin], "origin_airport_id": [origin_airport_id], "origin_WAC": [origin_wac], "dest": [destination], "dest_airport_id": [dest_airport_id], "dest_WAC": [dest_wac], "CRS_DEP_TIME":[int(scheduled_dep_time.replace(":", ""))], "CRS_ARR_TIME":[int(scheduled_arr_time.replace(":", ""))], "airport":[origin] }) #print(airport_df.info()) #print(airport_df) return airport_df # %% # Define the function to predict flight delay based on user inputs def predict_delay(origin, destination,scheduled_dep_time, scheduled_arr_time): #test code to try running Gradio app origin=origin.split()[0] destination=destination.split()[0] #error handling try: # check if correct hour format by trying to convert to datetime objects datetime.strptime(scheduled_dep_time, "%H:%M") datetime.strptime(scheduled_arr_time, "%H:%M") except ValueError: # else error return "Error: Please enter scheduled departure and arrival times in 24-hour format (HH:MM)." if origin == destination: return "Error: Origin and destination airports cannot be the same. Please select different airports." #Get data from APIs selected_airports_iata = [origin,destination] weather_data=get_weather_data(selected_airports_iata) pressure_data=get_pressure_data(selected_airports_iata) flight_data=get_flight_data(origin, destination,scheduled_dep_time, scheduled_arr_time) #Merge data weather_delay_data = pd.merge(pressure_data, weather_data, on="airport") # fix order of columns so that it is same as in training weather_delay_data=weather_delay_data.reindex(sorted(weather_delay_data.columns), axis=1) #merge columns flight_weather_data=pd.merge(flight_data, weather_delay_data, on="airport") #drop objects flight_weather_data.drop(columns=['airport', 'origin', 'dest'], inplace=True) #fix type columns_to_float64 = ['HourlyPrecipitation', 'HourlyVisibility', 'HourlyWindGustSpeed', 'HourlyWindSpeed'] for column in columns_to_float64: # Convert to int64 flight_weather_data[column] = flight_weather_data[column].astype('float64') #flight_weather_data.info() flight_weather_data=torch.tensor(flight_weather_data.values, dtype=torch.float32) print(flight_weather_data) #flight_weather_data=scaler.transform(flight_weather_data.reshape(1, -1)) flight_weather_data=scaler.transform(flight_weather_data) print(flight_weather_data) # transform np array to torch tensor flight_weather_data_tensor=torch.tensor(flight_weather_data, dtype=torch.float32) print(flight_weather_data_tensor) output=model(flight_weather_data_tensor) """ return_dict = { 'Origin Airport': origin, 'Destination Airport': destination, 'Scheduled Departure': scheduled_dep_time, 'Scheduled Arrival': scheduled_arr_time, 'Predicted Departure Delay': int(output.item()) } # Convert the dictionary to a Pandas DataFrame df = pd.DataFrame([return_dict]) return df """ global results new_prediction = { 'Origin Airport': origin, 'Destination Airport': destination, 'Scheduled Departure': scheduled_dep_time, 'Scheduled Arrival': scheduled_arr_time, 'Predicted Departure Delay': int(output.item()) } # Append the new prediction to the existing DataFrame results = pd.concat([ pd.DataFrame([new_prediction]),results]) return results #return "Predicted delay for {} to {} with the scheduled departure time {} and scheduled " \ # "arrival time {} is {} minutes".format(origin, destination, scheduled_dep_time, scheduled_arr_time,int(output.item())) # %% # Create Gradio interface with dropdowns for airport selection with gr.Blocks() as demo: gr.Markdown("# Flight departure delay predictor using Flight data and Weather Data") gr.Markdown("Input origin airport and destination airport from the dropdown boxes. Also input the scheduled departure time and scheduled arrival time") gr.Markdown("The scheduled departure time should be within one hour from now since live weather data for the airports will be fetched") with gr.Row(): output = gr.Dataframe(headers=["Origin Airport", "Destination Airport", "Scheduled Departure", "Scheduled Arrival", "Predicted Departure Delay"], row_count=3,col_count=5,type="pandas",label="Predicted Departure Delay") with gr.Column(): origin_dropdown = gr.Dropdown(choices=airports, label="Origin Airport") destination_dropdown = gr.Dropdown(choices=airports, label="Destination Airport") scheduled_dep_time_text = gr.Textbox(type="text", label="Enter scheduled Departure time in 24-hour format HH:MM(eg. 17:59)") scheduled_arr_time_text = gr.Textbox(type="text", label="Enter scheduled Arrival time in 24-hour format HH:MM (eg. 20:59)") with gr.Row(): submit_button = gr.Button("Predict Departure Delay") submit_button.click(predict_delay, inputs=[origin_dropdown, destination_dropdown, scheduled_dep_time_text, scheduled_arr_time_text], outputs=output) demo.launch()