testing

.gitattributes

@@ -1,35 +1,2 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+data/*.csv filter=lfs diff=lfs merge=lfs -text
+data/* filter=lfs diff=lfs merge=lfs -text

DynamicMap.py

@@ -0,0 +1,297 @@
+# Library Importation
+import streamlit as st
+import pandas as pd
+import geopandas as gpd
+import matplotlib.pyplot as plt
+from matplotlib.colors import LinearSegmentedColormap
+import folium
+from folium.plugins import MarkerCluster, Search, MousePosition, MiniMap
+from shapely import wkt
+import branca.colormap as cm
+import streamlit_folium as st_folium
+from streamlit_extras.grid import grid
+
+
+# Load data function
+@st.cache_data
+def load_data():
+    geodata = pd.read_csv("data/stats-area.csv", low_memory=False)
+    data_2018 = pd.read_csv("data/2018_march_ruc.csv", low_memory=False)
+    data_2022to2023 = pd.read_csv("data/2023_march_ruc.csv", low_memory=False)
+    return geodata, data_2018, data_2022to2023
+
+# Data preprocessing function
+def preprocess_data(data_2018, data_2022to2023):
+    light_vehicles = ['1', '12', 'NaN']
+    trailers = ['24', '28', '29', '30', '33', '37', '43', '951', '929', '939']
+    vintage_vehicles = ['402', '403', '404']
+    mobile_cranes = ['299', '399', '499', '599', '699', '799']
+    type_H_vehicles = ['H01', 'H61', 'H62', 'H71', 'H73', 'H74', 'H77', 'H81', 'H82', 'H83', 'H84', 'H75', 'H76', 'H91', 'H92', 'H93', 'H94', 'H95', 'H63',
+                       'H97', 'H98', 'H99', 'H72', 'H78', 'H79', 'H30', 'H31', 'H32', 'H33', 'H34', 'H35', 'H11', 'H12', 'H13', 'H14', 'H15', 'H36', 'H37',
+                       'H38', 'H17', 'H18', 'H19']
+    list_to_filter = light_vehicles + trailers + vintage_vehicles + mobile_cranes + type_H_vehicles
+
+    mask = ~data_2018.ruc_type.isin(list_to_filter)
+    mask2 = ~data_2022to2023.ruc_type.isin(list_to_filter)
+    filtered_2018 = data_2018[mask]
+    filtered_2023 = data_2022to2023[mask2]
+
+    return filtered_2018, filtered_2023
+def create_percentage_change_df():
+    sum_by_sa2_zone_2018 = filtered_2018.groupby(['start_sa2'])['num_trips'].sum().reset_index()
+    sum_by_sa2_zone_2023 = filtered_2023.groupby(['start_sa2'])['num_trips'].sum().reset_index()
+    sum_by_sa2_zone_2018['num_trips'] = sum_by_sa2_zone_2018['num_trips'].fillna(0)
+    sum_by_sa2_zone_2023['num_trips'] = sum_by_sa2_zone_2023['num_trips'].fillna(0)
+    changes_origin_sa2_zone = pd.merge(sum_by_sa2_zone_2018, sum_by_sa2_zone_2023, on = 'start_sa2', how = 'outer')
+    changes_origin_sa2_zone['num_trips_x'] = changes_origin_sa2_zone['num_trips_x'].fillna(0)
+    changes_origin_sa2_zone['num_trips_y'] = changes_origin_sa2_zone['num_trips_y'].fillna(0)
+    changes_origin_sa2_zone['difference'] = abs((changes_origin_sa2_zone['num_trips_y']-changes_origin_sa2_zone['num_trips_x'])/changes_origin_sa2_zone['num_trips_y']) *100
+
+    percentage_change = changes_origin_sa2_zone
+    percentage_change = percentage_change.sort_values('num_trips_y', ascending=False).reset_index(drop=True)
+    percentage_change = pd.merge(percentage_change, geodata, left_on='start_sa2', right_on='SA22018_V1_00', how= 'outer')
+    percentage_change = percentage_change.drop(columns=['start_sa2', 'SA22018_V1_00'])
+    percentage_change = percentage_change[['SA22018_V1_NAME', 'num_trips_x', 'num_trips_y', 'difference']]
+    percentage_change.columns = ['SA2 Zone', 'Number of Trips 2018', 'Number of Trips 2023', 'Absolute Percentage Change (%)']
+    return percentage_change
+
+def create_vehicle_count_df(data):
+    vehicle_counts = data[['SA22018_V1_NAME', 'num_machines']].groupby('SA22018_V1_NAME').sum().reset_index()
+    vehicle_counts = vehicle_counts.sort_values('num_machines', ascending=False).reset_index(drop=True)
+    vehicle_counts.index += 1  # Start index at 1 instead of 0
+    vehicle_counts.columns = ['SA2 Zone', 'Number of Vehicles']
+    return vehicle_counts
+
+# Function to create Folium map
+def create_folium_map(data, column='difference', title=''):
+    # Create a base map centered on New Zealand
+    map = folium.Map(location=[-40.9006, 174.8860], zoom_start=5)  # New Zealand coordinates - 40.9006° S, 174.8860° E
+    
+    # Create a colormap
+    colormap = cm.linear.PuBuGn_04.scale(0, 100)
+    
+    # Add a GeoJson layer
+    GeoJson = folium.GeoJson(
+        data,
+        highlight_function=lambda feature: {
+            "fillColor": ("#87CEFA")
+        },
+        style_function=lambda feature: {
+            'fillColor': colormap(feature['properties'][column]),
+            'color': 'black',
+            'weight': 1,
+            'fillOpacity': 0.7,
+        },
+        tooltip=folium.GeoJsonTooltip(fields=['SA22018_V1_NAME', column, 'num_trips_x', 'num_trips_y'],
+                                      aliases=['Area:', 'Percentage change (%):', 'Number of Trips in 2018', 'Number of Trips in 2023'],
+                                      localize=True,
+                                      sticky=False,
+                                      labels=True)
+    ).add_to(map)
+    
+    # Add a type in search
+    search = Search(
+        layer=GeoJson,
+        geom_type='Polygon',
+        placeholder='Type in the place you like to search',
+        collapsed=False,
+        search_label='SA22018_V1_NAME',
+        weight=3
+    ).add_to(map)
+    
+    # Add colormap to the map
+    colormap.add_to(map)
+    colormap.caption = 'Percentage Change'
+    
+    folium.plugins.Fullscreen(
+    position="topright",
+    title="Expand me",
+    title_cancel="Exit me",
+    force_separate_button=True,
+    ).add_to(map)
+    MiniMap().add_to(map)
+    MousePosition().add_to(map)
+    return map
+
+# Streamlit app
+st.title("National Freight Analysis")
+
+# Load data with a loading indicator
+with st.spinner("Loading data... Please wait."):
+    geodata, data_2018, data_2022to2023 = load_data()
+
+# Preprocess data with a loading indicator
+with st.spinner("Preprocessing data... Please wait."):
+    filtered_2018, filtered_2023 = preprocess_data(data_2018, data_2022to2023)
+
+# Prepare geodata
+geodata['geometry'] = geodata['WKT'].apply(wkt.loads)
+geodata = gpd.GeoDataFrame(geodata, geometry='geometry', crs="EPSG:4326")
+#geodata = geodata.drop(columns=['WKT', 'LAND_AREA_SQ_KM', 'AREA_SQ_KM', 'Shape_Length'])
+
+
+
+# Sidebar for navigation
+page = st.sidebar.selectbox("Choose a zone", ["Changes","Origin", "Destination", "Sanity Checks"])
+
+if page == "Changes":
+    st.header("Change in Freight Pre-covid and Post-covid")
+    sum_by_sa2_zone_2018 = filtered_2018.groupby(['start_sa2'])['num_trips'].sum().reset_index()
+    sum_by_sa2_zone_2023 = filtered_2023.groupby(['start_sa2'])['num_trips'].sum().reset_index()
+    sum_by_sa2_zone_2018['num_trips'] = sum_by_sa2_zone_2018['num_trips'].fillna(0)
+    sum_by_sa2_zone_2023['num_trips'] = sum_by_sa2_zone_2023['num_trips'].fillna(0)
+    changes_origin_sa2_zone = pd.merge(sum_by_sa2_zone_2018, sum_by_sa2_zone_2023, on = 'start_sa2', how = 'outer')
+    changes_origin_sa2_zone['num_trips_x'] = changes_origin_sa2_zone['num_trips_x'].fillna(0)
+    changes_origin_sa2_zone['num_trips_y'] = changes_origin_sa2_zone['num_trips_y'].fillna(0)
+    changes_origin_sa2_zone['difference'] = abs((changes_origin_sa2_zone['num_trips_y']-changes_origin_sa2_zone['num_trips_x'])/changes_origin_sa2_zone['num_trips_y']) *100
+    changes_origin_sa2_zone = pd.merge(changes_origin_sa2_zone, geodata, left_on='start_sa2', right_on='SA22018_V1_00', how= 'outer')
+    changes_origin_sa2_zone = changes_origin_sa2_zone.drop(columns='start_sa2')
+    changes_origin_sa2_zone= changes_origin_sa2_zone.fillna(0)
+    changes_origin_sa2_zone = gpd.GeoDataFrame(changes_origin_sa2_zone, geometry='geometry', crs="EPSG:4326")
+    changes_origin_sa2_zone_map = create_folium_map(changes_origin_sa2_zone, title='Number of Vehicles in March 2023 by Origin')
+    st_folium.folium_static(changes_origin_sa2_zone_map, width=1100, height=600)
+    change = create_percentage_change_df()
+
+    st.subheader("Percentage Change in 2018 and 2023 (Sort by trips 2023)")
+
+    st.dataframe(change, width= 1000, height=400)
+
+'''
+elif page == "Origin":
+    st.header("Origin Zone")
+
+    # Origin data
+    sum_by_sa2_zone_2018 = filtered_2018.groupby(['start_sa2'])['num_machines'].sum().reset_index()
+    sum_by_sa2_zone_2023 = filtered_2023.groupby(['start_sa2'])['num_machines'].sum().reset_index()
+
+    sa2_zone_2018_start = pd.merge(sum_by_sa2_zone_2018, geodata, left_on='start_sa2', right_on='SA22018_V1_00', how= 'outer')
+    sa2_zone_2023_start = pd.merge(sum_by_sa2_zone_2023, geodata, left_on='start_sa2', right_on='SA22018_V1_00', how='outer')
+    sa2_zone_2018_start['num_machines'] = sa2_zone_2018_start['num_machines'].fillna(0)
+    sa2_zone_2023_start['num_machines'] = sa2_zone_2023_start['num_machines'].fillna(0)
+
+    # Convert to GeoDataFrame
+    sa2_zone_2018_start = gpd.GeoDataFrame(sa2_zone_2018_start, geometry='geometry', crs="EPSG:4326")
+    sa2_zone_2023_start = gpd.GeoDataFrame(sa2_zone_2023_start, geometry='geometry', crs="EPSG:4326")
+
+    # Display maps and tables side by side
+    st.subheader("Number of Vehicles by Origin: March 2018 vs March 2023")
+    
+    grid_layout_origin = grid(3, vertical_align="start")
+    with st.spinner("Loading data... Please wait."):
+        with grid_layout_origin.container():
+            map_2018 = create_folium_map(sa2_zone_2018_start, title='Number of Vehicles in March 2018 by Origin')
+            st_folium.folium_static(map_2018, width=450, height=400)
+            
+            st.markdown("#### Vehicle Counts by Region (2018)")
+            counts_2018 = create_vehicle_count_df(sa2_zone_2018_start)
+            st.dataframe(counts_2018, width= 400, height=400)
+    with st.spinner("Loading data... Please Wait"):
+        with grid_layout_origin.container():
+            st.empty()
+        with grid_layout_origin.container():
+            map_2023 = create_folium_map(sa2_zone_2023_start, title='Number of Vehicles in March 2023 by Origin')
+            st_folium.folium_static(map_2023, width=400, height=400)
+            
+            st.markdown("#### Vehicle Counts by Region (2023)")
+            counts_2023 = create_vehicle_count_df(sa2_zone_2023_start)
+            st.dataframe(counts_2023, width=400,height=400, use_container_width=True)
+    
+
+elif page == "Destination":
+    st.header("Destination Zone")
+
+    # Destination data
+    sum_by_sa2_zone_2018_dest = filtered_2018.groupby(['end_sa2'])['num_machines'].sum().reset_index()
+    sum_by_sa2_zone_2023_dest = filtered_2023.groupby(['end_sa2'])['num_machines'].sum().reset_index()
+
+    sa2_zone_2018_dest = pd.merge(sum_by_sa2_zone_2018_dest, geodata, left_on='end_sa2', right_on='SA22018_V1_00', how='outer')
+    sa2_zone_2023_dest = pd.merge(sum_by_sa2_zone_2023_dest, geodata, left_on='end_sa2', right_on='SA22018_V1_00', how='outer')
+    sa2_zone_2018_dest['num_machines'] = sa2_zone_2018_dest['num_machines'].fillna(0)
+    sa2_zone_2023_dest['num_machines'] = sa2_zone_2023_dest['num_machines'].fillna(0)
+
+    # Convert to GeoDataFrame
+    sa2_zone_2018_dest = gpd.GeoDataFrame(sa2_zone_2018_dest, geometry='geometry', crs="EPSG:4326")
+    sa2_zone_2023_dest = gpd.GeoDataFrame(sa2_zone_2023_dest, geometry='geometry', crs="EPSG:4326")
+
+    # Display maps and tables side by side
+    st.subheader("Number of Vehicles by Destination: March 2018 vs March 2023")
+    
+    grid_layout_destination = grid(3, vertical_align="start")
+
+    with st.spinner("Loading data... Please wait."):
+        with grid_layout_destination.container():
+            map_2018_dest = create_folium_map(sa2_zone_2018_dest, title='Number of Vehicles in March 2018 by Origin')
+            st_folium.folium_static(map_2018_dest, width=400, height=400)
+            
+            st.markdown("#### Vehicle Counts by Region (2018)")
+            counts_2018 = create_vehicle_count_df(sa2_zone_2018_dest)
+            st.dataframe(counts_2018, width= 400, height=400)
+    with st.spinner("Loading data... Please Wait"):
+        with grid_layout_destination.container():
+            st.empty()
+        with grid_layout_destination.container():
+            map_2023_dest = create_folium_map(sa2_zone_2023_dest, title='Number of Vehicles in March 2023 by Origin')
+            st_folium.folium_static(map_2023_dest= create_folium_map(sa2_zone_2023_dest, title='Number of Vehicles in March 2023 by Origin')
+, width=400, height=400)
+            
+            st.markdown("#### Vehicle Counts by Region (2023)")
+            counts_2023 = create_vehicle_count_df(sa2_zone_2023_dest)
+            st.dataframe(counts_2023, width=400,height=400, use_container_width=True)
+'''
+if page == "Sanity Checks":
+    # Number of trips March 2018 vs March 2023
+    st.subheader("Number of Trips Comparison")
+    sum_origin_trips_2018 = filtered_2018.num_trips.sum()
+    sum_origin_trips_2023 = filtered_2023.num_trips.sum()
+    trips = {'March 2018': sum_origin_trips_2018, 'March 2023': sum_origin_trips_2023}
+    st.bar_chart(trips, color= '#29AB87')
+
+    # Number of all Machines March 2018 vs March 2023
+    st.subheader("Number of Vehicles Comparison")
+    sum_origin_machines_2018 = filtered_2018.num_machines.sum()
+    sum_origin_machines_2023 = filtered_2023.num_machines.sum()
+    machines = {"March 2018": sum_origin_machines_2018, "March 2023": sum_origin_machines_2023}
+    st.bar_chart(machines, color= '#00A693')
+def calculate_percentage_difference(value_2018, value_2023):
+    return ((value_2023 - value_2018) / value_2018) * 100
+
+if page == "Sanity Checks":
+    # Number of trips March 2018 vs March 2023
+    st.subheader("Percentage Change in Number of Trips")
+    sum_origin_trips_2018 = filtered_2018.num_trips.sum()
+    sum_origin_trips_2023 = filtered_2023.num_trips.sum()
+    trips_percentage_diff = calculate_percentage_difference(sum_origin_trips_2018, sum_origin_trips_2023)
+
+    # Number of all Machines March 2018 vs March 2023
+    st.subheader("Percentage Change in Number of Vehicles")
+    sum_origin_machines_2018 = filtered_2018.num_machines.sum()
+    sum_origin_machines_2023 = filtered_2023.num_machines.sum()
+    machines_percentage_diff = calculate_percentage_difference(sum_origin_machines_2018, sum_origin_machines_2023)
+
+    # Create a DataFrame for the percentage differences
+    data = pd.DataFrame({
+        'Category': ['Trips', 'Vehicles'],
+        'Percentage Change': [trips_percentage_diff, machines_percentage_diff]
+    })
+
+    # Create the bar chart using Plotly
+    fig = go.Figure(go.Bar(
+        x=data['Category'],
+        y=data['Percentage Change'],
+        text=data['Percentage Change'].apply(lambda x: f'{x:.2f}%'),
+        textposition='outside',
+        marker_color=['#29AB87', '#00A693']
+    ))
+
+    fig.update_layout(
+        title='Percentage Change from March 2018 to March 2023',
+        yaxis_title='Percentage Change',
+        yaxis_tickformat=',.2f%',
+        yaxis_ticksuffix='%'
+    )
+
+    # Display the chart in Streamlit
+    st.plotly_chart(fig)
+    # TO-DO Sum of all Trips March 2018 compared to March 2023
+
+

data/2018_march_ruc.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88227cf28102e5d54fd37df502109f093a6153f0d0c183de36691c8506844a28
+size 29789168

data/2023_march_ruc.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67fa04f796ebdf801940cd25e99b933df717db6effea9b37bcf06563eff62c49
+size 38923909

data/stats-area.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d09ff8de9215e60d341a600e6618971b8ed6da634d2a2452ce1817e93d934cb
+size 88767394

requirements.txt

@@ -0,0 +1,9 @@
+streamlit
+pandas
+geopandas
+matplotlib
+folium
+shapely
+branca
+streamlit-folium
+streamlit-extras