Test av borttagning av dubbletter, tillägg av hållplatser samt start på sortering

app.py

@@ -74,50 +74,70 @@ def get_buses():
     short_bus_list = list(pd.unique(bus_df["route_short_name"]))
     return bus_df, bus_list, short_bus_list
 
-
-def remove_duplicate_trips(df, route_id_col="route_id", trip_id_col = "trip_id", stop_id_col = "stop_name", datetime_col = "datetime", time_window='3min'):
+# Function to remove duplicates
+def remove_near_duplicates(data):
+    print(data["trip_id"].nunique())
+    result = []
+    data["datetime"] = pd.to_datetime(data["datetime"])
+    for _, group in data.groupby(['route_id', 'stop_name']):
+        # Initialize a list to store rows that are not duplicates
+        filtered_rows = []
+        last_row = None
+
+        for idx, row in group.iterrows():
+            if last_row is None or (row['datetime'] - last_row['datetime'] > pd.Timedelta(minutes = 3)):
+                # Keep the row if it's the first or sufficiently far apart in time
+                filtered_rows.append(row)
+                last_row = row
+
+        # Add filtered rows to the result
+        result.extend(filtered_rows)
+    filtered_df = pd.DataFrame(result)
+    # Return the filtered dataframe
+    print(filtered_df["trip_id"].nunique())
+    return filtered_df
+
+def remove_duplicate_trips(df, route_id_col="route_id", trip_id_col="trip_id", datetime_col="datetime", time_window='3min'):
     """
-    Removes duplicate trips based on route_id, start stop_id, and starting time proximity within a time window.
+    Removes duplicate trips based on route_id and starting time proximity within a time window.
     
     Parameters:
         df (pd.DataFrame): Input DataFrame containing trip data.
         route_id_col (str): Column name for route IDs.
         trip_id_col (str): Column name for trip IDs.
-        stop_id_col (str): Column name for stop IDs.
         datetime_col (str): Column name for departure times.
         time_window (str): Time window for considering trips as duplicates (e.g., '3min').
     
     Returns:
         pd.DataFrame: Filtered DataFrame with duplicates removed.
     """
+    print(df["trip_id"].nunique())
+
     # Ensure the datetime column is of datetime type
     df[datetime_col] = pd.to_datetime(df[datetime_col])
-    
-    # Sort by route_id, stop_id, and datetime for proper grouping and filtering
-    df = df.sort_values(by=[route_id_col, stop_id_col, datetime_col])
-    
-    # Find the first stop for each trip
-    first_stops = df.groupby(trip_id_col).first().reset_index()
-    
-    # Identify duplicate trips based on route_id, stop_id, and datetime proximity
-    def filter_duplicates(group):
-        # Compare trips starting within the time window
-        group['keep'] = ~((group[stop_id_col] == group[stop_id_col].shift()) &
-                          (group[datetime_col] - group[datetime_col].shift() <= pd.Timedelta(time_window)))
-        group['keep'] = group['keep'].cumsum() == 1  # Keep only the first trip in each duplicate group
-        return group[group['keep']]
-    
-    # Apply filtering for each route_id group
-    filtered_first_stops = first_stops.groupby(route_id_col, group_keys=False).apply(filter_duplicates)
-    
-    # Filter the original DataFrame to retain only the non-duplicate trips
-    unique_trip_ids = filtered_first_stops[trip_id_col].unique()
-    result = df[df[trip_id_col].isin(unique_trip_ids)]
-    
+
+    # Sort by route_id and datetime for correct chronological order within each route
+    df = df.sort_values(by=[route_id_col, datetime_col])
+
+    # Calculate time differences between consecutive rows within each route_id group
+    df['time_diff'] = df.groupby(route_id_col)[datetime_col].diff().fillna(pd.Timedelta('0s'))
+
+    # Mark rows as duplicates if the time difference is within the time window
+    time_window_timedelta = pd.to_timedelta(time_window)
+    df['is_duplicate'] = df['time_diff'] <= time_window_timedelta
+
+    # Keep only the first row within each group of duplicates (based on time window)
+    df['keep'] = ~df.groupby(route_id_col)['is_duplicate'].transform('any')
+
+    # Filter rows: Keep only those that are marked as 'keep'
+    result = df[df['keep']].drop(columns=['time_diff', 'is_duplicate', 'keep'])
+
+    print(result["trip_id"].nunique())
     return result
 
 def plot_graph(plot_df):
     #Nu vill vi plotta!
+    #TODO ska den bara visa de stopp man vill eller alla?
     categories =  {0 : 'Empty',
     1: 'Many seats available',
     2:'Few seats available',
@@ -125,7 +145,7 @@ def plot_graph(plot_df):
     4:'Crushed standing room',
     5: 'Full'}
     
-    plot_df = plot_df[["datetime", "vehicle_occupancystatus", "stop_name"]]
+    plot_df = plot_df[["datetime", "vehicle_occupancystatus", "stop_name", "route_id"]]
     plot_df = plot_df.sort_values("datetime")
     st.write(plot_df.head())
     st.write(plot_df.tail())
@@ -146,6 +166,36 @@ def plot_graph(plot_df):
     )
     st.altair_chart(chart, use_container_width=True)
 
+def plot_graph_title(plot_df, stop, time):
+    #Nu vill vi plotta!
+    #TODO ska den bara visa de stopp man vill eller alla?
+    categories =  {0 : 'Empty',
+    1: 'Many seats available',
+    2:'Few seats available',
+    3:'Standing room only',
+    4:'Crushed standing room',
+    5: 'Full'}
+    
+    plot_df = plot_df[["datetime", "vehicle_occupancystatus", "stop_name", "route_id"]]
+    plot_df = plot_df.sort_values("datetime")
+    #plot_df = plot_df.set_index("datetime")
+    plot_df["Occupancy"] = plot_df["vehicle_occupancystatus"].map(categories)
+    # Explicitly set the order for Y_category
+    category_order = list(categories.values())  # ['Empty', 'Many seats available', ..., 'Full']
+    category_order.reverse()
+
+    #st.line_chart(plot_df)
+    # Create the Altair chart
+    chart = alt.Chart(plot_df).mark_line(point=True, interpolate="step-after").encode(
+        x=alt.X('stop_name:N', title="Stop name"),  # Use column name as string
+        y=alt.Y('Occupancy:N', title="Vehicle Occupancy Status (Categories)", sort=category_order, scale=alt.Scale(domain=category_order)),  # Treat Y as categorical
+        tooltip=["datetime", 'stop_name', 'Occupancy']  # Add tooltips for interactivity
+    ).properties(
+        title=f"Vehicle Occupancy For Bus arriving at {stop} at {time}"
+    )
+    st.altair_chart(chart, use_container_width=True)
+
+
 def visualize(filtered_data):
     import folium
     from streamlit_folium import st_folium
@@ -198,7 +248,7 @@ def main():
         st.session_state.data = load_local_data("data.csv")
         if "first" not in st.session_state:
             st.session_state.first = True
-            st.session_state.data = remove_duplicate_trips(st.session_state.data)
+            #st.session_state.data = remove_near_duplicates(st.session_state.data)
     else:
         # Fetch data if local data is invalid
         if st.session_state.hopsworks_project is None:
@@ -252,7 +302,18 @@ def main():
             st.session_state.direction = not st.session_state.direction
             print(st.session_state.direction)
 
-            
+        #Plocka alla aktuella trip_ids från buses
+        trips = buses_df[buses_df["route_long_name"]==bus]
+        bus_trips = st.session_state.data[st.session_state.data["route_long_name"]==bus]
+        bus_trips["datetime"] = pd.to_datetime(bus_trips["datetime"])
+        bus_trips["datetime"] = bus_trips["datetime"].dt.tz_convert(None)
+
+        stops = list(pd.unique(bus_trips["stop_name"]))
+        stop_choice = st.sidebar.selectbox(
+            "Select your bus stop:",
+            options=stops,
+            help="Select one bus stop to se occupancy."
+            )  
         #direction = st.sidebar.checkbox('Direction of bus', value=True) 
 
         today = datetime.now()
@@ -271,17 +332,16 @@ def main():
         start_time = st.sidebar.time_input("Select a start time", value=None)
         end_time = st.sidebar.time_input("Select an end time", value=None)
 
-        #Plocka alla aktuella trip_ids från buses
-        trips = buses_df[buses_df["route_long_name"]==bus]
-        bus_trips = st.session_state.data[st.session_state.data["route_long_name"]==bus]
-        bus_trips["datetime"] = pd.to_datetime(bus_trips["datetime"])
-        bus_trips["datetime"] = bus_trips["datetime"].dt.tz_convert(None)
+        
         
         #TODO remove
-        trip_ids = list(trips["trip_id"])
-        plot_df = st.session_state.data[st.session_state.data["trip_id"]==trip_ids[0]]
+        #trip_ids = list(trips["trip_id"])
+        #plot_df = st.session_state.data[st.session_state.data["trip_id"]==trip_ids[0]]
+
+        #TODO hållsplats
+        #Kolla på route_id för att plocka alla hållplatser
+
 
-        #TODO direction
 
         print(f"start time {type(start_time)}")
         print(f"end time {type(end_time)}")
@@ -290,10 +350,27 @@ def main():
         if start_time != None and end_time != None:
             #TODO hur filtrera på tid?
             st.write(f"Displaying buses between {start_time.strftime('%H:%M')} and {end_time.strftime('%H:%M')} the {day_choice}")
+            """selected_trips = bus_trips[(bus_trips["datetime"] >= datetime.combine(date_options[day_choice], start_time)) 
+                                       & (bus_trips["datetime"] <= datetime.combine(date_options[day_choice], end_time))
+                                       & (bus_trips["direction_id"] == st.session_state.direction )]"""
             selected_trips = bus_trips[(bus_trips["datetime"] >= datetime.combine(date_options[day_choice], start_time)) 
                                        & (bus_trips["datetime"] <= datetime.combine(date_options[day_choice], end_time))
-                                       & (bus_trips["direction_id"] == st.session_state.direction )]
+                                       & (bus_trips["direction_id"] == st.session_state.direction )
+                                       & (bus_trips["stop_name"] == stop_choice)]
             trip_ids = list(pd.unique(selected_trips["trip_id"]))
+
+            chioce = selected_trips[selected_trips["stop_name"]==stop_choice]
+            chioce.head()
+            chioce = chioce[["trip_id", "stop_name", "datetime"]]
+            #Ev lägga stop_chioce i session_state
+
+            chioce = chioce.sort_values(by=["datetime"])
+            chioce = chioce.drop_duplicates("datetime")
+
+            for idx, row in chioce.iterrows():
+                st.write(f"The bus arrives at {row['stop_name']} at {row['datetime'].strftime('%H:%M')}")
+                plot_graph_title(st.session_state.data[st.session_state.data["trip_id"]==row["trip_id"]], row["stop_name"], row['datetime'].strftime('%H:%M'))
+
             st.write(f"Length {len(trip_ids)}")
             for id in trip_ids:
                 plot_graph(st.session_state.data[st.session_state.data["trip_id"]==id])

test.py

@@ -0,0 +1,44 @@
+import os
+import pandas as pd
+
+# Load local data
+def load_local_data():
+    
+    #filepath = os.path.join(current_dir, "test_data.csv")
+    filepath = "WheelyFunTimes/test_data.csv"
+    return pd.read_csv(filepath)
+    """if os.path.exists(filepath):
+        return pd.read_csv(filepath)
+    else:
+        return None"""
+    
+def remove_near_duplicates(data):
+    print(data["trip_id"].nunique())
+    result = []
+    data["datetime"] = pd.to_datetime(data["datetime"])
+    for _, group in data.groupby(['route_id', 'stop_name']):
+        # Initialize a list to store rows that are not duplicates
+        filtered_rows = []
+        last_row = None
+
+        for idx, row in group.iterrows():
+            if last_row is None or (row['datetime'] - last_row['datetime'] > pd.Timedelta(minutes = 3)):
+                # Keep the row if it's the first or sufficiently far apart in time
+                filtered_rows.append(row)
+                last_row = row
+
+        # Add filtered rows to the result
+        result.extend(filtered_rows)
+    filtered_df = pd.DataFrame(result)
+    # Return the filtered dataframe
+    print(filtered_df["trip_id"].nunique())
+    return filtered_df
+    
+df = load_local_data()
+print(df.head(12))
+df = remove_near_duplicates(df)
+print(df.head(12))
+
+
+
+    

test_data.csv

@@ -0,0 +1,17 @@
+direction_id,route_short_name,route_long_name,route_id,datetime,stop_lat,stop_lon,trip_id,temperature_2m,precipitation,wind_speed_10m,hourly_cloud_cover,dag_i_vecka,arbetsfri_dag,holiday,helgdag,squeeze_day,helgdagsafton,day_before_holiday,hour,minute,vehicle_occupancystatus,stop_name
+FALSE,NO4,Kristianstad - Simrishamn,1,2025-01-05 13:04:37+00:01,9986.36.00,167.37.00,2,1.6,0.0,9935.39.00,87.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,13,15,1,Malmö Spångatan
+TRUE,8,Lindängen - Västra hamnen,2,2025-01-05 15:50:14+00:00,9586.03.00,410.28.00,3,01.25,0.0,1623.03.00,83.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,15,50,0,Malmö Nydala
+FALSE,3,Lindängen - Västra hamnen,2,2025-01-05 07:45:42+00:00,363.43.00,3756.51.00,3,1.9,0.0,4126.46.00,100.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,7,45,0,Viby Kvarnnäsvägen
+FALSE,4,Lindängen - Västra hamnen,2,2025-01-05 14:27:33+00:00,11973.42.00,3358.39.00,3,01.35,0.0,10456.25.00,84.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,14,27,0,Lund Smörlyckan
+FALSE,502,Lindängen - Västra hamnen,2,2025-01-05 14:45:15+00:00,7171.11.00,14479.54.00,3,01.35,0.0,10456.25.00,84.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,14,45,0,Båstad Kalkvägen
+TRUE,511,Lund C - ESS (spårvagn),3,2025-01-05 12:44:36+00:00,12025.35.00,3912.03.00,5,01.25,0.0,1623.03.00,83.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,15,48,0,Lund Solbjer
+TRUE,150,Lund C - ESS (spårvagn),3,2025-01-05 10:08:00+00:00,10014.42.00,4345.02.00,4,1.8,0.0,16011.59.00,97.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,10,8,1,Klågerup busstation
+FALSE,5,Lund C - ESS (spårvagn),3,2025-01-05 10:09:00+00:00,10015.42.00,4346.02.00,6,01.25,0.0,1623.03.00,83.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,15,55,0,Klågerup busstation
+TRUE,511,Lund C - ESS (spårvagn),3,2025-01-05 12:44:36+00:00,12025.35.00,3912.03.00,7,01.25,0.0,1623.03.00,83.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,15,48,0,Lund Solbjer
+TRUE,150,Lund C - ESS (spårvagn),3,2025-01-05 10:08:00+00:00,10014.42.00,4345.02.00,7,1.8,0.0,16011.59.00,97.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,10,8,1,Klågerup busstation
+TRUE,1,Lund C - ESS (spårvagn),3,2025-01-05 12:43:36+00:00,12024.35.00,3911.03.00,6,02.15,0.0,11580.39.00,90.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,12,43,0,Lund Solbjer
+FALSE,817,Kristianstad - Simrishamn,1,2025-01-05 21:49:00+00:01,14016.02.00,1519.11.00,2,01.35,0.0,10456.25.00,84.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,14,20,1,Degeberga Skaddevägen
+TRUE,174,Kristianstad - Simrishamn,1,2025-01-05 13:04:37+00:00,9985.36.00,166.37.00,1,1.6,0.0,9935.39.00,87.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,13,4,0,Malmö Spångatan
+TRUE,1,Lund C - ESS (spårvagn),3,2025-01-05 12:43:36+00:00,12024.35.00,3911.03.00,4,02.15,0.0,11580.39.00,90.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,12,43,0,Lund Solbjer
+FALSE,3,Kristianstad - Simrishamn,1,2025-01-05 21:49:00+00:00,14015.02.00,1518.11.00,1,1.4,0.2,1678.34.00,100.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,21,49,0,Degeberga Skaddevägen
+FALSE,5,Lund C - ESS (spårvagn),3,2025-01-05 10:09:00+00:00,10015.42.00,4346.02.00,5,01.25,0.0,1623.03.00,83.0,7,TRUE,TRUE,FALSE,FALSE,TRUE,FALSE,15,55,0,Klågerup busstation