fix: add moving avg and top 5 agents to toggle

app.py

@@ -534,7 +534,7 @@ def write_debug_info(df, fig):
         return False
 
 def create_combined_time_series_graph(df):
-    """Create a combined time series graph for all agents using Plotly"""
+    """Create a time series graph showing average APR values across all agents"""
     if len(df) == 0:
         logger.error("No data to plot combined graph")
         fig = go.Figure()
@@ -593,10 +593,6 @@ def create_combined_time_series_graph(df):
     # Create Plotly figure in a clean state
     fig = go.Figure()
     
-    # Get unique agents
-    unique_agents = df['agent_id'].unique()
-    colors = px.colors.qualitative.Plotly[:len(unique_agents)]
-    
     # Update y-axis range to include negative values
     min_apr = min(df['apr'].min() * 1.1, -10)  # Add 10% padding, minimum of -10
     max_apr = max(df['apr'].max() * 1.1, 10)  # Add 10% padding, minimum of 10
@@ -633,61 +629,141 @@ def create_combined_time_series_graph(df):
         x0=min_time, x1=max_time
     )
     
-    # MODIFIED: Changed order of trace addition - only need APR values now
-    # Add data for each agent
-    for i, agent_id in enumerate(unique_agents):
-        agent_data = df[df['agent_id'] == agent_id].copy()
-        agent_name = agent_data['agent_name'].iloc[0]
-        color = colors[i % len(colors)]
+    # MODIFIED: Calculate average APR values across all agents for each timestamp
+    # Filter for APR data only
+    apr_data = df[df['metric_type'] == 'APR'].copy()
+    
+    # Group by timestamp and calculate mean APR
+    avg_apr_data = apr_data.groupby('timestamp')['apr'].mean().reset_index()
+    
+    # Sort by timestamp
+    avg_apr_data = avg_apr_data.sort_values('timestamp')
+    
+    # Log the average APR data
+    logger.info(f"Calculated average APR data with {len(avg_apr_data)} points")
+    for idx, row in avg_apr_data.iterrows():
+        logger.info(f"  Average point {idx}: timestamp={row['timestamp']}, avg_apr={row['apr']}")
+    
+    # Calculate moving average based on a time window (2 hours)
+    # Sort data by timestamp
+    apr_data_sorted = apr_data.sort_values('timestamp')
+    
+    # Create a new dataframe for the moving average
+    avg_apr_data_with_ma = avg_apr_data.copy()
+    avg_apr_data_with_ma['moving_avg'] = None  # Initialize the moving average column
+    
+    # Define the time window for the moving average (2 hours)
+    time_window = pd.Timedelta(hours=2)
+    logger.info(f"Calculating moving average with time window of {time_window}")
+    
+    # Calculate the moving average for each timestamp
+    for i, row in avg_apr_data_with_ma.iterrows():
+        current_time = row['timestamp']
+        window_start = current_time - time_window
         
-        # Sort the data by timestamp
-        agent_data = agent_data.sort_values('timestamp')
+        # Get all data points within the time window
+        window_data = apr_data_sorted[
+            (apr_data_sorted['timestamp'] >= window_start) & 
+            (apr_data_sorted['timestamp'] <= current_time)
+        ]
         
-        # Log actual points being plotted for this agent
-        logger.info(f"Plotting agent: {agent_name} (ID: {agent_id}) with {len(agent_data)} points")
-        for idx, row in agent_data.iterrows():
-            logger.info(f"  Point {idx}: timestamp={row['timestamp']}, apr={row['apr']}, type={row['metric_type']}")
+        # Calculate the average APR for the time window
+        if not window_data.empty:
+            avg_apr_data_with_ma.at[i, 'moving_avg'] = window_data['apr'].mean()
+            logger.debug(f"Time window {window_start} to {current_time}: {len(window_data)} points, avg={window_data['apr'].mean()}")
+        else:
+            # If no data points in the window, use the current value
+            avg_apr_data_with_ma.at[i, 'moving_avg'] = row['apr']
+            logger.debug(f"No data points in time window for {current_time}, using current value {row['apr']}")
+    
+    logger.info(f"Calculated time-based moving average with {len(avg_apr_data_with_ma)} points")
+    
+    # Plot individual agent data points with agent names in hover, but limit display for scalability
+    if not apr_data.empty:
+        # Group by agent to use different colors for each agent
+        unique_agents = apr_data['agent_name'].unique()
+        colors = px.colors.qualitative.Plotly[:len(unique_agents)]
         
-        # Get the APR data - this is what we'll plot
-        apr_data = agent_data[agent_data['metric_type'] == 'APR']
+        # Create a color map for agents
+        color_map = {agent: colors[i % len(colors)] for i, agent in enumerate(unique_agents)}
         
-        # SIMPLIFIED APPROACH: Use a single trace with lines+markers mode
-        # This is much more reliable across different platforms
-        if not apr_data.empty:
-            logger.info(f"  Adding combined line+markers for {agent_name}")
+        # Calculate the total number of data points per agent to determine which are most active
+        agent_counts = apr_data['agent_name'].value_counts()
+        
+        # Determine how many agents to show individually (limit to top 5 most active)
+        MAX_VISIBLE_AGENTS = 5
+        top_agents = agent_counts.nlargest(min(MAX_VISIBLE_AGENTS, len(agent_counts))).index.tolist()
+        
+        logger.info(f"Showing {len(top_agents)} agents by default out of {len(unique_agents)} total agents")
+        
+        # Add data points for each agent, but only make top agents visible by default
+        for agent_name in unique_agents:
+            agent_data = apr_data[apr_data['agent_name'] == agent_name]
             
             # Explicitly convert to Python lists
-            x_values = apr_data['timestamp'].tolist()
-            y_values = apr_data['apr'].tolist()
+            x_values = agent_data['timestamp'].tolist()
+            y_values = agent_data['apr'].tolist()
             
-            # Log what we're about to plot
-            for i, (x, y) in enumerate(zip(x_values, y_values)):
-                logger.info(f"    Point {i+1}: x={x}, y={y}")
+            # Determine if this agent should be visible by default
+            is_visible = agent_name in top_agents
             
-            # Use a single trace for both markers and lines
+            # Add data points as markers
             fig.add_trace(
                 go.Scatter(
                     x=x_values,
                     y=y_values,
-                    mode='lines+markers',  # Important: use both lines and markers
+                    mode='markers',  # Only markers for original data
                     marker=dict(
-                        color='blue',
+                        color=color_map[agent_name],
                         symbol='circle',
-                        size=12,
-                        line=dict(width=2, color='black')
+                        size=10,
+                        line=dict(width=1, color='black')
                     ),
-                    line=dict(color='blue', width=2),
-                    name=agent_name,
-                    legendgroup=agent_name,
-                    showlegend=True,
-                    hovertemplate='Time: %{x}<br>APR: %{y:.2f}<br>Agent: ' + agent_name + '<extra></extra>'
+                    name=f'Agent: {agent_name}',
+                    hovertemplate='Time: %{x}<br>APR: %{y:.2f}<br>Agent: ' + agent_name + '<extra></extra>',
+                    visible=is_visible  # Only top agents visible by default
                 )
             )
-            logger.info(f"  Added combined line+markers trace for {agent_name}")
+            logger.info(f"Added data points for agent {agent_name} with {len(x_values)} points (visible: {is_visible})")
+        
+        # Add moving average as a smooth line
+        x_values_ma = avg_apr_data_with_ma['timestamp'].tolist()
+        y_values_ma = avg_apr_data_with_ma['moving_avg'].tolist()
+        
+        # Create a more detailed hover template for the moving average line
+        # that includes information about all agents at each timestamp and the time window
+        hover_data = []
+        for idx, row in avg_apr_data_with_ma.iterrows():
+            timestamp = row['timestamp']
+            window_start = timestamp - time_window
+            
+            # Find all agents with data in the time window
+            agents_in_window = apr_data[
+                (apr_data['timestamp'] >= window_start) & 
+                (apr_data['timestamp'] <= timestamp)
+            ]
+            
+            # Simplified hover text without detailed data points
+            hover_data.append(
+                f"Time: {timestamp}<br>Moving Avg APR (2h window): {row['moving_avg']:.2f}"
+            )
+        
+        fig.add_trace(
+            go.Scatter(
+                x=x_values_ma,
+                y=y_values_ma,
+                mode='lines',  # Only lines for moving average
+                line=dict(color='red', width=3),
+                name='Moving Average APR (2h window)',
+                hovertext=hover_data,
+                hoverinfo='text'
+            )
+        )
+        logger.info(f"Added time-based moving average APR trace with {len(x_values_ma)} points")
     
     # Update layout - use simple boolean values everywhere
     fig.update_layout(
-        title="APR Values for All Agents",
+        title="Average APR Values Across All Agents",
         xaxis_title="Time",
         yaxis_title="Value",
         template="plotly_white",
@@ -705,6 +781,17 @@ def create_combined_time_series_graph(df):
         hovermode="closest"
     )
     
+    # Add a note about hidden agents if there are more than MAX_VISIBLE_AGENTS
+    if len(unique_agents) > MAX_VISIBLE_AGENTS:
+        fig.add_annotation(
+            text=f"Note: Only showing top {MAX_VISIBLE_AGENTS} agents by default. Toggle others in legend.",
+            xref="paper", yref="paper",
+            x=0.5, y=1.05,
+            showarrow=False,
+            font=dict(size=12, color="gray"),
+            align="center"
+        )
+    
     # FORCE FIXED Y-AXIS RANGE
     fig.update_yaxes(
         showgrid=True, 
@@ -760,30 +847,87 @@ def create_combined_time_series_graph(df):
             x0=min_time, x1=max_time
         )
         
-        # Simply plot each agent's data as a line with markers
-        for i, agent_id in enumerate(unique_agents):
-            agent_data = df[df['agent_id'] == agent_id].copy()
-            agent_name = agent_data['agent_name'].iloc[0]
-            color = colors[i % len(colors)]
-            
+        # Define colors for the fallback graph
+        fallback_colors = px.colors.qualitative.Plotly
+        
+        # Simply plot the average APR data with moving average
+        if not avg_apr_data.empty:
             # Sort by timestamp
-            agent_data = agent_data.sort_values('timestamp')
+            avg_apr_data = avg_apr_data.sort_values('timestamp')
+            
+            # Calculate time-based moving average for the fallback graph
+            avg_apr_data_with_ma = avg_apr_data.copy()
+            avg_apr_data_with_ma['moving_avg'] = None
+            
+            # Define the time window (2 hours)
+            time_window = pd.Timedelta(hours=2)
+            
+            # Calculate the moving average for each timestamp
+            for i, row in avg_apr_data_with_ma.iterrows():
+                current_time = row['timestamp']
+                window_start = current_time - time_window
+                
+                # Get all data points within the time window
+                window_data = apr_data[
+                    (apr_data['timestamp'] >= window_start) & 
+                    (apr_data['timestamp'] <= current_time)
+                ]
+                
+                # Calculate the average APR for the time window
+                if not window_data.empty:
+                    avg_apr_data_with_ma.at[i, 'moving_avg'] = window_data['apr'].mean()
+                else:
+                    # If no data points in the window, use the current value
+                    avg_apr_data_with_ma.at[i, 'moving_avg'] = row['apr']
+            
+            # Add data points for each agent, but only make top agents visible by default
+            unique_agents = apr_data['agent_name'].unique()
+            colors = px.colors.qualitative.Plotly[:len(unique_agents)]
+            color_map = {agent: colors[i % len(colors)] for i, agent in enumerate(unique_agents)}
             
-            # Add a single trace with markers+lines
+            # Calculate the total number of data points per agent
+            agent_counts = apr_data['agent_name'].value_counts()
+            
+            # Determine how many agents to show individually (limit to top 5 most active)
+            MAX_VISIBLE_AGENTS = 5
+            top_agents = agent_counts.nlargest(min(MAX_VISIBLE_AGENTS, len(agent_counts))).index.tolist()
+            
+            for agent_name in unique_agents:
+                agent_data = apr_data[apr_data['agent_name'] == agent_name]
+                
+                # Determine if this agent should be visible by default
+                is_visible = agent_name in top_agents
+                
+                # Add data points as markers
+                simple_fig.add_trace(
+                    go.Scatter(
+                        x=agent_data['timestamp'],
+                        y=agent_data['apr'],
+                        mode='markers',
+                        name=f'Agent: {agent_name}',
+                        marker=dict(
+                            size=10, 
+                            color=color_map[agent_name]
+                        ),
+                        hovertemplate='Time: %{x}<br>APR: %{y:.2f}<br>Agent: ' + agent_name + '<extra></extra>',
+                        visible=is_visible  # Only top agents visible by default
+                    )
+                )
+            
+            # Add moving average as a line
             simple_fig.add_trace(
                 go.Scatter(
-                    x=agent_data['timestamp'],
-                    y=agent_data['apr'],
-                    mode='lines+markers',
-                    name=agent_name,
-                    marker=dict(size=10),
-                    line=dict(width=2)
+                    x=avg_apr_data_with_ma['timestamp'],
+                    y=avg_apr_data_with_ma['moving_avg'],
+                    mode='lines',
+                    name='Moving Average APR (2h window)',
+                    line=dict(width=3, color='red')
                 )
             )
         
         # Simplified layout
         simple_fig.update_layout(
-            title="APR Values for All Agents",
+            title="Average APR Values Across All Agents",
             xaxis_title="Time",
             yaxis_title="Value",
             yaxis=dict(range=[min_apr, max_apr]),
@@ -791,6 +935,17 @@ def create_combined_time_series_graph(df):
             width=1000
         )
         
+        # Add a note about hidden agents if there are more than MAX_VISIBLE_AGENTS
+        if len(unique_agents) > MAX_VISIBLE_AGENTS:
+            simple_fig.add_annotation(
+                text=f"Note: Only showing top {MAX_VISIBLE_AGENTS} agents by default. Toggle others in legend.",
+                xref="paper", yref="paper",
+                x=0.5, y=1.05,
+                showarrow=False,
+                font=dict(size=12, color="gray"),
+                align="center"
+            )
+        
         # Return the simple figure
         return simple_fig
 
@@ -1235,15 +1390,7 @@ def dashboard():
                         )
                         return error_fig
                 
-                # Set up the button click event with error handling
-                try:
-                    # Use Gradio's button click properly
-                    refresh_btn.click(fn=update_apr_graph, outputs=combined_graph)
-                except Exception as e:
-                    logger.error(f"Error setting up button handler: {e}")
-                    
-                # Initialize the graph on load
-                # We'll use placeholder figure initially
+                # Initialize the graph on load with a placeholder
                 placeholder_fig = go.Figure()
                 placeholder_fig.add_annotation(
                     text="Click 'Refresh APR Data' to load APR graph", 
@@ -1252,6 +1399,9 @@ def dashboard():
                     font=dict(size=15)
                 )
                 combined_graph.value = placeholder_fig
+                
+                # Set up the button click event
+                refresh_btn.click(fn=update_apr_graph, outputs=[combined_graph])
         
     return demo