fix: update graph apr and roi

app.py

@@ -195,6 +195,10 @@ def extract_apr_value(attr: Dict[str, Any]) -> Dict[str, Any]:
         if "calculation_metrics" in json_data and json_data["calculation_metrics"] is not None:
             roi = json_data["calculation_metrics"].get("f_i_ratio")
         
+        # Filter ROI values to -10 to 10 range
+        if roi is not None and (roi < -10 or roi > 10):
+            roi = None  # Exclude ROI values outside the range
+        
         logger.debug(f"Agent {agent_id}: Raw APR value: {apr}, adjusted APR value: {adjusted_apr}, ROI value: {roi}, timestamp: {timestamp}")
         
         # Convert timestamp to datetime if it exists
@@ -302,22 +306,18 @@ def fetch_apr_data_from_db():
                 
                 # Process ROI data
                 if data["roi"] is not None:
-                    # Include all ROI values except extreme outliers
-                    if data["roi"] > -10 and data["roi"] < 10:  # Filter extreme outliers
-                        roi_entry = {
-                            "roi": data["roi"],
-                            "timestamp": data["timestamp"],
-                            "agent_id": data["agent_id"],
-                            "agent_name": agent_name,
-                            "is_dummy": False,
-                            "metric_type": "ROI"
-                        }
-                        logger.debug(f"Agent {agent_name} ({attr['agent_id']}): ROI value: {data['roi']}")
-                        # Add to the ROI data list
-                        roi_data_list.append(roi_entry)
-                    else:
-                        # Log that we're skipping extreme outlier values
-                        logger.debug(f"Skipping ROI value for agent {agent_name} ({attr['agent_id']}): {data['roi']} (extreme outlier)")
+                    # Include all ROI values
+                    roi_entry = {
+                        "roi": data["roi"],
+                        "timestamp": data["timestamp"],
+                        "agent_id": data["agent_id"],
+                        "agent_name": agent_name,
+                        "is_dummy": False,
+                        "metric_type": "ROI"
+                    }
+                    logger.debug(f"Agent {agent_name} ({attr['agent_id']}): ROI value: {data['roi']}")
+                    # Add to the ROI data list
+                    roi_data_list.append(roi_entry)
         
         logger.info(f"Extracted {len(apr_data_list)} valid APR data points and {len(roi_data_list)} valid ROI data points")
         
@@ -1213,28 +1213,17 @@ def create_combined_roi_time_series_graph(df):
         x0=min_time, x1=max_time
     )
     
-    # Filter out extreme outliers for better visualization (ROI values above 100% or below -100%)
-    outlier_data = df[(df['roi'] > 200) | (df['roi'] < -200)].copy()
-    df_filtered = df[(df['roi'] <= 200) & (df['roi'] >= -200)].copy()
-    
-    # Log the outliers for better debugging
-    if len(outlier_data) > 0:
-        excluded_count = len(outlier_data)
-        logger.info(f"Excluded {excluded_count} data points with outlier ROI values (>200% or <-200%)")
-        
-        # Group outliers by agent for detailed logging
-        outlier_agents = outlier_data.groupby('agent_name')
-        for agent_name, agent_outliers in outlier_agents:
-            logger.info(f"Agent '{agent_name}' has {len(agent_outliers)} outlier values:")
-            for idx, row in agent_outliers.iterrows():
-                logger.info(f"  - ROI: {row['roi']}, timestamp: {row['timestamp']}")
+    # Filter ROI outliers for better visualization (±200% range)
+    before_outlier_filter = len(df)
+    df = df[(df['roi'] <= 200) & (df['roi'] >= -200)]
+    after_outlier_filter = len(df)
+    excluded_by_outlier = before_outlier_filter - after_outlier_filter
     
-    # Use the filtered data for all subsequent operations
-    df = df_filtered
+    logger.info(f"ROI outlier filtering: {before_outlier_filter} -> {after_outlier_filter} data points ({excluded_by_outlier} excluded)")
     
-    # IMPORTANT: Filter data by hardcoded date range (June 6 to June 19, 2025)
+    # IMPORTANT: Filter data by hardcoded date range (June 6 to June 22, 2025)
     min_date = datetime(2025, 6, 6)
-    max_date = datetime(2025, 6, 19, 23, 59, 59)  # Include all of June 19th
+    max_date = datetime(2025, 6, 22, 23, 59, 59)  # Include all of June 22nd
     logger.info(f"Filtering ROI data to date range: {min_date} to {max_date}")
     
     # Count data points before filtering
@@ -1420,9 +1409,9 @@ def create_combined_roi_time_series_graph(df):
         title=None  # Remove the built-in axis title since we're using annotations
     )
     
-    # Update x-axis with better formatting and hardcoded date range (June 6 to June 19)
+    # Update x-axis with better formatting and hardcoded date range (June 6 to June 22)
     min_date = datetime(2025, 6, 6)  # Hardcoded start date: June 6, 2025
-    max_date = datetime(2025, 6, 19)  # Hardcoded end date: June 19, 2025
+    max_date = datetime(2025, 6, 22)  # Hardcoded end date: June 22, 2025
     logger.info(f"ROI Graph - Hardcoded date range: min_date = {min_date}, max_date = {max_date}")
     fig.update_xaxes(
         showgrid=True, 
@@ -1816,28 +1805,17 @@ def create_combined_time_series_graph(df):
     # Filter for APR data only
     apr_data = df[df['metric_type'] == 'APR'].copy()
     
-    # Filter out outliers (APR values above 200 or below -200)
-    outlier_data = apr_data[(apr_data['apr'] > 200) | (apr_data['apr'] < -200)].copy()
-    apr_data_filtered = apr_data[(apr_data['apr'] <= 200) & (apr_data['apr'] >= -200)].copy()
-    
-    # Log the outliers for better debugging
-    if len(outlier_data) > 0:
-        excluded_count = len(outlier_data)
-        logger.info(f"Excluded {excluded_count} data points with outlier APR values (>200 or <-200)")
-        
-        # Group outliers by agent for detailed logging
-        outlier_agents = outlier_data.groupby('agent_name')
-        for agent_name, agent_outliers in outlier_agents:
-            logger.info(f"Agent '{agent_name}' has {len(agent_outliers)} outlier values:")
-            for idx, row in agent_outliers.iterrows():
-                logger.info(f"  - APR: {row['apr']}, timestamp: {row['timestamp']}")
+    # Filter APR outliers (±200% range)
+    before_outlier_filter = len(apr_data)
+    apr_data = apr_data[(apr_data['apr'] <= 200) & (apr_data['apr'] >= -200)]
+    after_outlier_filter = len(apr_data)
+    excluded_by_outlier = before_outlier_filter - after_outlier_filter
     
-    # Use the filtered data for all subsequent operations
-    apr_data = apr_data_filtered
+    logger.info(f"APR outlier filtering: {before_outlier_filter} -> {after_outlier_filter} data points ({excluded_by_outlier} excluded)")
     
-    # IMPORTANT: Filter data by hardcoded date range (June 6 to June 19, 2025)
+    # IMPORTANT: Filter data by hardcoded date range (June 6 to June 22, 2025)
     min_date = datetime(2025, 6, 6)
-    max_date = datetime(2025, 6, 19, 23, 59, 59)  # Include all of June 19th
+    max_date = datetime(2025, 6, 22, 23, 59, 59)  # Include all of June 22nd
     logger.info(f"Filtering APR data to date range: {min_date} to {max_date}")
     
     # Count data points before filtering
@@ -2145,9 +2123,9 @@ def create_combined_time_series_graph(df):
         title=None  # Remove the built-in axis title since we're using annotations
     )
     
-    # Update x-axis with better formatting and hardcoded date range (June 6 to June 19)
+    # Update x-axis with better formatting and hardcoded date range (June 6 to June 22)
     min_date = datetime(2025, 6, 6)  # Hardcoded start date: June 6, 2025
-    max_date = datetime(2025, 6, 19)  # Hardcoded end date: June 19, 2025
+    max_date = datetime(2025, 6, 22)  # Hardcoded end date: June 22, 2025
     logger.info(f"APR Graph - Hardcoded date range: min_date = {min_date}, max_date = {max_date}")
     fig.update_xaxes(
         showgrid=True,