Spaces:
Sleeping
Sleeping
gauravlochab
commited on
Commit
·
75be6c1
1
Parent(s):
6c05858
chore: fix counts for daily active agents
Browse files
app.py
CHANGED
|
@@ -33,12 +33,20 @@ logging.basicConfig(
|
|
| 33 |
level=logging.INFO, # Use INFO level instead of DEBUG to reduce verbosity
|
| 34 |
format="%(asctime)s - %(levelname)s - %(message)s",
|
| 35 |
handlers=[
|
| 36 |
-
logging.FileHandler("app_debug.log"), #
|
| 37 |
logging.StreamHandler() # Also log to console
|
| 38 |
-
]
|
|
|
|
| 39 |
)
|
| 40 |
logger = logging.getLogger(__name__)
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
# Reduce third-party library logging
|
| 43 |
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
| 44 |
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
@@ -759,11 +767,11 @@ def log_adjusted_apr_availability(df):
|
|
| 759 |
logger.info(f"Agent {agent_name} (ID: {agent_id}): Missing adjusted_apr from {gap_start} to {gap_end} ({gap_duration.days} days, {gap_duration.seconds//3600} hours)")
|
| 760 |
|
| 761 |
def generate_apr_visualizations():
|
| 762 |
-
"""Generate APR visualizations
|
| 763 |
global global_df
|
| 764 |
|
| 765 |
-
#
|
| 766 |
-
logger.info("
|
| 767 |
df, csv_file = load_apr_data_from_csv()
|
| 768 |
|
| 769 |
if not df.empty:
|
|
@@ -775,136 +783,70 @@ def generate_apr_visualizations():
|
|
| 775 |
combined_fig = create_combined_time_series_graph(df)
|
| 776 |
return combined_fig, csv_file
|
| 777 |
|
| 778 |
-
# FALLBACK: If CSV not available,
|
| 779 |
-
logger.
|
| 780 |
-
|
| 781 |
-
|
| 782 |
-
|
| 783 |
-
|
| 784 |
-
|
| 785 |
-
|
| 786 |
-
|
| 787 |
-
|
| 788 |
-
|
| 789 |
-
|
| 790 |
-
|
| 791 |
-
|
| 792 |
-
|
| 793 |
-
|
| 794 |
-
fig.update_layout(
|
| 795 |
-
xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
|
| 796 |
-
yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)
|
| 797 |
-
)
|
| 798 |
-
|
| 799 |
-
# Save as static file for reference
|
| 800 |
-
fig.write_html("optimus_apr_combined_graph.html")
|
| 801 |
-
fig.write_image("optimus_apr_combined_graph.png")
|
| 802 |
-
|
| 803 |
-
csv_file = None
|
| 804 |
-
return fig, csv_file
|
| 805 |
-
|
| 806 |
-
# Apply preprocessing to fix APR and ROI values
|
| 807 |
-
logger.info("Applying preprocessing to fix APR and ROI values...")
|
| 808 |
-
df = fix_apr_and_roi(df) # Apply preprocessing
|
| 809 |
-
global_df = df
|
| 810 |
-
|
| 811 |
-
# IMPORTANT: Also fix the ROI DataFrame with corrected values
|
| 812 |
-
logger.info("Extracting corrected ROI values from fixed APR data...")
|
| 813 |
-
if not df.empty and 'roi' in df.columns:
|
| 814 |
-
# Create corrected ROI DataFrame from the fixed APR data
|
| 815 |
-
corrected_roi_data = []
|
| 816 |
-
for idx, row in df.iterrows():
|
| 817 |
-
if not row['is_dummy'] and pd.notna(row['roi']):
|
| 818 |
-
roi_entry = {
|
| 819 |
-
"roi": row["roi"], # This is now the corrected ROI value
|
| 820 |
-
"timestamp": row["timestamp"],
|
| 821 |
-
"agent_id": row["agent_id"],
|
| 822 |
-
"agent_name": row["agent_name"],
|
| 823 |
-
"is_dummy": False,
|
| 824 |
-
"metric_type": "ROI"
|
| 825 |
-
}
|
| 826 |
-
corrected_roi_data.append(roi_entry)
|
| 827 |
-
|
| 828 |
-
# Replace the original ROI DataFrame with corrected values
|
| 829 |
-
if corrected_roi_data:
|
| 830 |
-
corrected_roi_df = pd.DataFrame(corrected_roi_data)
|
| 831 |
-
|
| 832 |
-
# Combine with dummy ROI data if it exists
|
| 833 |
-
if global_roi_df is not None and not global_roi_df.empty:
|
| 834 |
-
dummy_roi_data = global_roi_df[global_roi_df['is_dummy'] == True]
|
| 835 |
-
if not dummy_roi_data.empty:
|
| 836 |
-
global_roi_df = pd.concat([corrected_roi_df, dummy_roi_data], ignore_index=True)
|
| 837 |
-
else:
|
| 838 |
-
global_roi_df = corrected_roi_df
|
| 839 |
-
else:
|
| 840 |
-
global_roi_df = corrected_roi_df
|
| 841 |
-
|
| 842 |
-
logger.info(f"Updated ROI DataFrame with {len(corrected_roi_data)} corrected ROI values")
|
| 843 |
-
else:
|
| 844 |
-
logger.warning("No corrected ROI values found to update ROI DataFrame")
|
| 845 |
-
|
| 846 |
-
# Save preprocessed data to CSV before creating visualizations
|
| 847 |
-
logger.info("Saving preprocessed APR data to CSV...")
|
| 848 |
-
csv_file = save_to_csv(df)
|
| 849 |
-
|
| 850 |
-
# Create visualizations using the saved CSV data
|
| 851 |
-
logger.info("Creating APR visualizations from preprocessed data...")
|
| 852 |
-
combined_fig = create_combined_time_series_graph(df)
|
| 853 |
-
|
| 854 |
-
return combined_fig, csv_file
|
| 855 |
-
|
| 856 |
-
except Exception as e:
|
| 857 |
-
logger.error(f"Error fetching APR data from API: {e}")
|
| 858 |
-
# Return error visualization
|
| 859 |
-
fig = go.Figure()
|
| 860 |
-
fig.add_annotation(
|
| 861 |
-
x=0.5, y=0.5,
|
| 862 |
-
text=f"Error loading data: {str(e)}",
|
| 863 |
-
font=dict(size=16, color="red"),
|
| 864 |
-
showarrow=False
|
| 865 |
-
)
|
| 866 |
-
fig.update_layout(
|
| 867 |
-
xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
|
| 868 |
-
yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)
|
| 869 |
-
)
|
| 870 |
-
return fig, None
|
| 871 |
|
| 872 |
def generate_roi_visualizations():
|
| 873 |
-
"""Generate ROI visualizations
|
| 874 |
global global_roi_df
|
| 875 |
|
| 876 |
-
#
|
| 877 |
-
logger.info("Loading
|
| 878 |
df_apr, csv_file = load_apr_data_from_csv()
|
| 879 |
|
| 880 |
if not df_apr.empty and 'roi' in df_apr.columns:
|
| 881 |
-
#
|
| 882 |
-
|
| 883 |
-
|
| 884 |
-
|
| 885 |
-
|
| 886 |
-
|
| 887 |
-
|
| 888 |
-
|
| 889 |
-
|
| 890 |
-
|
| 891 |
-
|
| 892 |
-
|
| 893 |
-
|
| 894 |
-
|
| 895 |
-
|
| 896 |
-
|
| 897 |
-
|
| 898 |
-
|
| 899 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 900 |
global_roi_df = df_roi
|
| 901 |
|
| 902 |
-
# Create visualizations using
|
| 903 |
-
logger.info("Creating ROI visualizations from
|
| 904 |
combined_fig = create_combined_roi_time_series_graph(df_roi)
|
| 905 |
-
return combined_fig,
|
| 906 |
else:
|
| 907 |
-
logger.warning("No ROI data found in APR CSV")
|
| 908 |
else:
|
| 909 |
logger.warning("APR CSV not available or missing ROI column")
|
| 910 |
|
|
@@ -933,68 +875,16 @@ def generate_roi_visualizations():
|
|
| 933 |
yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)
|
| 934 |
)
|
| 935 |
|
| 936 |
-
|
| 937 |
-
fig.write_html("optimus_roi_graph.html")
|
| 938 |
-
fig.write_image("optimus_roi_graph.png")
|
| 939 |
-
|
| 940 |
-
csv_file = None
|
| 941 |
-
return fig, csv_file
|
| 942 |
|
| 943 |
# Set global_roi_df for access by other functions
|
| 944 |
global_roi_df = df_roi
|
| 945 |
|
| 946 |
-
#
|
| 947 |
-
logger.info("
|
| 948 |
-
if not df_roi.empty:
|
| 949 |
-
# Check if this ROI data contains uncorrected values (from API)
|
| 950 |
-
uncorrected_roi = df_roi[df_roi['is_dummy'] == False]
|
| 951 |
-
if not uncorrected_roi.empty:
|
| 952 |
-
logger.info("ROI data contains uncorrected values, applying corrections...")
|
| 953 |
-
|
| 954 |
-
# We need to get the corrected APR data to extract corrected ROI values
|
| 955 |
-
if global_df is not None and not global_df.empty:
|
| 956 |
-
# Extract corrected ROI values from the fixed APR data
|
| 957 |
-
corrected_roi_data = []
|
| 958 |
-
for idx, row in global_df.iterrows():
|
| 959 |
-
if not row['is_dummy'] and pd.notna(row['roi']):
|
| 960 |
-
roi_entry = {
|
| 961 |
-
"roi": row["roi"], # This is the corrected ROI value
|
| 962 |
-
"timestamp": row["timestamp"],
|
| 963 |
-
"agent_id": row["agent_id"],
|
| 964 |
-
"agent_name": row["agent_name"],
|
| 965 |
-
"is_dummy": False,
|
| 966 |
-
"metric_type": "ROI"
|
| 967 |
-
}
|
| 968 |
-
corrected_roi_data.append(roi_entry)
|
| 969 |
-
|
| 970 |
-
if corrected_roi_data:
|
| 971 |
-
corrected_roi_df = pd.DataFrame(corrected_roi_data)
|
| 972 |
-
|
| 973 |
-
# Combine with dummy ROI data if it exists
|
| 974 |
-
dummy_roi_data = df_roi[df_roi['is_dummy'] == True]
|
| 975 |
-
if not dummy_roi_data.empty:
|
| 976 |
-
df_roi = pd.concat([corrected_roi_df, dummy_roi_data], ignore_index=True)
|
| 977 |
-
else:
|
| 978 |
-
df_roi = corrected_roi_df
|
| 979 |
-
|
| 980 |
-
global_roi_df = df_roi
|
| 981 |
-
logger.info(f"Updated ROI DataFrame with {len(corrected_roi_data)} corrected ROI values")
|
| 982 |
-
else:
|
| 983 |
-
logger.warning("No corrected ROI values found in APR data")
|
| 984 |
-
else:
|
| 985 |
-
logger.warning("No corrected APR data available to extract ROI values from")
|
| 986 |
-
else:
|
| 987 |
-
logger.info("ROI data contains only dummy values, no correction needed")
|
| 988 |
-
|
| 989 |
-
# Save preprocessed ROI data to CSV before creating visualizations
|
| 990 |
-
logger.info("Saving preprocessed ROI data to CSV...")
|
| 991 |
-
csv_file = save_roi_to_csv(df_roi)
|
| 992 |
-
|
| 993 |
-
# Create visualizations using the saved CSV data
|
| 994 |
-
logger.info("Creating ROI visualizations from preprocessed data...")
|
| 995 |
combined_fig = create_combined_roi_time_series_graph(df_roi)
|
| 996 |
|
| 997 |
-
return combined_fig,
|
| 998 |
|
| 999 |
except Exception as e:
|
| 1000 |
logger.error(f"Error fetching ROI data from API: {e}")
|
|
@@ -1030,6 +920,13 @@ def aggregate_daily_data(df, metric_column):
|
|
| 1030 |
df = df.copy()
|
| 1031 |
df['date'] = df['timestamp'].dt.date
|
| 1032 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1033 |
# NEW: Add detailed logging to verify median calculation
|
| 1034 |
logger.info(f"=== MEDIAN CALCULATION DEBUG for {metric_column} ===")
|
| 1035 |
|
|
@@ -1197,39 +1094,8 @@ def create_combined_roi_time_series_graph(df):
|
|
| 1197 |
for agent_id, data in agent_runtimes.items():
|
| 1198 |
logger.info(f"Agent {data['agent_name']} (ID: {agent_id}): Runtime = {data['runtime_days']:.2f} days, Last report: {data['last_report']}")
|
| 1199 |
|
| 1200 |
-
#
|
| 1201 |
-
logger.info("
|
| 1202 |
-
|
| 1203 |
-
def clean_roi_value(value):
|
| 1204 |
-
"""Clean and convert ROI value to float"""
|
| 1205 |
-
if pd.isna(value):
|
| 1206 |
-
return None
|
| 1207 |
-
|
| 1208 |
-
# If it's already a number, return it
|
| 1209 |
-
if isinstance(value, (int, float)):
|
| 1210 |
-
return float(value)
|
| 1211 |
-
|
| 1212 |
-
# If it's a string, try to extract numeric value
|
| 1213 |
-
if isinstance(value, str):
|
| 1214 |
-
# Remove any non-numeric characters except decimal point and minus sign
|
| 1215 |
-
import re
|
| 1216 |
-
# Look for patterns like "value': 16.007665648354" and extract the number
|
| 1217 |
-
match = re.search(r'[\d\.-]+', value)
|
| 1218 |
-
if match:
|
| 1219 |
-
try:
|
| 1220 |
-
return float(match.group())
|
| 1221 |
-
except ValueError:
|
| 1222 |
-
logger.warning(f"Could not convert ROI value to float: {value}")
|
| 1223 |
-
return None
|
| 1224 |
-
else:
|
| 1225 |
-
logger.warning(f"No numeric value found in ROI string: {value}")
|
| 1226 |
-
return None
|
| 1227 |
-
|
| 1228 |
-
logger.warning(f"Unexpected ROI value type: {type(value)} - {value}")
|
| 1229 |
-
return None
|
| 1230 |
-
|
| 1231 |
-
# Apply cleaning function to ROI column
|
| 1232 |
-
df['roi'] = df['roi'].apply(clean_roi_value)
|
| 1233 |
|
| 1234 |
# Remove rows with invalid ROI values
|
| 1235 |
initial_count = len(df)
|
|
@@ -1240,10 +1106,9 @@ def create_combined_roi_time_series_graph(df):
|
|
| 1240 |
if removed_count > 0:
|
| 1241 |
logger.warning(f"Removed {removed_count} rows with invalid ROI values")
|
| 1242 |
|
| 1243 |
-
# Ensure
|
| 1244 |
df['roi'] = df['roi'].astype(float)
|
| 1245 |
-
|
| 1246 |
-
df['metric_type'] = df['metric_type'].astype(str) # Ensure metric_type is string
|
| 1247 |
|
| 1248 |
# Get min and max time for shapes
|
| 1249 |
min_time = df['timestamp'].min()
|
|
@@ -1406,6 +1271,12 @@ def create_combined_roi_time_series_graph(df):
|
|
| 1406 |
# Calculate number of active agents on this date
|
| 1407 |
active_agents = len(daily_agent_data[daily_agent_data['timestamp'] == timestamp]['agent_id'].unique())
|
| 1408 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1409 |
hover_data_roi.append(
|
| 1410 |
f"Date: {formatted_timestamp}<br>Median ROI (7d window): {row['moving_avg']:.2f}%<br>Active agents: {active_agents}"
|
| 1411 |
)
|
|
@@ -1858,22 +1729,22 @@ def create_combined_time_series_graph(df):
|
|
| 1858 |
min_time = df['timestamp'].min()
|
| 1859 |
max_time = df['timestamp'].max()
|
| 1860 |
|
| 1861 |
-
# Add shape for positive APR region (above zero)
|
| 1862 |
fig.add_shape(
|
| 1863 |
type="rect",
|
| 1864 |
fillcolor="rgba(230, 243, 255, 0.3)",
|
| 1865 |
line=dict(width=0),
|
| 1866 |
-
y0=0, y1=
|
| 1867 |
x0=min_time, x1=max_time,
|
| 1868 |
layer="below"
|
| 1869 |
)
|
| 1870 |
|
| 1871 |
-
# Add shape for negative APR region (below zero)
|
| 1872 |
fig.add_shape(
|
| 1873 |
type="rect",
|
| 1874 |
fillcolor="rgba(255, 230, 230, 0.3)",
|
| 1875 |
line=dict(width=0),
|
| 1876 |
-
y0
|
| 1877 |
x0=min_time, x1=max_time,
|
| 1878 |
layer="below"
|
| 1879 |
)
|
|
@@ -1888,7 +1759,22 @@ def create_combined_time_series_graph(df):
|
|
| 1888 |
|
| 1889 |
# MODIFIED: Calculate average APR values across all agents for each timestamp
|
| 1890 |
# Filter for APR data only
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1891 |
apr_data = df[df['metric_type'] == 'APR'].copy()
|
|
|
|
|
|
|
|
|
|
| 1892 |
|
| 1893 |
# Filter APR outliers (±200% range)
|
| 1894 |
before_outlier_filter = len(apr_data)
|
|
@@ -2046,14 +1932,39 @@ def create_combined_time_series_graph(df):
|
|
| 2046 |
y_values_ma = daily_medians_with_ma['moving_avg'].tolist()
|
| 2047 |
|
| 2048 |
# Create hover template for the APR moving average line
|
|
|
|
| 2049 |
hover_data_apr = []
|
| 2050 |
for idx, row in daily_medians_with_ma.iterrows():
|
| 2051 |
timestamp = row['timestamp']
|
| 2052 |
# Format timestamp to show only date for daily data
|
| 2053 |
formatted_timestamp = timestamp.strftime('%Y-%m-%d')
|
| 2054 |
|
| 2055 |
-
#
|
| 2056 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2057 |
|
| 2058 |
hover_data_apr.append(
|
| 2059 |
f"Date: {formatted_timestamp}<br>Median APR (7d window): {row['moving_avg']:.2f}%<br>Active agents: {active_agents}"
|
|
@@ -2079,14 +1990,38 @@ def create_combined_time_series_graph(df):
|
|
| 2079 |
y_values_adj_ma = daily_medians_adjusted_with_ma['moving_avg'].tolist()
|
| 2080 |
|
| 2081 |
# Create hover template for the adjusted APR moving average line
|
|
|
|
| 2082 |
hover_data_adj = []
|
| 2083 |
for idx, row in daily_medians_adjusted_with_ma.iterrows():
|
| 2084 |
timestamp = row['timestamp']
|
| 2085 |
# Format timestamp to show only date for daily data
|
| 2086 |
formatted_timestamp = timestamp.strftime('%Y-%m-%d')
|
| 2087 |
|
| 2088 |
-
#
|
| 2089 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2090 |
|
| 2091 |
hover_data_adj.append(
|
| 2092 |
f"Date: {formatted_timestamp}<br>Median Adjusted APR (7d window): {row['moving_avg']:.2f}%<br>Active agents: {active_agents}"
|
|
@@ -2141,25 +2076,13 @@ def create_combined_time_series_graph(df):
|
|
| 2141 |
hovermode="closest"
|
| 2142 |
)
|
| 2143 |
|
| 2144 |
-
# Add
|
| 2145 |
-
fig.add_annotation(
|
| 2146 |
-
x=-0.08, # Position further from the y-axis to avoid overlapping with tick labels
|
| 2147 |
-
y=-25, # Middle of the negative region
|
| 2148 |
-
xref="paper",
|
| 2149 |
-
yref="y",
|
| 2150 |
-
text="Percent drawdown [%]",
|
| 2151 |
-
showarrow=False,
|
| 2152 |
-
font=dict(size=16, family="Arial, sans-serif", color="black", weight="bold"), # Adjusted font size
|
| 2153 |
-
textangle=-90, # Rotate text to be vertical
|
| 2154 |
-
align="center"
|
| 2155 |
-
)
|
| 2156 |
-
|
| 2157 |
fig.add_annotation(
|
| 2158 |
x=-0.08, # Position further from the y-axis to avoid overlapping with tick labels
|
| 2159 |
-
y=
|
| 2160 |
xref="paper",
|
| 2161 |
yref="y",
|
| 2162 |
-
text="Agent APR
|
| 2163 |
showarrow=False,
|
| 2164 |
font=dict(size=16, family="Arial, sans-serif", color="black", weight="bold"), # Adjusted font size
|
| 2165 |
textangle=-90, # Rotate text to be vertical
|
|
|
|
| 33 |
level=logging.INFO, # Use INFO level instead of DEBUG to reduce verbosity
|
| 34 |
format="%(asctime)s - %(levelname)s - %(message)s",
|
| 35 |
handlers=[
|
| 36 |
+
logging.FileHandler("app_debug.log", mode='a'), # Append mode for persistence
|
| 37 |
logging.StreamHandler() # Also log to console
|
| 38 |
+
],
|
| 39 |
+
force=True # Force reconfiguration of logging
|
| 40 |
)
|
| 41 |
logger = logging.getLogger(__name__)
|
| 42 |
|
| 43 |
+
# Ensure the logger level is set correctly
|
| 44 |
+
logger.setLevel(logging.INFO)
|
| 45 |
+
|
| 46 |
+
# Test logging to verify it's working
|
| 47 |
+
logger.info("=== LOGGING SYSTEM INITIALIZED ===")
|
| 48 |
+
logger.info("Debug logs will be written to app_debug.log")
|
| 49 |
+
|
| 50 |
# Reduce third-party library logging
|
| 51 |
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
| 52 |
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
|
|
| 767 |
logger.info(f"Agent {agent_name} (ID: {agent_id}): Missing adjusted_apr from {gap_start} to {gap_end} ({gap_duration.days} days, {gap_duration.seconds//3600} hours)")
|
| 768 |
|
| 769 |
def generate_apr_visualizations():
|
| 770 |
+
"""Generate APR visualizations using CSV data only for consistency with ROI graph"""
|
| 771 |
global global_df
|
| 772 |
|
| 773 |
+
# CONSISTENCY FIX: Always use CSV data to match ROI graph behavior
|
| 774 |
+
logger.info("Loading APR data from CSV files for consistency with ROI graph...")
|
| 775 |
df, csv_file = load_apr_data_from_csv()
|
| 776 |
|
| 777 |
if not df.empty:
|
|
|
|
| 783 |
combined_fig = create_combined_time_series_graph(df)
|
| 784 |
return combined_fig, csv_file
|
| 785 |
|
| 786 |
+
# FALLBACK: If CSV not available, return error message
|
| 787 |
+
logger.error("CSV data not available and API fallback disabled for consistency")
|
| 788 |
+
# Create empty visualization with a message using Plotly
|
| 789 |
+
fig = go.Figure()
|
| 790 |
+
fig.add_annotation(
|
| 791 |
+
x=0.5, y=0.5,
|
| 792 |
+
text="No APR data available - CSV file missing",
|
| 793 |
+
font=dict(size=20),
|
| 794 |
+
showarrow=False
|
| 795 |
+
)
|
| 796 |
+
fig.update_layout(
|
| 797 |
+
xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
|
| 798 |
+
yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)
|
| 799 |
+
)
|
| 800 |
+
|
| 801 |
+
return fig, None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 802 |
|
| 803 |
def generate_roi_visualizations():
|
| 804 |
+
"""Generate ROI visualizations directly from optimus_apr_values.csv"""
|
| 805 |
global global_roi_df
|
| 806 |
|
| 807 |
+
# SIMPLIFIED APPROACH: Load ROI data directly from APR CSV
|
| 808 |
+
logger.info("Loading ROI data directly from optimus_apr_values.csv...")
|
| 809 |
df_apr, csv_file = load_apr_data_from_csv()
|
| 810 |
|
| 811 |
if not df_apr.empty and 'roi' in df_apr.columns:
|
| 812 |
+
# CONSISTENCY FIX: Apply same filtering as APR graph
|
| 813 |
+
logger.info("=== ROI GRAPH DATA FILTERING DEBUG ===")
|
| 814 |
+
logger.info(f"Initial APR data loaded: {len(df_apr)} records")
|
| 815 |
+
logger.info(f"Unique agents in initial data: {df_apr['agent_id'].nunique()}")
|
| 816 |
+
logger.info(f"Agent IDs in initial data: {sorted(df_apr['agent_id'].unique().tolist())}")
|
| 817 |
+
|
| 818 |
+
# Check metric_type distribution
|
| 819 |
+
if 'metric_type' in df_apr.columns:
|
| 820 |
+
metric_counts = df_apr['metric_type'].value_counts()
|
| 821 |
+
logger.info(f"Metric type distribution: {metric_counts.to_dict()}")
|
| 822 |
+
else:
|
| 823 |
+
logger.warning("No 'metric_type' column found in APR data")
|
| 824 |
+
|
| 825 |
+
# First filter by metric_type == 'APR' to match APR graph logic
|
| 826 |
+
df_apr_filtered = df_apr[df_apr['metric_type'] == 'APR'].copy()
|
| 827 |
+
logger.info(f"After metric_type == 'APR' filter: {len(df_apr_filtered)} records")
|
| 828 |
+
logger.info(f"Unique agents after APR filter: {df_apr_filtered['agent_id'].nunique()}")
|
| 829 |
+
logger.info(f"Agent IDs after APR filter: {sorted(df_apr_filtered['agent_id'].unique().tolist())}")
|
| 830 |
+
|
| 831 |
+
# Then filter for rows with valid ROI values
|
| 832 |
+
df_roi = df_apr_filtered[df_apr_filtered['roi'].notna()].copy()
|
| 833 |
+
logger.info(f"After ROI filter: {len(df_roi)} records")
|
| 834 |
+
logger.info(f"Unique agents after ROI filter: {df_roi['agent_id'].nunique()}")
|
| 835 |
+
logger.info(f"Agent IDs after ROI filter: {sorted(df_roi['agent_id'].unique().tolist())}")
|
| 836 |
+
|
| 837 |
+
if not df_roi.empty:
|
| 838 |
+
# Add metric_type column for consistency
|
| 839 |
+
df_roi['metric_type'] = 'ROI'
|
| 840 |
+
|
| 841 |
+
logger.info(f"Successfully loaded {len(df_roi)} ROI records from APR CSV")
|
| 842 |
global_roi_df = df_roi
|
| 843 |
|
| 844 |
+
# Create visualizations using ROI data from APR CSV
|
| 845 |
+
logger.info("Creating ROI visualizations from APR CSV data...")
|
| 846 |
combined_fig = create_combined_roi_time_series_graph(df_roi)
|
| 847 |
+
return combined_fig, csv_file
|
| 848 |
else:
|
| 849 |
+
logger.warning("No valid ROI data found in APR CSV")
|
| 850 |
else:
|
| 851 |
logger.warning("APR CSV not available or missing ROI column")
|
| 852 |
|
|
|
|
| 875 |
yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)
|
| 876 |
)
|
| 877 |
|
| 878 |
+
return fig, None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 879 |
|
| 880 |
# Set global_roi_df for access by other functions
|
| 881 |
global_roi_df = df_roi
|
| 882 |
|
| 883 |
+
# Create visualizations using API data
|
| 884 |
+
logger.info("Creating ROI visualizations from API data...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 885 |
combined_fig = create_combined_roi_time_series_graph(df_roi)
|
| 886 |
|
| 887 |
+
return combined_fig, None
|
| 888 |
|
| 889 |
except Exception as e:
|
| 890 |
logger.error(f"Error fetching ROI data from API: {e}")
|
|
|
|
| 920 |
df = df.copy()
|
| 921 |
df['date'] = df['timestamp'].dt.date
|
| 922 |
|
| 923 |
+
# DEBUG: Log July 8th data specifically
|
| 924 |
+
july_8_data = df[df['date'] == pd.to_datetime('2025-07-08').date()]
|
| 925 |
+
if not july_8_data.empty:
|
| 926 |
+
july_8_agents = july_8_data['agent_id'].unique()
|
| 927 |
+
logger.info(f"DAILY AGGREGATION DEBUG ({metric_column}) - July 8th agents before aggregation: {len(july_8_agents)}")
|
| 928 |
+
logger.info(f"DAILY AGGREGATION DEBUG ({metric_column}) - July 8th agent IDs: {sorted(july_8_agents.tolist())}")
|
| 929 |
+
|
| 930 |
# NEW: Add detailed logging to verify median calculation
|
| 931 |
logger.info(f"=== MEDIAN CALCULATION DEBUG for {metric_column} ===")
|
| 932 |
|
|
|
|
| 1094 |
for agent_id, data in agent_runtimes.items():
|
| 1095 |
logger.info(f"Agent {data['agent_name']} (ID: {agent_id}): Runtime = {data['runtime_days']:.2f} days, Last report: {data['last_report']}")
|
| 1096 |
|
| 1097 |
+
# SIMPLIFIED: ROI data is already clean from CSV, just ensure proper data types
|
| 1098 |
+
logger.info("Processing ROI data from CSV...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1099 |
|
| 1100 |
# Remove rows with invalid ROI values
|
| 1101 |
initial_count = len(df)
|
|
|
|
| 1106 |
if removed_count > 0:
|
| 1107 |
logger.warning(f"Removed {removed_count} rows with invalid ROI values")
|
| 1108 |
|
| 1109 |
+
# Ensure proper data types
|
| 1110 |
df['roi'] = df['roi'].astype(float)
|
| 1111 |
+
df['metric_type'] = df['metric_type'].astype(str)
|
|
|
|
| 1112 |
|
| 1113 |
# Get min and max time for shapes
|
| 1114 |
min_time = df['timestamp'].min()
|
|
|
|
| 1271 |
# Calculate number of active agents on this date
|
| 1272 |
active_agents = len(daily_agent_data[daily_agent_data['timestamp'] == timestamp]['agent_id'].unique())
|
| 1273 |
|
| 1274 |
+
# DEBUG: Log agent counts for July 8th specifically
|
| 1275 |
+
if formatted_timestamp == '2025-07-08':
|
| 1276 |
+
agents_on_date = daily_agent_data[daily_agent_data['timestamp'] == timestamp]['agent_id'].unique()
|
| 1277 |
+
logger.info(f"ROI GRAPH - July 8th active agents: {active_agents}")
|
| 1278 |
+
logger.info(f"ROI GRAPH - July 8th agent IDs: {sorted(agents_on_date.tolist())}")
|
| 1279 |
+
|
| 1280 |
hover_data_roi.append(
|
| 1281 |
f"Date: {formatted_timestamp}<br>Median ROI (7d window): {row['moving_avg']:.2f}%<br>Active agents: {active_agents}"
|
| 1282 |
)
|
|
|
|
| 1729 |
min_time = df['timestamp'].min()
|
| 1730 |
max_time = df['timestamp'].max()
|
| 1731 |
|
| 1732 |
+
# Add shape for positive APR region (above zero) - use reasonable fixed range
|
| 1733 |
fig.add_shape(
|
| 1734 |
type="rect",
|
| 1735 |
fillcolor="rgba(230, 243, 255, 0.3)",
|
| 1736 |
line=dict(width=0),
|
| 1737 |
+
y0=0, y1=200, # Fixed positive range to avoid extreme outliers affecting the view
|
| 1738 |
x0=min_time, x1=max_time,
|
| 1739 |
layer="below"
|
| 1740 |
)
|
| 1741 |
|
| 1742 |
+
# Add shape for negative APR region (below zero) - use reasonable fixed range
|
| 1743 |
fig.add_shape(
|
| 1744 |
type="rect",
|
| 1745 |
fillcolor="rgba(255, 230, 230, 0.3)",
|
| 1746 |
line=dict(width=0),
|
| 1747 |
+
y0=-200, y1=0, # Fixed negative range to avoid extreme outliers affecting the view
|
| 1748 |
x0=min_time, x1=max_time,
|
| 1749 |
layer="below"
|
| 1750 |
)
|
|
|
|
| 1759 |
|
| 1760 |
# MODIFIED: Calculate average APR values across all agents for each timestamp
|
| 1761 |
# Filter for APR data only
|
| 1762 |
+
logger.info("=== APR GRAPH DATA FILTERING DEBUG ===")
|
| 1763 |
+
logger.info(f"Initial APR data loaded: {len(df)} records")
|
| 1764 |
+
logger.info(f"Unique agents in initial data: {df['agent_id'].nunique()}")
|
| 1765 |
+
logger.info(f"Agent IDs in initial data: {sorted(df['agent_id'].unique().tolist())}")
|
| 1766 |
+
|
| 1767 |
+
# Check metric_type distribution
|
| 1768 |
+
if 'metric_type' in df.columns:
|
| 1769 |
+
metric_counts = df['metric_type'].value_counts()
|
| 1770 |
+
logger.info(f"Metric type distribution: {metric_counts.to_dict()}")
|
| 1771 |
+
else:
|
| 1772 |
+
logger.warning("No 'metric_type' column found in APR data")
|
| 1773 |
+
|
| 1774 |
apr_data = df[df['metric_type'] == 'APR'].copy()
|
| 1775 |
+
logger.info(f"After metric_type == 'APR' filter: {len(apr_data)} records")
|
| 1776 |
+
logger.info(f"Unique agents after APR filter: {apr_data['agent_id'].nunique()}")
|
| 1777 |
+
logger.info(f"Agent IDs after APR filter: {sorted(apr_data['agent_id'].unique().tolist())}")
|
| 1778 |
|
| 1779 |
# Filter APR outliers (±200% range)
|
| 1780 |
before_outlier_filter = len(apr_data)
|
|
|
|
| 1932 |
y_values_ma = daily_medians_with_ma['moving_avg'].tolist()
|
| 1933 |
|
| 1934 |
# Create hover template for the APR moving average line
|
| 1935 |
+
# CONSISTENCY FIX: Use ROI daily agent data for active agent counts
|
| 1936 |
hover_data_apr = []
|
| 1937 |
for idx, row in daily_medians_with_ma.iterrows():
|
| 1938 |
timestamp = row['timestamp']
|
| 1939 |
# Format timestamp to show only date for daily data
|
| 1940 |
formatted_timestamp = timestamp.strftime('%Y-%m-%d')
|
| 1941 |
|
| 1942 |
+
# FIXED: Use ROI data to get consistent active agent counts
|
| 1943 |
+
# Load ROI data to get the correct agent counts
|
| 1944 |
+
try:
|
| 1945 |
+
df_roi_for_counts, _ = load_apr_data_from_csv()
|
| 1946 |
+
if not df_roi_for_counts.empty and 'roi' in df_roi_for_counts.columns:
|
| 1947 |
+
# Filter for ROI data and same date
|
| 1948 |
+
df_roi_filtered = df_roi_for_counts[
|
| 1949 |
+
(df_roi_for_counts['metric_type'] == 'APR') &
|
| 1950 |
+
(df_roi_for_counts['roi'].notna())
|
| 1951 |
+
].copy()
|
| 1952 |
+
|
| 1953 |
+
# Aggregate daily for ROI data
|
| 1954 |
+
roi_daily_agent_data = aggregate_daily_data(df_roi_filtered, 'roi')
|
| 1955 |
+
|
| 1956 |
+
# Get active agents from ROI data for this date
|
| 1957 |
+
active_agents = len(roi_daily_agent_data[roi_daily_agent_data['timestamp'] == timestamp]['agent_id'].unique())
|
| 1958 |
+
else:
|
| 1959 |
+
# Fallback to APR data if ROI not available
|
| 1960 |
+
active_agents = len(daily_agent_data[daily_agent_data['timestamp'] == timestamp]['agent_id'].unique())
|
| 1961 |
+
except:
|
| 1962 |
+
# Fallback to APR data if there's any error
|
| 1963 |
+
active_agents = len(daily_agent_data[daily_agent_data['timestamp'] == timestamp]['agent_id'].unique())
|
| 1964 |
+
|
| 1965 |
+
# DEBUG: Log agent counts for July 8th specifically
|
| 1966 |
+
if formatted_timestamp == '2025-07-08':
|
| 1967 |
+
logger.info(f"APR GRAPH - July 8th active agents (using ROI logic): {active_agents}")
|
| 1968 |
|
| 1969 |
hover_data_apr.append(
|
| 1970 |
f"Date: {formatted_timestamp}<br>Median APR (7d window): {row['moving_avg']:.2f}%<br>Active agents: {active_agents}"
|
|
|
|
| 1990 |
y_values_adj_ma = daily_medians_adjusted_with_ma['moving_avg'].tolist()
|
| 1991 |
|
| 1992 |
# Create hover template for the adjusted APR moving average line
|
| 1993 |
+
# CONSISTENCY FIX: Use ROI daily agent data for active agent counts (same as regular APR)
|
| 1994 |
hover_data_adj = []
|
| 1995 |
for idx, row in daily_medians_adjusted_with_ma.iterrows():
|
| 1996 |
timestamp = row['timestamp']
|
| 1997 |
# Format timestamp to show only date for daily data
|
| 1998 |
formatted_timestamp = timestamp.strftime('%Y-%m-%d')
|
| 1999 |
|
| 2000 |
+
# FIXED: Use ROI data to get consistent active agent counts (same logic as APR)
|
| 2001 |
+
try:
|
| 2002 |
+
df_roi_for_counts, _ = load_apr_data_from_csv()
|
| 2003 |
+
if not df_roi_for_counts.empty and 'roi' in df_roi_for_counts.columns:
|
| 2004 |
+
# Filter for ROI data and same date
|
| 2005 |
+
df_roi_filtered = df_roi_for_counts[
|
| 2006 |
+
(df_roi_for_counts['metric_type'] == 'APR') &
|
| 2007 |
+
(df_roi_for_counts['roi'].notna())
|
| 2008 |
+
].copy()
|
| 2009 |
+
|
| 2010 |
+
# Aggregate daily for ROI data
|
| 2011 |
+
roi_daily_agent_data = aggregate_daily_data(df_roi_filtered, 'roi')
|
| 2012 |
+
|
| 2013 |
+
# Get active agents from ROI data for this date
|
| 2014 |
+
active_agents = len(roi_daily_agent_data[roi_daily_agent_data['timestamp'] == timestamp]['agent_id'].unique())
|
| 2015 |
+
else:
|
| 2016 |
+
# Fallback to adjusted APR data if ROI not available
|
| 2017 |
+
active_agents = len(daily_agent_data_adjusted[daily_agent_data_adjusted['timestamp'] == timestamp]['agent_id'].unique()) if 'daily_agent_data_adjusted' in locals() else 0
|
| 2018 |
+
except:
|
| 2019 |
+
# Fallback to adjusted APR data if there's any error
|
| 2020 |
+
active_agents = len(daily_agent_data_adjusted[daily_agent_data_adjusted['timestamp'] == timestamp]['agent_id'].unique()) if 'daily_agent_data_adjusted' in locals() else 0
|
| 2021 |
+
|
| 2022 |
+
# DEBUG: Log agent counts for July 8th specifically
|
| 2023 |
+
if formatted_timestamp == '2025-07-08':
|
| 2024 |
+
logger.info(f"ADJUSTED APR GRAPH - July 8th active agents (using ROI logic): {active_agents}")
|
| 2025 |
|
| 2026 |
hover_data_adj.append(
|
| 2027 |
f"Date: {formatted_timestamp}<br>Median Adjusted APR (7d window): {row['moving_avg']:.2f}%<br>Active agents: {active_agents}"
|
|
|
|
| 2076 |
hovermode="closest"
|
| 2077 |
)
|
| 2078 |
|
| 2079 |
+
# Add single annotation for y-axis with proper spacing
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2080 |
fig.add_annotation(
|
| 2081 |
x=-0.08, # Position further from the y-axis to avoid overlapping with tick labels
|
| 2082 |
+
y=0, # Center of the y-axis
|
| 2083 |
xref="paper",
|
| 2084 |
yref="y",
|
| 2085 |
+
text="Percent drawdown (%) Agent APR (%)",
|
| 2086 |
showarrow=False,
|
| 2087 |
font=dict(size=16, family="Arial, sans-serif", color="black", weight="bold"), # Adjusted font size
|
| 2088 |
textangle=-90, # Rotate text to be vertical
|