added trace download links

.gitignore

@@ -5,3 +5,6 @@ evals_live/*
 evals_processed/*
 *.db
 .env
+encrypted_files/*
+evals_live_old/*
+evals_upload_old/*

app.py

@@ -527,7 +527,7 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
                             ci_metrics=["Accuracy", "Total Cost"]
                         ),
                         select_columns=SelectColumns(
-                            default_selection=config.APPWORLD_ON_LOAD_COLUMNS + ["Verified"],
+                            default_selection=config.APPWORLD_ON_LOAD_COLUMNS + ["Verified", "Traces"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
@@ -567,7 +567,7 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
                     Leaderboard(
                         value=create_leaderboard(parse_json_files(os.path.join(abs_path, "evals_live"), 'appworld_test_normal'), ci_metrics=["Accuracy", "Total Cost"]),
                         select_columns=SelectColumns(
-                            default_selection=config.APPWORLD_ON_LOAD_COLUMNS + ["Verified"],
+                            default_selection=config.APPWORLD_ON_LOAD_COLUMNS + ["Verified", "Traces"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
@@ -603,7 +603,7 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
                     Leaderboard(
                         value=create_leaderboard(parse_json_files(os.path.join(abs_path, "evals_live"), 'corebench_easy'), ci_metrics=["Accuracy", "Total Cost"]),
                         select_columns=SelectColumns(
-                            default_selection=config.COREBENCH_ON_LOAD_COLUMNS + ["Verified"],
+                            default_selection=config.COREBENCH_ON_LOAD_COLUMNS + ["Verified", "Traces"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
@@ -647,7 +647,7 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
                     Leaderboard(
                         value=create_leaderboard(parse_json_files(os.path.join(abs_path, "evals_live"), 'corebench_hard'), ci_metrics=["Accuracy", "Total Cost"]),
                         select_columns=SelectColumns(
-                            default_selection=config.COREBENCH_ON_LOAD_COLUMNS + ["Verified"],
+                            default_selection=config.COREBENCH_ON_LOAD_COLUMNS + ["Verified", "Traces"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
@@ -685,7 +685,7 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
                     Leaderboard(
                         value=create_leaderboard(parse_json_files(os.path.join(abs_path, "evals_live"), 'corebench_medium'), ci_metrics=["Accuracy", "Total Cost"]),
                         select_columns=SelectColumns(
-                            default_selection=config.COREBENCH_ON_LOAD_COLUMNS + ["Verified"],
+                            default_selection=config.COREBENCH_ON_LOAD_COLUMNS + ["Verified", "Traces"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
@@ -745,7 +745,7 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
                             ci_metrics=["Accuracy", "Total Cost"]
                         ),
                         select_columns=SelectColumns(
-                            default_selection=config.CYBENCH_ON_LOAD_COLUMNS + ["Verified"],
+                            default_selection=config.CYBENCH_ON_LOAD_COLUMNS + ["Verified", "Traces"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
@@ -839,7 +839,7 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
                             ci_metrics=["Accuracy", "Total Cost"]
                         ),
                         select_columns=SelectColumns(
-                            default_selection=config.GAIA_ON_LOAD_COLUMNS + ["Verified"],
+                            default_selection=config.GAIA_ON_LOAD_COLUMNS + ["Verified", "Traces"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
@@ -932,7 +932,7 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
                             ci_metrics=["Accuracy", "Total Cost"]
                         ),
                         select_columns=SelectColumns(
-                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS + ["Verified"],
+                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS + ["Verified", "Traces"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
@@ -1000,7 +1000,7 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
                     Leaderboard(
                         value=create_leaderboard(parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_verified_mini'), ci_metrics=["Accuracy", "Total Cost"]),
                         select_columns=SelectColumns(
-                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS + ["Verified"],
+                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS + ["Verified", "Traces"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
@@ -1033,7 +1033,7 @@ with gr.Blocks(theme=my_theme, css='css.css', title="HAL: Holistic Agent Leaderb
                     Leaderboard(
                         value=create_leaderboard(parse_json_files(os.path.join(abs_path, "evals_live"), 'usaco'), ci_metrics=["Accuracy", "Total Cost"]),
                         select_columns=SelectColumns(
-                            default_selection=config.USACO_ON_LOAD_COLUMNS + ["Verified"],
+                            default_selection=config.USACO_ON_LOAD_COLUMNS + ["Verified", "Traces"],
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),

utils/db.py

@@ -67,8 +67,10 @@ AGGREGATION_RULES = {
     'scenario_goal_completion': 'mean',
     'Verified': 'first',
     'Runs': 'first',
+    'Traces': 'first',
     'accuracy_ci': 'first',
     'cost_ci': 'first',
+    
 }
 
 # Define column display names
@@ -409,7 +411,13 @@ class TracePreprocessor:
                 df.loc[df['agent_name'] == agent_name, 'accuracy_ci'] = accuracy_ci
                 df.loc[df['agent_name'] == agent_name, 'cost_ci'] = cost_ci
 
-        df = df.drop(columns=['successful_tasks', 'failed_tasks', 'run_id'], axis=1)
+        # Before dropping run_id, create new column from it with download link
+        df['Traces'] = df['run_id'].apply(
+            lambda x: f'[load](https://huggingface.co/datasets/agent-evals/agent_traces/resolve/main/{x}.zip?download=true)'
+            if x else ''
+        )
+        
+        df = df.drop(columns=['successful_tasks', 'failed_tasks'], axis=1)
 
         if aggregate:
             df = df.groupby('agent_name').agg(AGGREGATION_RULES).reset_index()
@@ -540,10 +548,7 @@ class TracePreprocessor:
         
         # Drop temp column
         results_df = results_df.drop('agent_name_temp', axis=1)
-                
-        # Fill any missing costs with 0
-        # results_df['Total Cost'] = results_df['Total Cost'].fillna(0)
-        
+                    
         if aggregate:
             # Aggregate results
             results_df = results_df.groupby('Agent Name').agg({
@@ -567,6 +572,7 @@ class TracePreprocessor:
                 'Level 2 Accuracy': 'mean',
                 'Level 3 Accuracy': 'mean',
                 'Verified': 'first',
+                'Traces': 'first',
                 'Runs': 'first',
                 'Accuracy CI': 'first',
                 'Total Cost CI': 'first'

utils/processing.py

@@ -29,25 +29,10 @@ async def check_and_process_uploads():
         if not os.path.exists(live_path) and not os.path.exists(processed_path):
             unprocessed_uploads.append(upload)
         elif os.path.exists(processed_path):
-            # with open(upload_path, 'r') as f:
-            #     new_data = json.load(f)
-            
-            # with open(processed_path, 'r') as f:
-            #     processed_data = json.load(f)
-            
-            # TODO we can use a better comparison method with exact comparison
-            # if new_data != processed_data:
-            #     unprocessed_uploads.append(upload)
+
             print(f"Upload {upload} is already in processed directory.")
-        elif os.path.exists(live_path):
-            with open(upload_path, 'r') as f:
-                new_data = json.load(f)
-            
-            with open(live_path, 'r') as f:
-                live_data = json.load(f)
             
-            # if new_data != live_data:
-            #     unprocessed_uploads.append(upload)
+        elif os.path.exists(live_path):
             print(f"Upload {upload} is already in live directory.")
         else:
             unprocessed_uploads.append(upload)