Add 'Show only open models' checkbox filter

- Added checkbox to filter leaderboard table and scatter plots to show only open-source/open-weight models
- Checkbox shows count of open and closed models (e.g., '4 open, 5 closed')
- Filter works independently and in combination with 'Show incomplete entries' checkbox
- Filter applies to both Cost/Performance and Runtime/Performance scatter plots
- Updated timer/refresh logic to maintain filter state on data refresh

Co-authored-by: openhands <openhands@all-hands.dev>

ui_components.py

@@ -582,8 +582,18 @@ def create_leaderboard_display(
     else:
         df_view_complete = df_view_full.copy()
     
+    # Prepare open-only filtered versions (filter before prepare_df_for_display drops Openness column)
+    if 'Openness' in df_view_full.columns:
+        df_view_open = df_view_full[df_view_full['Openness'].str.lower() == 'open'].copy()
+        df_view_complete_open = df_view_complete[df_view_complete['Openness'].str.lower() == 'open'].copy()
+    else:
+        df_view_open = df_view_full.copy()
+        df_view_complete_open = df_view_complete.copy()
+    
     df_display_complete = prepare_df_for_display(df_view_complete)
     df_display_all = prepare_df_for_display(df_view_full)
+    df_display_open = prepare_df_for_display(df_view_open)
+    df_display_complete_open = prepare_df_for_display(df_view_complete_open)
     
     # If no complete entries exist, show all entries by default
     has_complete_entries = len(df_display_complete) > 0
@@ -620,13 +630,17 @@ def create_leaderboard_display(
             plot_type='runtime'
         )
     
-    # Create initial cost scatter plots for both complete and all data
+    # Create initial cost scatter plots for all filter combinations
     cost_scatter_complete = create_cost_scatter_plot(df_view_complete) if has_complete_entries else go.Figure()
     cost_scatter_all = create_cost_scatter_plot(df_view_full)
+    cost_scatter_open = create_cost_scatter_plot(df_view_open) if len(df_view_open) > 0 else go.Figure()
+    cost_scatter_complete_open = create_cost_scatter_plot(df_view_complete_open) if len(df_view_complete_open) > 0 else go.Figure()
     
-    # Create initial runtime scatter plots for both complete and all data
+    # Create initial runtime scatter plots for all filter combinations
     runtime_scatter_complete = create_runtime_scatter_plot(df_view_complete) if has_complete_entries else go.Figure()
     runtime_scatter_all = create_runtime_scatter_plot(df_view_full)
+    runtime_scatter_open = create_runtime_scatter_plot(df_view_open) if len(df_view_open) > 0 else go.Figure()
+    runtime_scatter_complete_open = create_runtime_scatter_plot(df_view_complete_open) if len(df_view_complete_open) > 0 else go.Figure()
     
     # Now get headers from the renamed dataframe (use all entries to ensure headers are present)
     df_headers = df_display_all.columns.tolist()
@@ -651,12 +665,14 @@ def create_leaderboard_display(
     # 5. Combine all the lists to create the final, fully dynamic list.
     final_column_widths = fixed_start_widths + dynamic_widths + fixed_end_widths
 
-    # Calculate counts for the checkbox label
+    # Calculate counts for the checkbox labels
     num_complete = len(df_display_complete)
     num_total = len(df_display_all)
     num_incomplete = num_total - num_complete
+    num_open = len(df_display_open)
+    num_closed = num_total - num_open
     
-    # Add toggle for showing incomplete entries ABOVE the plot
+    # Add toggle checkboxes ABOVE the plot
     if has_complete_entries:
         show_incomplete_checkbox = gr.Checkbox(
             label=f"Show incomplete entries ({num_incomplete} entries with fewer than 5 categories)",
@@ -666,6 +682,16 @@ def create_leaderboard_display(
     else:
         show_incomplete_checkbox = None
         gr.Markdown(f"*No entries with all 5 categories completed yet. Showing all {num_total} entries.*")
+    
+    # Add checkbox for open models only (always show this if there are open models)
+    if num_open > 0 and num_closed > 0:
+        show_open_only_checkbox = gr.Checkbox(
+            label=f"Show only open models ({num_open} open, {num_closed} closed)",
+            value=False,
+            elem_id="show-open-only-toggle"
+        )
+    else:
+        show_open_only_checkbox = None
 
     # Plot components - show complete entries by default if available
     # Cost/Performance plot
@@ -700,18 +726,46 @@ def create_leaderboard_display(
                 elem_id="main-leaderboard"
             )
             
-            # Update function for the toggle - updates both table and plots
-            def update_display(show_incomplete):
-                if show_incomplete:
-                    return df_display_all, cost_scatter_all, runtime_scatter_all
+            # Update function for filters - handles both checkboxes
+            def update_display(show_incomplete, show_open_only):
+                # Determine which data to show based on checkbox states
+                if show_open_only:
+                    if show_incomplete:
+                        return df_display_open, cost_scatter_open, runtime_scatter_open
+                    else:
+                        return df_display_complete_open, cost_scatter_complete_open, runtime_scatter_complete_open
                 else:
-                    return df_display_complete, cost_scatter_complete, runtime_scatter_complete
+                    if show_incomplete:
+                        return df_display_all, cost_scatter_all, runtime_scatter_all
+                    else:
+                        return df_display_complete, cost_scatter_complete, runtime_scatter_complete
             
-            show_incomplete_checkbox.change(
-                fn=update_display,
-                inputs=[show_incomplete_checkbox],
-                outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
-            )
+            # Connect both checkboxes to the update function
+            checkbox_inputs = [show_incomplete_checkbox]
+            if show_open_only_checkbox is not None:
+                checkbox_inputs.append(show_open_only_checkbox)
+                show_incomplete_checkbox.change(
+                    fn=update_display,
+                    inputs=checkbox_inputs,
+                    outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
+                )
+                show_open_only_checkbox.change(
+                    fn=update_display,
+                    inputs=checkbox_inputs,
+                    outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
+                )
+            else:
+                # No open only checkbox, use simplified version
+                def update_display_simple(show_incomplete):
+                    if show_incomplete:
+                        return df_display_all, cost_scatter_all, runtime_scatter_all
+                    else:
+                        return df_display_complete, cost_scatter_complete, runtime_scatter_complete
+                show_incomplete_checkbox.change(
+                    fn=update_display_simple,
+                    inputs=[show_incomplete_checkbox],
+                    outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
+                )
         else:
             dataframe_component = gr.DataFrame(
                 headers=df_headers,
@@ -724,13 +778,27 @@ def create_leaderboard_display(
                 show_search="search",
                 elem_id="main-leaderboard"
             )
+            
+            # Even without incomplete checkbox, we may have open_only checkbox
+            if show_open_only_checkbox is not None:
+                def update_display_open_only(show_open_only):
+                    if show_open_only:
+                        return df_display_open, cost_scatter_open, runtime_scatter_open
+                    else:
+                        return df_display_all, cost_scatter_all, runtime_scatter_all
+                
+                show_open_only_checkbox.change(
+                    fn=update_display_open_only,
+                    inputs=[show_open_only_checkbox],
+                    outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
+                )
         
         legend_markdown = create_legend_markdown(category_name)
         gr.HTML(value=legend_markdown, elem_id="legend-markdown")
     
     # Add a timer to periodically check for data updates and refresh the UI
     # This runs every 60 seconds to check if new data is available
-    def check_and_refresh_data(current_checkbox_state):
+    def check_and_refresh_data(show_incomplete, show_open_only=False):
         """Check if data has been refreshed and return updated data if so."""
         current_version = get_data_version()
         if current_version > initial_data_version:
@@ -747,40 +815,70 @@ def create_leaderboard_display(
                 else:
                     new_df_view_complete = new_df_view_full.copy()
                 
+                # Prepare open-only versions
+                if 'Openness' in new_df_view_full.columns:
+                    new_df_view_open = new_df_view_full[new_df_view_full['Openness'].str.lower() == 'open'].copy()
+                    new_df_view_complete_open = new_df_view_complete[new_df_view_complete['Openness'].str.lower() == 'open'].copy()
+                else:
+                    new_df_view_open = new_df_view_full.copy()
+                    new_df_view_complete_open = new_df_view_complete.copy()
+                
                 new_df_display_complete = prepare_df_for_display(new_df_view_complete)
                 new_df_display_all = prepare_df_for_display(new_df_view_full)
+                new_df_display_open = prepare_df_for_display(new_df_view_open)
+                new_df_display_complete_open = prepare_df_for_display(new_df_view_complete_open)
                 
-                # Create new scatter plots (both cost and runtime)
+                # Create new scatter plots for all combinations
                 new_cost_scatter_complete = create_cost_scatter_plot(new_df_view_complete) if len(new_df_display_complete) > 0 else go.Figure()
                 new_cost_scatter_all = create_cost_scatter_plot(new_df_view_full)
+                new_cost_scatter_open = create_cost_scatter_plot(new_df_view_open) if len(new_df_view_open) > 0 else go.Figure()
+                new_cost_scatter_complete_open = create_cost_scatter_plot(new_df_view_complete_open) if len(new_df_view_complete_open) > 0 else go.Figure()
+                
                 new_runtime_scatter_complete = create_runtime_scatter_plot(new_df_view_complete) if len(new_df_display_complete) > 0 else go.Figure()
                 new_runtime_scatter_all = create_runtime_scatter_plot(new_df_view_full)
+                new_runtime_scatter_open = create_runtime_scatter_plot(new_df_view_open) if len(new_df_view_open) > 0 else go.Figure()
+                new_runtime_scatter_complete_open = create_runtime_scatter_plot(new_df_view_complete_open) if len(new_df_view_complete_open) > 0 else go.Figure()
                 
-                # Return the appropriate data based on checkbox state
-                if current_checkbox_state:
-                    return new_df_display_all, new_cost_scatter_all, new_runtime_scatter_all
+                # Return the appropriate data based on checkbox states
+                if show_open_only:
+                    if show_incomplete:
+                        return new_df_display_open, new_cost_scatter_open, new_runtime_scatter_open
+                    else:
+                        return new_df_display_complete_open, new_cost_scatter_complete_open, new_runtime_scatter_complete_open
                 else:
-                    return new_df_display_complete, new_cost_scatter_complete, new_runtime_scatter_complete
+                    if show_incomplete:
+                        return new_df_display_all, new_cost_scatter_all, new_runtime_scatter_all
+                    else:
+                        return new_df_display_complete, new_cost_scatter_complete, new_runtime_scatter_complete
         
-        # No change, return current values
-        if current_checkbox_state:
-            return df_display_all, cost_scatter_all, runtime_scatter_all
+        # No change, return current values based on checkbox states
+        if show_open_only:
+            if show_incomplete:
+                return df_display_open, cost_scatter_open, runtime_scatter_open
+            else:
+                return df_display_complete_open, cost_scatter_complete_open, runtime_scatter_complete_open
         else:
-            return df_display_complete, cost_scatter_complete, runtime_scatter_complete
+            if show_incomplete:
+                return df_display_all, cost_scatter_all, runtime_scatter_all
+            else:
+                return df_display_complete, cost_scatter_complete, runtime_scatter_complete
     
     # Create a timer that checks for updates every 60 seconds
     refresh_timer = gr.Timer(value=60)
     
     # Connect the timer to the refresh function
     if show_incomplete_checkbox is not None:
+        timer_inputs = [show_incomplete_checkbox]
+        if show_open_only_checkbox is not None:
+            timer_inputs.append(show_open_only_checkbox)
         refresh_timer.tick(
             fn=check_and_refresh_data,
-            inputs=[show_incomplete_checkbox],
+            inputs=timer_inputs,
             outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
         )
     else:
-        # If no checkbox, always show all data
-        def check_and_refresh_all():
+        # If no incomplete checkbox, always show all data (but still filter by open if needed)
+        def check_and_refresh_all(show_open_only=False):
             current_version = get_data_version()
             if current_version > initial_data_version:
                 print(f"[REFRESH] Data version changed, reloading...")
@@ -788,17 +886,33 @@ def create_leaderboard_display(
                 if not new_df.empty:
                     new_transformer = DataTransformer(new_df, new_tag_map)
                     new_df_view_full, _ = new_transformer.view(tag=category_name, use_plotly=True)
+                    
+                    if show_open_only and 'Openness' in new_df_view_full.columns:
+                        new_df_view_full = new_df_view_full[new_df_view_full['Openness'].str.lower() == 'open'].copy()
+                    
                     new_df_display_all = prepare_df_for_display(new_df_view_full)
                     new_cost_scatter_all = create_cost_scatter_plot(new_df_view_full)
                     new_runtime_scatter_all = create_runtime_scatter_plot(new_df_view_full)
                     return new_df_display_all, new_cost_scatter_all, new_runtime_scatter_all
+            
+            if show_open_only:
+                return df_display_open, cost_scatter_open, runtime_scatter_open
             return df_display_all, cost_scatter_all, runtime_scatter_all
         
-        refresh_timer.tick(
-            fn=check_and_refresh_all,
-            inputs=[],
-            outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
-        )
+        if show_open_only_checkbox is not None:
+            refresh_timer.tick(
+                fn=check_and_refresh_all,
+                inputs=[show_open_only_checkbox],
+                outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
+            )
+        else:
+            def check_and_refresh_simple():
+                return check_and_refresh_all(False)
+            refresh_timer.tick(
+                fn=check_and_refresh_simple,
+                inputs=[],
+                outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
+            )
 
     # Return the components so they can be referenced elsewhere.
     return cost_plot_component, runtime_plot_component, dataframe_component