Add 'Mark systems by' selector for scatter plot icons (Company/Openness/Country)

- Add MARK_BY_CHOICES and MARK_BY_DEFAULT constants
- Add country flag SVGs (US, China, France)
- Add COUNTRY_FLAG_MAP and MODEL_COUNTRY_MAP for country detection
- Add get_openness_icon() and get_country_from_model() functions
- Add get_marker_icon() to select appropriate icon based on mark_by setting
- Modify _plot_scatter_plotly() to accept mark_by parameter
- Add mark_by_dropdown to UI in ui_components.py
- Update update_display() to regenerate plots dynamically based on mark_by selection

Co-authored-by: openhands <openhands@all-hands.dev>

constants.py

@@ -3,3 +3,10 @@
 # Font settings
 FONT_FAMILY = "Arial, sans-serif"
 FONT_FAMILY_SHORT = "Arial"  # For places that don't accept fallbacks
+
+# Marker options for plot icons
+MARK_BY_COMPANY = "Company"
+MARK_BY_OPENNESS = "Openness"
+MARK_BY_COUNTRY = "Country"
+MARK_BY_CHOICES = [MARK_BY_COMPANY, MARK_BY_OPENNESS, MARK_BY_COUNTRY]
+MARK_BY_DEFAULT = MARK_BY_COMPANY

leaderboard_transformer.py

@@ -40,6 +40,37 @@ COMPANY_LOGO_MAP = {
     "minimax": {"path": "assets/logo-minimax.svg", "name": "MiniMax"},
 }
 
+# Openness icon mapping
+OPENNESS_ICON_MAP = {
+    "open": {"path": "assets/lock-open.svg", "name": "Open"},
+    "closed": {"path": "assets/lock-closed.svg", "name": "Closed"},
+}
+
+# Country flag mapping - maps model name patterns to country flags
+COUNTRY_FLAG_MAP = {
+    "us": {"path": "assets/flag-us.svg", "name": "United States"},
+    "cn": {"path": "assets/flag-cn.svg", "name": "China"},
+    "fr": {"path": "assets/flag-fr.svg", "name": "France"},
+}
+
+# Model to country mapping (based on company headquarters)
+MODEL_COUNTRY_MAP = {
+    # US companies
+    "anthropic": "us", "claude": "us",
+    "openai": "us", "gpt": "us", "o1": "us", "o3": "us",
+    "google": "us", "gemini": "us", "gemma": "us",
+    "meta": "us", "llama": "us",
+    "xai": "us", "grok": "us",
+    "cohere": "us", "command": "us",
+    # China companies
+    "deepseek": "cn",
+    "qwen": "cn", "alibaba": "cn",
+    "kimi": "cn", "moonshot": "cn",
+    "minimax": "cn",
+    # France companies
+    "mistral": "fr", "mixtral": "fr", "codestral": "fr",
+}
+
 # OpenHands branding constants
 OPENHANDS_LOGO_PATH_LIGHT = "assets/openhands_logo_color_forwhite.png"
 OPENHANDS_LOGO_PATH_DARK = "assets/openhands_logo_color_forblack.png"
@@ -150,6 +181,66 @@ def get_company_from_model(model_name: str) -> dict:
 
     return {"path": "assets/logo-unknown.svg", "name": "Unknown"}
 
+
+def get_openness_icon(openness: str) -> dict:
+    """
+    Gets the openness icon info (path and name) from openness value.
+    Returns closed icon as default.
+    """
+    if not openness:
+        return OPENNESS_ICON_MAP["closed"]
+    
+    openness_lower = str(openness).lower()
+    if openness_lower in OPENNESS_ICON_MAP:
+        return OPENNESS_ICON_MAP[openness_lower]
+    
+    return OPENNESS_ICON_MAP["closed"]
+
+
+def get_country_from_model(model_name: str) -> dict:
+    """
+    Gets the country flag info (path and name) from a model name.
+    Returns US flag as default.
+    """
+    if not model_name:
+        return COUNTRY_FLAG_MAP["us"]
+
+    # Handle list of models - use the first one
+    if isinstance(model_name, list):
+        model_name = model_name[0] if model_name else ""
+
+    model_lower = str(model_name).lower()
+
+    # Check each pattern
+    for pattern, country_code in MODEL_COUNTRY_MAP.items():
+        if pattern in model_lower:
+            return COUNTRY_FLAG_MAP.get(country_code, COUNTRY_FLAG_MAP["us"])
+
+    return COUNTRY_FLAG_MAP["us"]
+
+
+def get_marker_icon(model_name: str, openness: str, mark_by: str) -> dict:
+    """
+    Gets the appropriate icon based on the mark_by selection.
+    
+    Args:
+        model_name: The model name
+        openness: The openness value (open/closed)
+        mark_by: One of "Company", "Openness", or "Country"
+    
+    Returns:
+        dict with 'path' and 'name' keys
+    """
+    from constants import MARK_BY_COMPANY, MARK_BY_OPENNESS, MARK_BY_COUNTRY
+    
+    if mark_by == MARK_BY_OPENNESS:
+        return get_openness_icon(openness)
+    elif mark_by == MARK_BY_COUNTRY:
+        return get_country_from_model(model_name)
+    else:  # Default to company
+        return get_company_from_model(model_name)
+
+
 INFORMAL_TO_FORMAL_NAME_MAP = {
     # Short Names
     "lit": "Literature Understanding",
@@ -489,8 +580,12 @@ def _plot_scatter_plotly(
         y: str,
         agent_col: str = 'Agent',
         name: Optional[str] = None,
-        plot_type: str = 'cost'  # 'cost' or 'runtime'
+        plot_type: str = 'cost',  # 'cost' or 'runtime'
+        mark_by: Optional[str] = None  # 'Company', 'Openness', or 'Country'
 ) -> go.Figure:
+    from constants import MARK_BY_DEFAULT
+    if mark_by is None:
+        mark_by = MARK_BY_DEFAULT
 
     # --- Section 1: Define Mappings ---
     # Map openness to colors (simplified: open vs closed)
@@ -697,8 +792,9 @@ def _plot_scatter_plotly(
     
     for _, row in data_plot.iterrows():
         model_name = row.get('Language Model', '')
-        company_info = get_company_from_model(model_name)
-        logo_path = company_info['path']
+        openness = row.get('Openness', '')
+        marker_info = get_marker_icon(model_name, openness, mark_by)
+        logo_path = marker_info['path']
         
         # Read the SVG file and encode as base64 data URI
         if os.path.exists(logo_path):

main_page.py

@@ -36,8 +36,8 @@ def build_page():
 
     test_df, test_tag_map = get_full_leaderboard_data("test")
     if not test_df.empty:
-        # Get the checkbox returned from create_leaderboard_display
-        show_open_only_checkbox = create_leaderboard_display(
+        # Get the checkbox and dropdown returned from create_leaderboard_display
+        show_open_only_checkbox, mark_by_dropdown = create_leaderboard_display(
             full_df=test_df,
             tag_map=test_tag_map,
             category_name=CATEGORY_NAME,

ui_components.py

@@ -8,6 +8,7 @@ import re
 from huggingface_hub import HfApi
 
 import aliases
+from constants import MARK_BY_CHOICES, MARK_BY_DEFAULT
 from simple_data_loader import SimpleLeaderboardViewer
 from leaderboard_transformer import (
     DataTransformer,
@@ -688,25 +689,27 @@ def create_leaderboard_display(
         primary_runtime_col = f"{category_name} Runtime"
     
     # Function to create cost/performance scatter plot from data
-    def create_cost_scatter_plot(df_data):
+    def create_cost_scatter_plot(df_data, mark_by=MARK_BY_DEFAULT):
         return _plot_scatter_plotly(
             data=df_data,
             x=primary_cost_col if primary_cost_col in df_data.columns else None,
             y=primary_score_col if primary_score_col in df_data.columns else "Average Score",
             agent_col="SDK Version",
             name=category_name,
-            plot_type='cost'
+            plot_type='cost',
+            mark_by=mark_by
         )
     
     # Function to create runtime/performance scatter plot from data
-    def create_runtime_scatter_plot(df_data):
+    def create_runtime_scatter_plot(df_data, mark_by=MARK_BY_DEFAULT):
         return _plot_scatter_plotly(
             data=df_data,
             x=primary_runtime_col if primary_runtime_col in df_data.columns else None,
             y=primary_score_col if primary_score_col in df_data.columns else "Average Score",
             agent_col="SDK Version",
             name=category_name,
-            plot_type='runtime'
+            plot_type='runtime',
+            mark_by=mark_by
         )
     
     # Create initial cost scatter plots for all filter combinations
@@ -751,26 +754,36 @@ def create_leaderboard_display(
     num_open = len(df_display_open)
     num_closed = num_total - num_open
     
-    # Add toggle checkboxes ABOVE the plot
-    if has_complete_entries:
-        show_incomplete_checkbox = gr.Checkbox(
-            label=f"Show incomplete entries ({num_incomplete} entries with fewer than 5 categories)",
-            value=False,
-            elem_id="show-incomplete-toggle"
-        )
-    else:
-        show_incomplete_checkbox = None
-        gr.Markdown(f"*No entries with all 5 categories completed yet. Showing all {num_total} entries.*")
-    
-    # Add checkbox for open models only (always show this if there are open models)
-    if num_open > 0 and num_closed > 0:
-        show_open_only_checkbox = gr.Checkbox(
-            label=f"Show only open models ({num_open} open, {num_closed} closed)",
-            value=False,
-            elem_id="show-open-only-toggle"
-        )
-    else:
-        show_open_only_checkbox = None
+    # Add toggle checkboxes and dropdown ABOVE the plot
+    with gr.Row():
+        with gr.Column(scale=3):
+            if has_complete_entries:
+                show_incomplete_checkbox = gr.Checkbox(
+                    label=f"Show incomplete entries ({num_incomplete} entries with fewer than 5 categories)",
+                    value=False,
+                    elem_id="show-incomplete-toggle"
+                )
+            else:
+                show_incomplete_checkbox = None
+                gr.Markdown(f"*No entries with all 5 categories completed yet. Showing all {num_total} entries.*")
+            
+            # Add checkbox for open models only (always show this if there are open models)
+            if num_open > 0 and num_closed > 0:
+                show_open_only_checkbox = gr.Checkbox(
+                    label=f"Show only open models ({num_open} open, {num_closed} closed)",
+                    value=False,
+                    elem_id="show-open-only-toggle"
+                )
+            else:
+                show_open_only_checkbox = None
+        
+        with gr.Column(scale=1):
+            mark_by_dropdown = gr.Dropdown(
+                choices=MARK_BY_CHOICES,
+                value=MARK_BY_DEFAULT,
+                label="Mark systems by",
+                elem_id="mark-by-dropdown"
+            )
 
     # Plot components - show complete entries by default if available
     # Cost/Performance plot
@@ -805,46 +818,46 @@ def create_leaderboard_display(
                 elem_id="main-leaderboard"
             )
             
-            # Update function for filters - handles both checkboxes
-            def update_display(show_incomplete, show_open_only):
-                # Determine which data to show based on checkbox states
+            # Update function for filters - handles checkboxes and mark_by dropdown
+            def update_display(show_incomplete, show_open_only, mark_by):
+                # Determine which dataframe to show based on checkbox states
                 if show_open_only:
-                    if show_incomplete:
-                        return df_display_open, cost_scatter_open, runtime_scatter_open
-                    else:
-                        return df_display_complete_open, cost_scatter_complete_open, runtime_scatter_complete_open
+                    df_to_show = df_display_open if show_incomplete else df_display_complete_open
+                    view_df = df_view_open if show_incomplete else df_view_complete_open
                 else:
-                    if show_incomplete:
-                        return df_display_all, cost_scatter_all, runtime_scatter_all
-                    else:
-                        return df_display_complete, cost_scatter_complete, runtime_scatter_complete
+                    df_to_show = df_display_all if show_incomplete else df_display_complete
+                    view_df = df_view_full if show_incomplete else df_view_complete
+                
+                # Regenerate plots with current mark_by setting
+                cost_plot = create_cost_scatter_plot(view_df, mark_by)
+                runtime_plot = create_runtime_scatter_plot(view_df, mark_by)
+                return df_to_show, cost_plot, runtime_plot
             
-            # Connect both checkboxes to the update function
-            checkbox_inputs = [show_incomplete_checkbox]
+            # Connect checkboxes and dropdown to the update function
+            filter_inputs = [show_incomplete_checkbox]
+            if show_open_only_checkbox is not None:
+                filter_inputs.append(show_open_only_checkbox)
+            else:
+                # Add a dummy value for show_open_only when checkbox doesn't exist
+                filter_inputs = [show_incomplete_checkbox, gr.State(value=False)]
+            filter_inputs.append(mark_by_dropdown)
+            
+            show_incomplete_checkbox.change(
+                fn=update_display,
+                inputs=filter_inputs,
+                outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
+            )
             if show_open_only_checkbox is not None:
-                checkbox_inputs.append(show_open_only_checkbox)
-                show_incomplete_checkbox.change(
-                    fn=update_display,
-                    inputs=checkbox_inputs,
-                    outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
-                )
                 show_open_only_checkbox.change(
                     fn=update_display,
-                    inputs=checkbox_inputs,
-                    outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
-                )
-            else:
-                # No open only checkbox, use simplified version
-                def update_display_simple(show_incomplete):
-                    if show_incomplete:
-                        return df_display_all, cost_scatter_all, runtime_scatter_all
-                    else:
-                        return df_display_complete, cost_scatter_complete, runtime_scatter_complete
-                show_incomplete_checkbox.change(
-                    fn=update_display_simple,
-                    inputs=[show_incomplete_checkbox],
+                    inputs=filter_inputs,
                     outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
                 )
+            mark_by_dropdown.change(
+                fn=update_display,
+                inputs=filter_inputs,
+                outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
+            )
         else:
             dataframe_component = gr.DataFrame(
                 headers=df_headers,
@@ -858,19 +871,36 @@ def create_leaderboard_display(
                 elem_id="main-leaderboard"
             )
             
-            # Even without incomplete checkbox, we may have open_only checkbox
+            # Update function for mark_by and optional open_only checkbox
+            def update_display_no_complete(show_open_only, mark_by):
+                if show_open_only:
+                    df_to_show = df_display_open
+                    view_df = df_view_open
+                else:
+                    df_to_show = df_display_all
+                    view_df = df_view_full
+                cost_plot = create_cost_scatter_plot(view_df, mark_by)
+                runtime_plot = create_runtime_scatter_plot(view_df, mark_by)
+                return df_to_show, cost_plot, runtime_plot
+            
+            filter_inputs_no_complete = []
+            if show_open_only_checkbox is not None:
+                filter_inputs_no_complete.append(show_open_only_checkbox)
+            else:
+                filter_inputs_no_complete.append(gr.State(value=False))
+            filter_inputs_no_complete.append(mark_by_dropdown)
+            
             if show_open_only_checkbox is not None:
-                def update_display_open_only(show_open_only):
-                    if show_open_only:
-                        return df_display_open, cost_scatter_open, runtime_scatter_open
-                    else:
-                        return df_display_all, cost_scatter_all, runtime_scatter_all
-                
                 show_open_only_checkbox.change(
-                    fn=update_display_open_only,
-                    inputs=[show_open_only_checkbox],
+                    fn=update_display_no_complete,
+                    inputs=filter_inputs_no_complete,
                     outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
                 )
+            mark_by_dropdown.change(
+                fn=update_display_no_complete,
+                inputs=filter_inputs_no_complete,
+                outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
+            )
         
         legend_markdown = create_legend_markdown(category_name)
         gr.HTML(value=legend_markdown, elem_id="legend-markdown")
@@ -993,8 +1023,8 @@ def create_leaderboard_display(
                 outputs=[dataframe_component, cost_plot_component, runtime_plot_component]
             )
 
-    # Return the show_open_only_checkbox so it can be used to update other sections
-    return show_open_only_checkbox
+    # Return the show_open_only_checkbox and mark_by_dropdown so they can be used to update other sections
+    return show_open_only_checkbox, mark_by_dropdown
 
 # # --- Detailed Benchmark Display ---
 def create_benchmark_details_display(