Add Evolution Over Time and Open Model Accuracy by Size visualizations

- Add visualizations.py with two new chart functions:
- create_evolution_over_time_chart: Line chart showing model performance evolution over release dates
- create_accuracy_by_size_chart: Scatter plot showing accuracy vs parameter count for open-weights models
- Update simple_data_loader.py to load new metadata fields (release_date, parameter_count_b, active_parameter_count_b)
- Update main_page.py to display the new visualizations below the leaderboard
- Update mock data with release dates and add sample open-weights models with parameter counts

These visualizations will display data once the openhands-index-results PR with release_date and parameter_count fields is merged.

Co-authored-by: openhands <openhands@all-hands.dev>

main_page.py

@@ -11,6 +11,11 @@ from content import (
     INTRO_PARAGRAPH
 )
 
+from visualizations import (
+    create_evolution_over_time_chart,
+    create_accuracy_by_size_chart
+)
+
 # --- Global State for Viewers (simple caching) ---
 CACHED_VIEWERS = {}
 CACHED_TAG_MAPS = {}
@@ -33,6 +38,26 @@ def build_page():
             category_name=CATEGORY_NAME,
             split_name="test"
         )
+        
+        # --- New Visualization Sections ---
+        gr.Markdown("---")
+        
+        # Evolution Over Time Section
+        gr.HTML('<h2>Evolution Over Time</h2>', elem_id="evolution-header")
+        gr.Markdown("Track how model performance has improved over time based on release dates.")
+        
+        evolution_fig = create_evolution_over_time_chart(test_df)
+        gr.Plot(value=evolution_fig, elem_id="evolution-chart")
+        
+        gr.Markdown("---")
+        
+        # Open Model Accuracy by Size Section
+        gr.HTML('<h2>Open Model Accuracy by Size</h2>', elem_id="size-accuracy-header")
+        gr.Markdown("Compare open-weights model performance against their parameter count. Marker size indicates total parameters; x-axis shows active parameters (relevant for MoE models).")
+        
+        size_fig = create_accuracy_by_size_chart(test_df)
+        gr.Plot(value=size_fig, elem_id="size-accuracy-chart")
+        
     else:
         gr.Markdown("No data available.")
 

mock_results/1.0.0-dev1/results/20250723_qwen3_coder/metadata.json

@@ -0,0 +1,11 @@
+{
+  "agent_version": "1.0.2",
+  "model": "qwen-3-coder",
+  "openness": "open_weights",
+  "tool_usage": "standard",
+  "submission_time": "2025-07-23T10:00:00.000000",
+  "directory_name": "20250723_qwen3_coder",
+  "release_date": "2025-07-23",
+  "parameter_count_b": 480,
+  "active_parameter_count_b": 35
+}

mock_results/1.0.0-dev1/results/20250723_qwen3_coder/scores.json

@@ -0,0 +1,16 @@
+[
+  {
+    "benchmark": "swe-bench",
+    "score": 38.0,
+    "metric": "resolve_rate",
+    "cost_per_instance": 0.12,
+    "average_runtime": 150
+  },
+  {
+    "benchmark": "gaia",
+    "score": 48.0,
+    "metric": "accuracy",
+    "cost_per_instance": 0.06,
+    "average_runtime": 45
+  }
+]

mock_results/1.0.0-dev1/results/20251106_kimi_k2_thinking/metadata.json

@@ -0,0 +1,11 @@
+{
+  "agent_version": "1.0.2",
+  "model": "kimi-k2-thinking",
+  "openness": "open_weights",
+  "tool_usage": "standard",
+  "submission_time": "2025-11-06T10:00:00.000000",
+  "directory_name": "20251106_kimi_k2_thinking",
+  "release_date": "2025-11-06",
+  "parameter_count_b": 1000,
+  "active_parameter_count_b": 32
+}

mock_results/1.0.0-dev1/results/20251106_kimi_k2_thinking/scores.json

@@ -0,0 +1,16 @@
+[
+  {
+    "benchmark": "swe-bench",
+    "score": 45.0,
+    "metric": "resolve_rate",
+    "cost_per_instance": 0.18,
+    "average_runtime": 200
+  },
+  {
+    "benchmark": "gaia",
+    "score": 52.0,
+    "metric": "accuracy",
+    "cost_per_instance": 0.10,
+    "average_runtime": 70
+  }
+]

mock_results/1.0.0-dev1/results/20251124_claude_3_5_sonnet_20241022/metadata.json

@@ -4,5 +4,6 @@
   "openness": "closed_api_available",
   "tool_usage": "standard",
   "submission_time": "2025-11-24T19:56:00.092865",
-  "directory_name": "20251124_claude_3_5_sonnet_20241022"
-}
+  "directory_name": "20251124_claude_3_5_sonnet_20241022",
+  "release_date": "2024-10-22"
+}

mock_results/1.0.0-dev1/results/20251124_claude_3_opus_20240229/metadata.json

@@ -4,5 +4,6 @@
   "openness": "closed_api_available",
   "tool_usage": "custom_interface",
   "submission_time": "2025-11-24T19:56:00.092922",
-  "directory_name": "20251124_claude_3_opus_20240229"
-}
+  "directory_name": "20251124_claude_3_opus_20240229",
+  "release_date": "2024-02-29"
+}

mock_results/1.0.0-dev1/results/20251124_gpt_4_turbo_2024_04_09/metadata.json

@@ -4,5 +4,6 @@
   "openness": "closed_api_available",
   "tool_usage": "standard",
   "submission_time": "2025-11-24T19:56:00.092908",
-  "directory_name": "20251124_gpt_4_turbo_2024_04_09"
-}
+  "directory_name": "20251124_gpt_4_turbo_2024_04_09",
+  "release_date": "2024-04-09"
+}

mock_results/1.0.0-dev1/results/20251124_gpt_4o_2024_11_20/metadata.json

@@ -4,5 +4,6 @@
   "openness": "closed_api_available",
   "tool_usage": "standard",
   "submission_time": "2025-11-24T19:56:00.092895",
-  "directory_name": "20251124_gpt_4o_2024_11_20"
-}
+  "directory_name": "20251124_gpt_4o_2024_11_20",
+  "release_date": "2024-11-20"
+}

mock_results/1.0.0-dev1/results/20251124_gpt_4o_mini_2024_07_18/metadata.json

@@ -4,5 +4,6 @@
   "openness": "closed_api_available",
   "tool_usage": "standard",
   "submission_time": "2025-11-24T19:56:00.092916",
-  "directory_name": "20251124_gpt_4o_mini_2024_07_18"
-}
+  "directory_name": "20251124_gpt_4o_mini_2024_07_18",
+  "release_date": "2024-07-18"
+}

mock_results/1.0.0-dev1/results/20251201_deepseek_v3/metadata.json

@@ -0,0 +1,10 @@
+{
+  "agent_version": "1.0.2",
+  "model": "deepseek-v3",
+  "openness": "open_weights",
+  "tool_usage": "standard",
+  "submission_time": "2025-12-01T10:00:00.000000",
+  "directory_name": "20251201_deepseek_v3",
+  "release_date": "2025-12-01",
+  "parameter_count_b": 685
+}

mock_results/1.0.0-dev1/results/20251201_deepseek_v3/scores.json

@@ -0,0 +1,16 @@
+[
+  {
+    "benchmark": "swe-bench",
+    "score": 42.5,
+    "metric": "resolve_rate",
+    "cost_per_instance": 0.15,
+    "average_runtime": 180
+  },
+  {
+    "benchmark": "gaia",
+    "score": 55.0,
+    "metric": "accuracy",
+    "cost_per_instance": 0.08,
+    "average_runtime": 60
+  }
+]

simple_data_loader.py

@@ -194,6 +194,9 @@ class SimpleLeaderboardViewer:
                     'llm_base': metadata.get('model', 'unknown'),
                     'openness': metadata.get('openness', 'unknown'),
                     'submission_time': metadata.get('submission_time', ''),
+                    'release_date': metadata.get('release_date', ''),  # Model release date
+                    'parameter_count_b': metadata.get('parameter_count_b'),  # Total params in billions
+                    'active_parameter_count_b': metadata.get('active_parameter_count_b'),  # Active params for MoE
                     'score': score_entry.get('score'),
                     'metric': score_entry.get('metric', 'unknown'),
                     'cost_per_instance': score_entry.get('cost_per_instance'),
@@ -257,6 +260,10 @@ class SimpleLeaderboardViewer:
                     'Language model': first_record['llm_base'],  # Will become "Language Model"
                     'openness': normalized_openness,  # Will become "Openness" (simplified to "open" or "closed")
                     'date': first_record['submission_time'],  # Will become "Date"
+                    # Model metadata for visualizations
+                    'release_date': first_record.get('release_date', ''),  # Model release date
+                    'parameter_count_b': first_record.get('parameter_count_b'),  # Total params in billions
+                    'active_parameter_count_b': first_record.get('active_parameter_count_b'),  # Active params for MoE
                     # Additional columns expected by the transformer
                     # Use agent_id (version_model) as unique identifier for Pareto frontier calculation
                     'id': agent_id,

visualizations.py

@@ -0,0 +1,311 @@
+"""
+Additional visualizations for the OpenHands Index leaderboard.
+"""
+import pandas as pd
+import plotly.graph_objects as go
+import plotly.express as px
+from datetime import datetime
+import aliases
+
+
+def create_evolution_over_time_chart(df: pd.DataFrame) -> go.Figure:
+    """
+    Create a line chart showing model performance evolution over release dates.
+    
+    Args:
+        df: DataFrame with columns including 'release_date', 'Language Model', 'average score', 'openness'
+    
+    Returns:
+        Plotly figure showing score evolution over time
+    """
+    if df.empty or 'release_date' not in df.columns:
+        fig = go.Figure()
+        fig.add_annotation(
+            text="No release date data available",
+            xref="paper", yref="paper",
+            x=0.5, y=0.5, showarrow=False,
+            font=dict(size=16)
+        )
+        return fig
+    
+    # Filter out rows without release dates
+    plot_df = df[df['release_date'].notna() & (df['release_date'] != '')].copy()
+    
+    if plot_df.empty:
+        fig = go.Figure()
+        fig.add_annotation(
+            text="No release date data available",
+            xref="paper", yref="paper",
+            x=0.5, y=0.5, showarrow=False,
+            font=dict(size=16)
+        )
+        return fig
+    
+    # Convert release_date to datetime
+    plot_df['release_date'] = pd.to_datetime(plot_df['release_date'], errors='coerce')
+    plot_df = plot_df.dropna(subset=['release_date'])
+    
+    # Sort by release date
+    plot_df = plot_df.sort_values('release_date')
+    
+    # Get the score column
+    score_col = 'average score' if 'average score' in plot_df.columns else None
+    if score_col is None:
+        for col in plot_df.columns:
+            if 'score' in col.lower() and 'average' in col.lower():
+                score_col = col
+                break
+    
+    if score_col is None:
+        fig = go.Figure()
+        fig.add_annotation(
+            text="No score data available",
+            xref="paper", yref="paper",
+            x=0.5, y=0.5, showarrow=False,
+            font=dict(size=16)
+        )
+        return fig
+    
+    # Get model name column
+    model_col = 'Language Model' if 'Language Model' in plot_df.columns else 'Language model'
+    
+    # Map openness to colors
+    color_map = {
+        aliases.CANONICAL_OPENNESS_OPEN: "#F0529C",  # Pink for open
+        aliases.CANONICAL_OPENNESS_CLOSED: "#FFD700",  # Yellow/gold for closed
+    }
+    for canonical_openness, openness_aliases in aliases.OPENNESS_ALIASES.items():
+        for openness_alias in openness_aliases:
+            color_map[openness_alias] = color_map[canonical_openness]
+    
+    fig = go.Figure()
+    
+    # Add scatter points for each model
+    for _, row in plot_df.iterrows():
+        openness = row.get('Openness', row.get('openness', 'unknown'))
+        color = color_map.get(openness, '#888888')
+        model_name = row.get(model_col, 'Unknown')
+        
+        fig.add_trace(go.Scatter(
+            x=[row['release_date']],
+            y=[row[score_col]],
+            mode='markers+text',
+            marker=dict(
+                size=12,
+                color=color,
+                line=dict(width=1, color='#333333')
+            ),
+            text=[model_name],
+            textposition='top center',
+            textfont=dict(size=10),
+            name=model_name,
+            hovertemplate=(
+                f"<b>{model_name}</b><br>"
+                f"Release: %{{x|%Y-%m-%d}}<br>"
+                f"Score: %{{y:.1f}}<br>"
+                f"<extra></extra>"
+            ),
+            showlegend=False
+        ))
+    
+    # Add trend line
+    if len(plot_df) > 1:
+        fig.add_trace(go.Scatter(
+            x=plot_df['release_date'],
+            y=plot_df[score_col],
+            mode='lines',
+            line=dict(color='#0FCB8C', width=2, dash='dash'),
+            name='Trend',
+            hoverinfo='skip',
+            showlegend=False
+        ))
+    
+    # Update layout
+    fig.update_layout(
+        title=dict(
+            text="Model Performance Evolution Over Time",
+            font=dict(size=18)
+        ),
+        xaxis=dict(
+            title="Model Release Date",
+            showgrid=True,
+            gridcolor='rgba(128,128,128,0.2)'
+        ),
+        yaxis=dict(
+            title="Average Score",
+            showgrid=True,
+            gridcolor='rgba(128,128,128,0.2)'
+        ),
+        plot_bgcolor='rgba(0,0,0,0)',
+        paper_bgcolor='rgba(0,0,0,0)',
+        hovermode='closest',
+        margin=dict(l=60, r=40, t=60, b=60),
+        height=400
+    )
+    
+    # Add legend for openness
+    fig.add_trace(go.Scatter(
+        x=[None], y=[None],
+        mode='markers',
+        marker=dict(size=10, color='#F0529C'),
+        name='Open Weights',
+        showlegend=True
+    ))
+    fig.add_trace(go.Scatter(
+        x=[None], y=[None],
+        mode='markers',
+        marker=dict(size=10, color='#FFD700'),
+        name='Closed',
+        showlegend=True
+    ))
+    
+    fig.update_layout(
+        legend=dict(
+            orientation="h",
+            yanchor="bottom",
+            y=1.02,
+            xanchor="right",
+            x=1
+        )
+    )
+    
+    return fig
+
+
+def create_accuracy_by_size_chart(df: pd.DataFrame) -> go.Figure:
+    """
+    Create a scatter plot showing accuracy vs parameter count for open-weights models.
+    
+    Args:
+        df: DataFrame with columns including 'parameter_count_b', 'active_parameter_count_b', 
+            'average score', 'openness', 'Language Model'
+    
+    Returns:
+        Plotly figure showing accuracy vs model size
+    """
+    if df.empty or 'parameter_count_b' not in df.columns:
+        fig = go.Figure()
+        fig.add_annotation(
+            text="No parameter count data available",
+            xref="paper", yref="paper",
+            x=0.5, y=0.5, showarrow=False,
+            font=dict(size=16)
+        )
+        return fig
+    
+    # Filter to only open-weights models with parameter data
+    open_aliases = [aliases.CANONICAL_OPENNESS_OPEN] + list(aliases.OPENNESS_ALIASES.get(aliases.CANONICAL_OPENNESS_OPEN, []))
+    
+    plot_df = df[
+        (df['parameter_count_b'].notna()) & 
+        (df['Openness'].isin(open_aliases) | df.get('openness', pd.Series()).isin(open_aliases))
+    ].copy()
+    
+    if plot_df.empty:
+        fig = go.Figure()
+        fig.add_annotation(
+            text="No open-weights models with parameter data available",
+            xref="paper", yref="paper",
+            x=0.5, y=0.5, showarrow=False,
+            font=dict(size=16)
+        )
+        return fig
+    
+    # Get the score column
+    score_col = 'average score' if 'average score' in plot_df.columns else None
+    if score_col is None:
+        for col in plot_df.columns:
+            if 'score' in col.lower() and 'average' in col.lower():
+                score_col = col
+                break
+    
+    if score_col is None:
+        fig = go.Figure()
+        fig.add_annotation(
+            text="No score data available",
+            xref="paper", yref="paper",
+            x=0.5, y=0.5, showarrow=False,
+            font=dict(size=16)
+        )
+        return fig
+    
+    # Get model name column
+    model_col = 'Language Model' if 'Language Model' in plot_df.columns else 'Language model'
+    
+    fig = go.Figure()
+    
+    # Determine if we should use active params (for MoE models) or total params
+    # Use active params if available, otherwise total params
+    for _, row in plot_df.iterrows():
+        total_params = row['parameter_count_b']
+        active_params = row.get('active_parameter_count_b')
+        model_name = row.get(model_col, 'Unknown')
+        score = row[score_col]
+        
+        # Use active params for x-axis if available (more meaningful for MoE)
+        x_val = active_params if pd.notna(active_params) else total_params
+        
+        # Create hover text
+        hover_text = f"<b>{model_name}</b><br>"
+        hover_text += f"Total Params: {total_params:.0f}B<br>"
+        if pd.notna(active_params):
+            hover_text += f"Active Params: {active_params:.0f}B<br>"
+        hover_text += f"Score: {score:.1f}<br>"
+        
+        # Marker size based on total params (larger models = larger markers)
+        marker_size = 10 + (total_params / 100)  # Scale marker size
+        marker_size = min(marker_size, 30)  # Cap at 30
+        
+        fig.add_trace(go.Scatter(
+            x=[x_val],
+            y=[score],
+            mode='markers+text',
+            marker=dict(
+                size=marker_size,
+                color='#F0529C',  # Pink for open models
+                line=dict(width=1, color='#333333'),
+                opacity=0.8
+            ),
+            text=[model_name],
+            textposition='top center',
+            textfont=dict(size=10),
+            name=model_name,
+            hovertemplate=hover_text + "<extra></extra>",
+            showlegend=False
+        ))
+    
+    # Update layout
+    fig.update_layout(
+        title=dict(
+            text="Open Model Accuracy by Size",
+            font=dict(size=18)
+        ),
+        xaxis=dict(
+            title="Active Parameters (Billions)",
+            showgrid=True,
+            gridcolor='rgba(128,128,128,0.2)',
+            type='log'  # Log scale for better visualization
+        ),
+        yaxis=dict(
+            title="Average Score",
+            showgrid=True,
+            gridcolor='rgba(128,128,128,0.2)'
+        ),
+        plot_bgcolor='rgba(0,0,0,0)',
+        paper_bgcolor='rgba(0,0,0,0)',
+        hovermode='closest',
+        margin=dict(l=60, r=40, t=60, b=60),
+        height=400
+    )
+    
+    # Add annotation explaining marker size
+    fig.add_annotation(
+        text="Marker size indicates total parameter count",
+        xref="paper", yref="paper",
+        x=0.02, y=-0.12,
+        showarrow=False,
+        font=dict(size=10, color='gray'),
+        align='left'
+    )
+    
+    return fig