"""
Additional visualizations for the OpenHands Index leaderboard.
"""
import pandas as pd
import plotly.graph_objects as go
import plotly.express as px
from datetime import datetime
import os
import base64
import aliases
# Import company logo mapping from ui_components
from ui_components import get_company_from_model, get_svg_as_data_uri
# Standard layout configuration matching existing charts
# Colors aligned with OpenHands brand
STANDARD_LAYOUT = dict(
template="plotly_white",
height=572,
font=dict(
family="Outfit, ui-sans-serif, sans-serif",
color="#0D0D0F", # neutral-950
),
hoverlabel=dict(
bgcolor="#222328", # neutral-800
font_size=12,
font_family="Outfit",
font_color="#F7F8FB", # neutral-50
),
legend=dict(
bgcolor='#F7F8FB', # neutral-50
),
margin=dict(b=80), # Extra margin for logo and URL
)
# Standard font for annotations
STANDARD_FONT = dict(
size=10,
color='#0D0D0F', # neutral-950
family='Outfit'
)
# OpenHands branding constants
OPENHANDS_LOGO_PATH = "assets/openhands_logo_color_forwhite.png"
OPENHANDS_URL = "https://index.openhands.dev"
# URL annotation for bottom right of charts
URL_ANNOTATION = dict(
text=OPENHANDS_URL,
xref="paper",
yref="paper",
x=1,
y=-0.15,
xanchor="right",
yanchor="bottom",
showarrow=False,
font=dict(
family="Outfit, ui-sans-serif, sans-serif",
size=14,
color="#82889B", # neutral-400
),
)
def get_openhands_logo_image():
"""Get the OpenHands logo as a Plotly image dict for chart branding."""
if os.path.exists(OPENHANDS_LOGO_PATH):
try:
with open(OPENHANDS_LOGO_PATH, "rb") as f:
logo_data = base64.b64encode(f.read()).decode('utf-8')
return dict(
source=f"data:image/png;base64,{logo_data}",
xref="paper",
yref="paper",
x=0,
y=-0.15,
sizex=0.15,
sizey=0.15,
xanchor="left",
yanchor="bottom",
)
except Exception:
pass
return None
def add_branding_to_figure(fig: go.Figure) -> go.Figure:
"""Add OpenHands logo and URL to a Plotly figure."""
# Add logo image
logo_image = get_openhands_logo_image()
if logo_image:
existing_images = list(fig.layout.images) if fig.layout.images else []
existing_images.append(logo_image)
fig.update_layout(images=existing_images)
# Add URL annotation
existing_annotations = list(fig.layout.annotations) if fig.layout.annotations else []
existing_annotations.append(URL_ANNOTATION)
fig.update_layout(annotations=existing_annotations)
return fig
def create_evolution_over_time_chart(df: pd.DataFrame) -> go.Figure:
"""
Create a chart showing model performance evolution over release dates.
Uses company logos as markers to match the existing chart styling.
Args:
df: DataFrame with columns including 'release_date' or 'Release_Date', 'Language Model', 'average score', 'openness'
Returns:
Plotly figure showing score evolution over time
"""
# Handle different column name formats
release_date_col = None
for col in ['release_date', 'Release_Date', 'Release Date']:
if col in df.columns:
release_date_col = col
break
if df.empty or release_date_col is None:
fig = go.Figure()
fig.add_annotation(
text="No release date data available",
xref="paper", yref="paper",
x=0.5, y=0.5, showarrow=False,
font=STANDARD_FONT
)
fig.update_layout(**STANDARD_LAYOUT)
return fig
# Filter out rows without release dates
plot_df = df[df[release_date_col].notna() & (df[release_date_col] != '')].copy()
if plot_df.empty:
fig = go.Figure()
fig.add_annotation(
text="No release date data available",
xref="paper", yref="paper",
x=0.5, y=0.5, showarrow=False,
font=STANDARD_FONT
)
fig.update_layout(**STANDARD_LAYOUT)
return fig
# Convert release_date to datetime (normalize column name)
plot_df['release_date'] = pd.to_datetime(plot_df[release_date_col], errors='coerce')
plot_df = plot_df.dropna(subset=['release_date'])
# Sort by release date
plot_df = plot_df.sort_values('release_date')
# Get the score column (handle different naming conventions)
score_col = None
for col in ['average score', 'Average Score', 'Average score']:
if col in plot_df.columns:
score_col = col
break
if score_col is None:
for col in plot_df.columns:
if 'score' in col.lower() and 'average' in col.lower():
score_col = col
break
if score_col is None:
fig = go.Figure()
fig.add_annotation(
text="No score data available",
xref="paper", yref="paper",
x=0.5, y=0.5, showarrow=False,
font=STANDARD_FONT
)
fig.update_layout(**STANDARD_LAYOUT)
return fig
# Get model name column
model_col = None
for col in ['Language Model', 'Language model', 'llm_base']:
if col in plot_df.columns:
model_col = col
break
if model_col is None:
model_col = 'Language Model' # Default
fig = go.Figure()
# Add Pareto frontier line (monotonically increasing best score over time)
if len(plot_df) > 1:
# Compute Pareto frontier: only include points that set a new best score
frontier_dates = []
frontier_scores = []
max_score_so_far = float('-inf')
for _, row in plot_df.iterrows():
current_score = row[score_col]
current_date = row['release_date']
if current_score > max_score_so_far:
# This point is on the Pareto frontier
frontier_dates.append(current_date)
frontier_scores.append(current_score)
max_score_so_far = current_score
if frontier_dates:
fig.add_trace(go.Scatter(
x=frontier_dates,
y=frontier_scores,
mode='lines',
line=dict(color='#FFE165', width=2, dash='dash'), # primary yellow, dashed
name='Pareto Frontier',
hoverinfo='skip',
showlegend=False
))
# Calculate axis ranges
min_date = plot_df['release_date'].min()
max_date = plot_df['release_date'].max()
min_score = plot_df[score_col].min()
max_score = plot_df[score_col].max()
y_min = min_score - 5 if min_score > 5 else 0
y_max = max_score + 10 # Extra space for labels
# Build hover text for each point
hover_texts = []
for _, row in plot_df.iterrows():
model_name = row.get(model_col, 'Unknown')
openness = row.get('Openness', row.get('openness', 'unknown'))
h_pad = " "
hover_text = f"
{h_pad}{model_name}{h_pad}
"
hover_text += f"{h_pad}Release: {row['release_date'].strftime('%Y-%m-%d')}{h_pad}
"
hover_text += f"{h_pad}Average Score: {row[score_col]:.1f}{h_pad}
"
hover_text += f"{h_pad}Openness: {openness}{h_pad}
"
hover_texts.append(hover_text)
plot_df['hover_text'] = hover_texts
# Add invisible markers for hover functionality
fig.add_trace(go.Scatter(
x=plot_df['release_date'],
y=plot_df[score_col],
mode='markers',
name='Models',
showlegend=False,
text=plot_df['hover_text'],
hoverinfo='text',
marker=dict(
color='rgba(0,0,0,0)', # Invisible markers
size=25, # Large enough for hover detection
opacity=0
)
))
# Add company logo images for each data point using data coordinates
layout_images = []
labels_data = []
for _, row in plot_df.iterrows():
model_name = row.get(model_col, '')
company_info = get_company_from_model(model_name)
logo_path = company_info['path']
# Read the SVG file and encode as base64 data URI
if os.path.exists(logo_path):
try:
with open(logo_path, 'rb') as f:
encoded_logo = base64.b64encode(f.read()).decode('utf-8')
logo_uri = f"data:image/svg+xml;base64,{encoded_logo}"
x_val = row['release_date']
y_val = row[score_col]
# Use data coordinates for precise alignment
layout_images.append(dict(
source=logo_uri,
xref="x",
yref="y",
x=x_val,
y=y_val,
sizex=15 * 24 * 60 * 60 * 1000, # ~15 days in milliseconds
sizey=3, # score units
xanchor="center",
yanchor="middle",
layer="above"
))
# Store label data for annotation
labels_data.append({
'x': x_val,
'y': y_val,
'label': model_name
})
except Exception:
pass
# Add model name labels above each point
for item in labels_data:
fig.add_annotation(
x=item['x'],
y=item['y'],
xref="x",
yref="y",
text=item['label'],
showarrow=False,
yshift=20,
font=STANDARD_FONT,
xanchor='center',
yanchor='bottom'
)
# Build layout configuration
layout_config = dict(
**STANDARD_LAYOUT,
title="Model Performance Evolution Over Time",
xaxis=dict(
title="Model Release Date",
range=[min_date - pd.Timedelta(days=15), max_date + pd.Timedelta(days=15)]
),
yaxis=dict(
title="Average Score",
range=[y_min, y_max]
),
)
# Add company logo images to the layout
if layout_images:
layout_config['images'] = layout_images
fig.update_layout(**layout_config)
# Add OpenHands branding
add_branding_to_figure(fig)
return fig
def create_accuracy_by_size_chart(df: pd.DataFrame) -> go.Figure:
"""
Create a scatter plot showing accuracy vs parameter count for open-weights models.
Uses company logos as markers to match the existing chart styling.
Args:
df: DataFrame with columns including 'parameter_count_b' or 'Parameter_Count_B',
'active_parameter_count_b' or 'Active_Parameter_Count_B',
'average score', 'openness', 'Language Model'
Returns:
Plotly figure showing accuracy vs model size
"""
import numpy as np
# Handle different column name formats for parameter count
param_col = None
for col in ['parameter_count_b', 'Parameter_Count_B', 'Parameter Count B']:
if col in df.columns:
param_col = col
break
active_param_col = None
for col in ['active_parameter_count_b', 'Active_Parameter_Count_B', 'Active Parameter Count B']:
if col in df.columns:
active_param_col = col
break
if df.empty or param_col is None:
fig = go.Figure()
fig.add_annotation(
text="No parameter count data available",
xref="paper", yref="paper",
x=0.5, y=0.5, showarrow=False,
font=STANDARD_FONT
)
fig.update_layout(**STANDARD_LAYOUT)
return fig
# Filter to only open-weights models with parameter data
open_aliases = [aliases.CANONICAL_OPENNESS_OPEN] + list(aliases.OPENNESS_ALIASES.get(aliases.CANONICAL_OPENNESS_OPEN, []))
# Get openness column
openness_col = 'Openness' if 'Openness' in df.columns else 'openness'
plot_df = df[
(df[param_col].notna()) &
(df[openness_col].isin(open_aliases))
].copy()
if plot_df.empty:
fig = go.Figure()
fig.add_annotation(
text="No open-weights models with parameter data available",
xref="paper", yref="paper",
x=0.5, y=0.5, showarrow=False,
font=STANDARD_FONT
)
fig.update_layout(**STANDARD_LAYOUT)
return fig
# Get the score column (handle different naming conventions)
score_col = None
for col in ['average score', 'Average Score', 'Average score']:
if col in plot_df.columns:
score_col = col
break
if score_col is None:
for col in plot_df.columns:
if 'score' in col.lower() and 'average' in col.lower():
score_col = col
break
if score_col is None:
fig = go.Figure()
fig.add_annotation(
text="No score data available",
xref="paper", yref="paper",
x=0.5, y=0.5, showarrow=False,
font=STANDARD_FONT
)
fig.update_layout(**STANDARD_LAYOUT)
return fig
# Get model name column
model_col = None
for col in ['Language Model', 'Language model', 'llm_base']:
if col in plot_df.columns:
model_col = col
break
if model_col is None:
model_col = 'Language Model' # Default
fig = go.Figure()
# Prepare data for plotting
x_values = []
y_values = []
hover_texts = []
model_names = []
total_params_list = []
for _, row in plot_df.iterrows():
total_params = row[param_col]
active_params = row.get(active_param_col) if active_param_col else None
model_name = row.get(model_col, 'Unknown')
score = row[score_col]
# Use active params for x-axis if available (more meaningful for MoE)
x_val = active_params if pd.notna(active_params) else total_params
x_values.append(x_val)
y_values.append(score)
model_names.append(model_name)
total_params_list.append(total_params)
# Create hover text matching existing chart style
h_pad = " "
hover_text = f"
{h_pad}{model_name}{h_pad}
"
hover_text += f"{h_pad}Total Params: {total_params:.0f}B{h_pad}
"
if pd.notna(active_params):
hover_text += f"{h_pad}Active Params: {active_params:.0f}B{h_pad}
"
hover_text += f"{h_pad}Average Score: {score:.1f}{h_pad}
"
hover_texts.append(hover_text)
# Calculate axis ranges for domain coordinate conversion
min_x = min(x_values)
max_x = max(x_values)
x_min_log = np.log10(min_x * 0.5) if min_x > 0 else 0
x_max_log = np.log10(max_x * 1.5) if max_x > 0 else 3
min_score = min(y_values)
max_score = max(y_values)
y_min = min_score - 5 if min_score > 5 else 0
y_max = max_score + 10 # Extra space for labels
# Add invisible markers for hover functionality
fig.add_trace(go.Scatter(
x=x_values,
y=y_values,
mode='markers',
name='Models',
showlegend=False,
text=hover_texts,
hoverinfo='text',
marker=dict(
color='rgba(0,0,0,0)', # Invisible markers
size=25, # Large enough for hover detection
opacity=0
)
))
# Add company logo images for each data point
layout_images = []
frontier_labels_data = []
for i, (x_val, y_val, model_name, total_params) in enumerate(zip(x_values, y_values, model_names, total_params_list)):
company_info = get_company_from_model(model_name)
logo_path = company_info['path']
# Read the SVG file and encode as base64 data URI
if os.path.exists(logo_path):
try:
with open(logo_path, 'rb') as f:
encoded_logo = base64.b64encode(f.read()).decode('utf-8')
logo_uri = f"data:image/svg+xml;base64,{encoded_logo}"
# Convert to domain coordinates (0-1 range) for log scale x-axis
if x_val > 0:
log_x = np.log10(x_val)
domain_x = (log_x - x_min_log) / (x_max_log - x_min_log)
else:
domain_x = 0
domain_y = (y_val - y_min) / (y_max - y_min) if (y_max - y_min) > 0 else 0.5
# Clamp to valid range
domain_x = max(0.02, min(0.98, domain_x))
domain_y = max(0.02, min(0.98, domain_y))
# Scale logo size based on total params
size_scale = 0.03 + (total_params / 2000) # Larger models = larger logos
size_scale = min(size_scale, 0.06) # Cap size
layout_images.append(dict(
source=logo_uri,
xref="x domain",
yref="y domain",
x=domain_x,
y=domain_y,
sizex=size_scale,
sizey=size_scale * 1.5,
xanchor="center",
yanchor="middle",
layer="above"
))
# Store label data for annotation
frontier_labels_data.append({
'x': domain_x,
'y': domain_y,
'label': model_name
})
except Exception:
pass
# Add model name labels above each point
for item in frontier_labels_data:
fig.add_annotation(
x=item['x'],
y=item['y'],
xref="x domain",
yref="y domain",
text=item['label'],
showarrow=False,
yshift=25,
font=STANDARD_FONT,
xanchor='center',
yanchor='bottom'
)
# Build layout configuration
layout_config = dict(
**STANDARD_LAYOUT,
title="Open Model Accuracy by Size",
xaxis=dict(
title="Active Parameters (Billions)",
type="log",
range=[x_min_log, x_max_log]
),
yaxis=dict(
title="Average Score",
range=[y_min, y_max]
),
)
# Add company logo images to the layout
if layout_images:
layout_config['images'] = layout_images
fig.update_layout(**layout_config)
# Add annotation explaining marker size
fig.add_annotation(
text="Logo size indicates total parameter count",
xref="paper", yref="paper",
x=0.02, y=-0.08,
showarrow=False,
font=STANDARD_FONT,
align='left'
)
# Add OpenHands branding
add_branding_to_figure(fig)
return fig