zhimin-z
commited on
Commit
·
ada7e18
1
Parent(s):
05edd71
add
Browse files
app.py
CHANGED
|
@@ -304,9 +304,9 @@ def create_monthly_metrics_plot(type="issue", top_n=5):
|
|
| 304 |
print_msg = "discussion"
|
| 305 |
elif type == "wanted":
|
| 306 |
metrics_key = 'wanted_issue_monthly_metrics'
|
| 307 |
-
total_field = '
|
| 308 |
no_data_msg = "No wanted issue data available for visualization"
|
| 309 |
-
total_label = "
|
| 310 |
print_msg = "wanted issue"
|
| 311 |
|
| 312 |
# Load from saved dataset
|
|
@@ -390,45 +390,41 @@ def create_monthly_metrics_plot(type="issue", top_n=5):
|
|
| 390 |
color = agent_colors[agent_name]
|
| 391 |
agent_data = data[agent_name]
|
| 392 |
|
| 393 |
-
# Add line trace for resolved rate (left y-axis)
|
| 394 |
-
resolved_rates = agent_data
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
)
|
| 418 |
|
| 419 |
# Add bar trace for total count (right y-axis)
|
| 420 |
# Only show bars for months where assistant has data
|
| 421 |
x_bars = []
|
| 422 |
y_bars = []
|
| 423 |
-
|
| 424 |
-
for month, count in zip(months, total_data):
|
| 425 |
if count > 0: # Only include months with data
|
| 426 |
x_bars.append(month)
|
| 427 |
y_bars.append(count)
|
| 428 |
|
| 429 |
if x_bars and y_bars: # Only add trace if there's data
|
| 430 |
-
# For wanted type without resolved_rates, show legend on bar chart
|
| 431 |
-
show_bar_legend = (top_n is not None and top_n <= 10) and not resolved_rates
|
| 432 |
fig.add_trace(
|
| 433 |
go.Bar(
|
| 434 |
x=x_bars,
|
|
@@ -436,7 +432,7 @@ def create_monthly_metrics_plot(type="issue", top_n=5):
|
|
| 436 |
name=agent_name,
|
| 437 |
marker=dict(color=color, opacity=0.6),
|
| 438 |
legendgroup=agent_name,
|
| 439 |
-
showlegend=
|
| 440 |
hovertemplate=f'<b>Assistant: %{{fullData.name}}</b><br>' +
|
| 441 |
f'Month: %{{x}}<br>' +
|
| 442 |
f'{total_label}: %{{y}}<br>' +
|
|
@@ -448,25 +444,15 @@ def create_monthly_metrics_plot(type="issue", top_n=5):
|
|
| 448 |
|
| 449 |
# Update axes labels
|
| 450 |
fig.update_xaxes(title_text=None)
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
else:
|
| 461 |
-
fig.update_yaxes(
|
| 462 |
-
title_text="<b>Resolved Rate (%)</b>",
|
| 463 |
-
range=[0, 100],
|
| 464 |
-
secondary_y=False,
|
| 465 |
-
showticklabels=True,
|
| 466 |
-
tickmode='linear',
|
| 467 |
-
dtick=10,
|
| 468 |
-
showgrid=True
|
| 469 |
-
)
|
| 470 |
fig.update_yaxes(title_text=f"<b>{total_label}</b>", secondary_y=True)
|
| 471 |
|
| 472 |
# Update layout
|
|
@@ -726,7 +712,6 @@ with gr.Blocks(title="SWE Assistant Issue & Discussion Leaderboard", theme=gr.th
|
|
| 726 |
value=pd.DataFrame(columns=[col[0] for col in LEADERBOARD_COLUMNS]), # Empty initially
|
| 727 |
datatype=LEADERBOARD_COLUMNS,
|
| 728 |
search_columns=["Assistant", "Website"],
|
| 729 |
-
height=400, # Minimum height to prevent rows from being obscured by filter controls
|
| 730 |
filter_columns=[
|
| 731 |
ColumnFilter(
|
| 732 |
"Issue Resolved Rate (%)",
|
|
@@ -776,7 +761,7 @@ with gr.Blocks(title="SWE Assistant Issue & Discussion Leaderboard", theme=gr.th
|
|
| 776 |
discussion_metrics_plot = gr.Plot()
|
| 777 |
|
| 778 |
with gr.Column():
|
| 779 |
-
gr.Markdown("*
|
| 780 |
wanted_metrics_plot = gr.Plot()
|
| 781 |
|
| 782 |
# Load monthly metrics when app starts
|
|
|
|
| 304 |
print_msg = "discussion"
|
| 305 |
elif type == "wanted":
|
| 306 |
metrics_key = 'wanted_issue_monthly_metrics'
|
| 307 |
+
total_field = 'total_wanted'
|
| 308 |
no_data_msg = "No wanted issue data available for visualization"
|
| 309 |
+
total_label = "Total Wanted Issues"
|
| 310 |
print_msg = "wanted issue"
|
| 311 |
|
| 312 |
# Load from saved dataset
|
|
|
|
| 390 |
color = agent_colors[agent_name]
|
| 391 |
agent_data = data[agent_name]
|
| 392 |
|
| 393 |
+
# Add line trace for resolved rate (left y-axis)
|
| 394 |
+
resolved_rates = agent_data['resolved_rates']
|
| 395 |
+
# Filter out None values for plotting
|
| 396 |
+
x_resolved = [month for month, rate in zip(months, resolved_rates) if rate is not None]
|
| 397 |
+
y_resolved = [rate for rate in resolved_rates if rate is not None]
|
| 398 |
+
|
| 399 |
+
if x_resolved and y_resolved: # Only add trace if there's data
|
| 400 |
+
fig.add_trace(
|
| 401 |
+
go.Scatter(
|
| 402 |
+
x=x_resolved,
|
| 403 |
+
y=y_resolved,
|
| 404 |
+
name=agent_name,
|
| 405 |
+
mode='lines+markers',
|
| 406 |
+
line=dict(color=color, width=2),
|
| 407 |
+
marker=dict(size=8),
|
| 408 |
+
legendgroup=agent_name,
|
| 409 |
+
showlegend=(top_n is not None and top_n <= 10), # Show legend for top N assistants
|
| 410 |
+
hovertemplate='<b>Assistant: %{fullData.name}</b><br>' +
|
| 411 |
+
'Month: %{x}<br>' +
|
| 412 |
+
'Resolved Rate: %{y:.2f}%<br>' +
|
| 413 |
+
'<extra></extra>'
|
| 414 |
+
),
|
| 415 |
+
secondary_y=False
|
| 416 |
+
)
|
|
|
|
| 417 |
|
| 418 |
# Add bar trace for total count (right y-axis)
|
| 419 |
# Only show bars for months where assistant has data
|
| 420 |
x_bars = []
|
| 421 |
y_bars = []
|
| 422 |
+
for month, count in zip(months, agent_data[total_field]):
|
|
|
|
| 423 |
if count > 0: # Only include months with data
|
| 424 |
x_bars.append(month)
|
| 425 |
y_bars.append(count)
|
| 426 |
|
| 427 |
if x_bars and y_bars: # Only add trace if there's data
|
|
|
|
|
|
|
| 428 |
fig.add_trace(
|
| 429 |
go.Bar(
|
| 430 |
x=x_bars,
|
|
|
|
| 432 |
name=agent_name,
|
| 433 |
marker=dict(color=color, opacity=0.6),
|
| 434 |
legendgroup=agent_name,
|
| 435 |
+
showlegend=False, # Hide duplicate legend entry (already shown in Scatter)
|
| 436 |
hovertemplate=f'<b>Assistant: %{{fullData.name}}</b><br>' +
|
| 437 |
f'Month: %{{x}}<br>' +
|
| 438 |
f'{total_label}: %{{y}}<br>' +
|
|
|
|
| 444 |
|
| 445 |
# Update axes labels
|
| 446 |
fig.update_xaxes(title_text=None)
|
| 447 |
+
fig.update_yaxes(
|
| 448 |
+
title_text="<b>Resolved Rate (%)</b>",
|
| 449 |
+
range=[0, 100],
|
| 450 |
+
secondary_y=False,
|
| 451 |
+
showticklabels=True,
|
| 452 |
+
tickmode='linear',
|
| 453 |
+
dtick=10,
|
| 454 |
+
showgrid=True
|
| 455 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 456 |
fig.update_yaxes(title_text=f"<b>{total_label}</b>", secondary_y=True)
|
| 457 |
|
| 458 |
# Update layout
|
|
|
|
| 712 |
value=pd.DataFrame(columns=[col[0] for col in LEADERBOARD_COLUMNS]), # Empty initially
|
| 713 |
datatype=LEADERBOARD_COLUMNS,
|
| 714 |
search_columns=["Assistant", "Website"],
|
|
|
|
| 715 |
filter_columns=[
|
| 716 |
ColumnFilter(
|
| 717 |
"Issue Resolved Rate (%)",
|
|
|
|
| 761 |
discussion_metrics_plot = gr.Plot()
|
| 762 |
|
| 763 |
with gr.Column():
|
| 764 |
+
gr.Markdown("*Wanted issue volume and resolved rate over time*")
|
| 765 |
wanted_metrics_plot = gr.Plot()
|
| 766 |
|
| 767 |
# Load monthly metrics when app starts
|
msr.py
CHANGED
|
@@ -1050,20 +1050,25 @@ def calculate_monthly_metrics_by_discussions(all_discussions_dict, assistants):
|
|
| 1050 |
}
|
| 1051 |
|
| 1052 |
|
| 1053 |
-
def calculate_monthly_metrics_by_wanted_issues(wanted_resolved_dict, assistants):
|
| 1054 |
-
"""Calculate monthly metrics for
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1055 |
"""
|
| 1056 |
identifier_to_name = {assistant.get('github_identifier'): assistant.get('name') for assistant in assistants if assistant.get('github_identifier')}
|
| 1057 |
|
| 1058 |
-
if not wanted_resolved_dict:
|
| 1059 |
return {'assistants': [], 'months': [], 'data': {}}
|
| 1060 |
|
| 1061 |
# Calculate the valid date range for filtering
|
| 1062 |
current_time = datetime.now(timezone.utc)
|
| 1063 |
start_date = current_time - timedelta(days=LEADERBOARD_TIME_FRAME_DAYS)
|
| 1064 |
start_month_key = f"{start_date.year}-{start_date.month:02d}"
|
|
|
|
| 1065 |
|
| 1066 |
-
agent_month_data = defaultdict(lambda: defaultdict(
|
| 1067 |
|
| 1068 |
# Process resolved wanted issues - group by closed_at date
|
| 1069 |
for agent_identifier, metadata_list in (wanted_resolved_dict or {}).items():
|
|
@@ -1083,11 +1088,17 @@ def calculate_monthly_metrics_by_wanted_issues(wanted_resolved_dict, assistants)
|
|
| 1083 |
continue
|
| 1084 |
|
| 1085 |
month_key = f"{dt.year}-{dt.month:02d}"
|
| 1086 |
-
agent_month_data[agent_name][month_key]
|
| 1087 |
except Exception as e:
|
| 1088 |
print(f"Warning: Could not parse wanted issue date '{closed_at}': {e}")
|
| 1089 |
continue
|
| 1090 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1091 |
all_months = set()
|
| 1092 |
for agent_data in agent_month_data.values():
|
| 1093 |
all_months.update(agent_data.keys())
|
|
@@ -1097,13 +1108,26 @@ def calculate_monthly_metrics_by_wanted_issues(wanted_resolved_dict, assistants)
|
|
| 1097 |
|
| 1098 |
result_data = {}
|
| 1099 |
for agent_name, month_dict in agent_month_data.items():
|
|
|
|
|
|
|
| 1100 |
resolved_wanted_list = []
|
| 1101 |
|
| 1102 |
for month in months:
|
| 1103 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1104 |
resolved_wanted_list.append(resolved_count)
|
| 1105 |
|
| 1106 |
result_data[agent_name] = {
|
|
|
|
|
|
|
| 1107 |
'resolved_wanted': resolved_wanted_list
|
| 1108 |
}
|
| 1109 |
|
|
@@ -1288,7 +1312,7 @@ def mine_all_agents():
|
|
| 1288 |
agent_discussions, assistants
|
| 1289 |
)
|
| 1290 |
wanted_issue_monthly_metrics = calculate_monthly_metrics_by_wanted_issues(
|
| 1291 |
-
wanted_resolved, assistants
|
| 1292 |
)
|
| 1293 |
save_leaderboard_data_to_hf(
|
| 1294 |
leaderboard_dict, issue_monthly_metrics, wanted_open, discussion_monthly_metrics, wanted_issue_monthly_metrics
|
|
|
|
| 1050 |
}
|
| 1051 |
|
| 1052 |
|
| 1053 |
+
def calculate_monthly_metrics_by_wanted_issues(wanted_resolved_dict, wanted_in_progress_dict, assistants):
|
| 1054 |
+
"""Calculate monthly metrics for wanted issues for all assistants for visualization.
|
| 1055 |
+
|
| 1056 |
+
Resolved wanted issues are grouped by their closing date.
|
| 1057 |
+
In-progress wanted issues are counted in the current month (representing ongoing work).
|
| 1058 |
+
Only months within LEADERBOARD_TIME_FRAME_DAYS are included.
|
| 1059 |
"""
|
| 1060 |
identifier_to_name = {assistant.get('github_identifier'): assistant.get('name') for assistant in assistants if assistant.get('github_identifier')}
|
| 1061 |
|
| 1062 |
+
if not wanted_resolved_dict and not wanted_in_progress_dict:
|
| 1063 |
return {'assistants': [], 'months': [], 'data': {}}
|
| 1064 |
|
| 1065 |
# Calculate the valid date range for filtering
|
| 1066 |
current_time = datetime.now(timezone.utc)
|
| 1067 |
start_date = current_time - timedelta(days=LEADERBOARD_TIME_FRAME_DAYS)
|
| 1068 |
start_month_key = f"{start_date.year}-{start_date.month:02d}"
|
| 1069 |
+
current_month_key = f"{current_time.year}-{current_time.month:02d}"
|
| 1070 |
|
| 1071 |
+
agent_month_data = defaultdict(lambda: defaultdict(lambda: {'resolved': 0, 'in_progress': 0}))
|
| 1072 |
|
| 1073 |
# Process resolved wanted issues - group by closed_at date
|
| 1074 |
for agent_identifier, metadata_list in (wanted_resolved_dict or {}).items():
|
|
|
|
| 1088 |
continue
|
| 1089 |
|
| 1090 |
month_key = f"{dt.year}-{dt.month:02d}"
|
| 1091 |
+
agent_month_data[agent_name][month_key]['resolved'] += 1
|
| 1092 |
except Exception as e:
|
| 1093 |
print(f"Warning: Could not parse wanted issue date '{closed_at}': {e}")
|
| 1094 |
continue
|
| 1095 |
|
| 1096 |
+
# Process in-progress wanted issues - count in current month (ongoing work)
|
| 1097 |
+
for agent_identifier, metadata_list in (wanted_in_progress_dict or {}).items():
|
| 1098 |
+
agent_name = identifier_to_name.get(agent_identifier, agent_identifier)
|
| 1099 |
+
# All in-progress issues count towards the current month
|
| 1100 |
+
agent_month_data[agent_name][current_month_key]['in_progress'] += len(metadata_list)
|
| 1101 |
+
|
| 1102 |
all_months = set()
|
| 1103 |
for agent_data in agent_month_data.values():
|
| 1104 |
all_months.update(agent_data.keys())
|
|
|
|
| 1108 |
|
| 1109 |
result_data = {}
|
| 1110 |
for agent_name, month_dict in agent_month_data.items():
|
| 1111 |
+
resolved_rates = []
|
| 1112 |
+
total_wanted_list = []
|
| 1113 |
resolved_wanted_list = []
|
| 1114 |
|
| 1115 |
for month in months:
|
| 1116 |
+
month_data = month_dict.get(month, {'resolved': 0, 'in_progress': 0})
|
| 1117 |
+
resolved_count = month_data['resolved']
|
| 1118 |
+
in_progress_count = month_data['in_progress']
|
| 1119 |
+
total_count = resolved_count + in_progress_count
|
| 1120 |
+
|
| 1121 |
+
# Resolved rate = resolved / total * 100
|
| 1122 |
+
resolved_rate = (resolved_count / total_count * 100) if total_count > 0 else None
|
| 1123 |
+
|
| 1124 |
+
resolved_rates.append(resolved_rate)
|
| 1125 |
+
total_wanted_list.append(total_count)
|
| 1126 |
resolved_wanted_list.append(resolved_count)
|
| 1127 |
|
| 1128 |
result_data[agent_name] = {
|
| 1129 |
+
'resolved_rates': resolved_rates,
|
| 1130 |
+
'total_wanted': total_wanted_list,
|
| 1131 |
'resolved_wanted': resolved_wanted_list
|
| 1132 |
}
|
| 1133 |
|
|
|
|
| 1312 |
agent_discussions, assistants
|
| 1313 |
)
|
| 1314 |
wanted_issue_monthly_metrics = calculate_monthly_metrics_by_wanted_issues(
|
| 1315 |
+
wanted_resolved, wanted_in_progress, assistants
|
| 1316 |
)
|
| 1317 |
save_leaderboard_data_to_hf(
|
| 1318 |
leaderboard_dict, issue_monthly_metrics, wanted_open, discussion_monthly_metrics, wanted_issue_monthly_metrics
|