zhimin-z commited on
Commit
ada7e18
·
1 Parent(s): 05edd71
Files changed (2) hide show
  1. app.py +38 -53
  2. msr.py +31 -7
app.py CHANGED
@@ -304,9 +304,9 @@ def create_monthly_metrics_plot(type="issue", top_n=5):
304
  print_msg = "discussion"
305
  elif type == "wanted":
306
  metrics_key = 'wanted_issue_monthly_metrics'
307
- total_field = 'resolved_wanted' # Only resolved_wanted is available now
308
  no_data_msg = "No wanted issue data available for visualization"
309
- total_label = "Resolved Wanted Issues"
310
  print_msg = "wanted issue"
311
 
312
  # Load from saved dataset
@@ -390,45 +390,41 @@ def create_monthly_metrics_plot(type="issue", top_n=5):
390
  color = agent_colors[agent_name]
391
  agent_data = data[agent_name]
392
 
393
- # Add line trace for resolved rate (left y-axis) - only if resolved_rates exists
394
- resolved_rates = agent_data.get('resolved_rates')
395
- if resolved_rates:
396
- # Filter out None values for plotting
397
- x_resolved = [month for month, rate in zip(months, resolved_rates) if rate is not None]
398
- y_resolved = [rate for rate in resolved_rates if rate is not None]
399
-
400
- if x_resolved and y_resolved: # Only add trace if there's data
401
- fig.add_trace(
402
- go.Scatter(
403
- x=x_resolved,
404
- y=y_resolved,
405
- name=agent_name,
406
- mode='lines+markers',
407
- line=dict(color=color, width=2),
408
- marker=dict(size=8),
409
- legendgroup=agent_name,
410
- showlegend=(top_n is not None and top_n <= 10), # Show legend for top N assistants
411
- hovertemplate='<b>Assistant: %{fullData.name}</b><br>' +
412
- 'Month: %{x}<br>' +
413
- 'Resolved Rate: %{y:.2f}%<br>' +
414
- '<extra></extra>'
415
- ),
416
- secondary_y=False
417
- )
418
 
419
  # Add bar trace for total count (right y-axis)
420
  # Only show bars for months where assistant has data
421
  x_bars = []
422
  y_bars = []
423
- total_data = agent_data.get(total_field, [])
424
- for month, count in zip(months, total_data):
425
  if count > 0: # Only include months with data
426
  x_bars.append(month)
427
  y_bars.append(count)
428
 
429
  if x_bars and y_bars: # Only add trace if there's data
430
- # For wanted type without resolved_rates, show legend on bar chart
431
- show_bar_legend = (top_n is not None and top_n <= 10) and not resolved_rates
432
  fig.add_trace(
433
  go.Bar(
434
  x=x_bars,
@@ -436,7 +432,7 @@ def create_monthly_metrics_plot(type="issue", top_n=5):
436
  name=agent_name,
437
  marker=dict(color=color, opacity=0.6),
438
  legendgroup=agent_name,
439
- showlegend=show_bar_legend, # Show legend if no line chart was added
440
  hovertemplate=f'<b>Assistant: %{{fullData.name}}</b><br>' +
441
  f'Month: %{{x}}<br>' +
442
  f'{total_label}: %{{y}}<br>' +
@@ -448,25 +444,15 @@ def create_monthly_metrics_plot(type="issue", top_n=5):
448
 
449
  # Update axes labels
450
  fig.update_xaxes(title_text=None)
451
-
452
- # For "wanted" type, there's no resolved_rates, so hide the left y-axis
453
- if type == "wanted":
454
- fig.update_yaxes(
455
- title_text=None,
456
- showticklabels=False,
457
- showgrid=False,
458
- secondary_y=False
459
- )
460
- else:
461
- fig.update_yaxes(
462
- title_text="<b>Resolved Rate (%)</b>",
463
- range=[0, 100],
464
- secondary_y=False,
465
- showticklabels=True,
466
- tickmode='linear',
467
- dtick=10,
468
- showgrid=True
469
- )
470
  fig.update_yaxes(title_text=f"<b>{total_label}</b>", secondary_y=True)
471
 
472
  # Update layout
@@ -726,7 +712,6 @@ with gr.Blocks(title="SWE Assistant Issue & Discussion Leaderboard", theme=gr.th
726
  value=pd.DataFrame(columns=[col[0] for col in LEADERBOARD_COLUMNS]), # Empty initially
727
  datatype=LEADERBOARD_COLUMNS,
728
  search_columns=["Assistant", "Website"],
729
- height=400, # Minimum height to prevent rows from being obscured by filter controls
730
  filter_columns=[
731
  ColumnFilter(
732
  "Issue Resolved Rate (%)",
@@ -776,7 +761,7 @@ with gr.Blocks(title="SWE Assistant Issue & Discussion Leaderboard", theme=gr.th
776
  discussion_metrics_plot = gr.Plot()
777
 
778
  with gr.Column():
779
- gr.Markdown("*Resolved wanted issues over time*")
780
  wanted_metrics_plot = gr.Plot()
781
 
782
  # Load monthly metrics when app starts
 
304
  print_msg = "discussion"
305
  elif type == "wanted":
306
  metrics_key = 'wanted_issue_monthly_metrics'
307
+ total_field = 'total_wanted'
308
  no_data_msg = "No wanted issue data available for visualization"
309
+ total_label = "Total Wanted Issues"
310
  print_msg = "wanted issue"
311
 
312
  # Load from saved dataset
 
390
  color = agent_colors[agent_name]
391
  agent_data = data[agent_name]
392
 
393
+ # Add line trace for resolved rate (left y-axis)
394
+ resolved_rates = agent_data['resolved_rates']
395
+ # Filter out None values for plotting
396
+ x_resolved = [month for month, rate in zip(months, resolved_rates) if rate is not None]
397
+ y_resolved = [rate for rate in resolved_rates if rate is not None]
398
+
399
+ if x_resolved and y_resolved: # Only add trace if there's data
400
+ fig.add_trace(
401
+ go.Scatter(
402
+ x=x_resolved,
403
+ y=y_resolved,
404
+ name=agent_name,
405
+ mode='lines+markers',
406
+ line=dict(color=color, width=2),
407
+ marker=dict(size=8),
408
+ legendgroup=agent_name,
409
+ showlegend=(top_n is not None and top_n <= 10), # Show legend for top N assistants
410
+ hovertemplate='<b>Assistant: %{fullData.name}</b><br>' +
411
+ 'Month: %{x}<br>' +
412
+ 'Resolved Rate: %{y:.2f}%<br>' +
413
+ '<extra></extra>'
414
+ ),
415
+ secondary_y=False
416
+ )
 
417
 
418
  # Add bar trace for total count (right y-axis)
419
  # Only show bars for months where assistant has data
420
  x_bars = []
421
  y_bars = []
422
+ for month, count in zip(months, agent_data[total_field]):
 
423
  if count > 0: # Only include months with data
424
  x_bars.append(month)
425
  y_bars.append(count)
426
 
427
  if x_bars and y_bars: # Only add trace if there's data
 
 
428
  fig.add_trace(
429
  go.Bar(
430
  x=x_bars,
 
432
  name=agent_name,
433
  marker=dict(color=color, opacity=0.6),
434
  legendgroup=agent_name,
435
+ showlegend=False, # Hide duplicate legend entry (already shown in Scatter)
436
  hovertemplate=f'<b>Assistant: %{{fullData.name}}</b><br>' +
437
  f'Month: %{{x}}<br>' +
438
  f'{total_label}: %{{y}}<br>' +
 
444
 
445
  # Update axes labels
446
  fig.update_xaxes(title_text=None)
447
+ fig.update_yaxes(
448
+ title_text="<b>Resolved Rate (%)</b>",
449
+ range=[0, 100],
450
+ secondary_y=False,
451
+ showticklabels=True,
452
+ tickmode='linear',
453
+ dtick=10,
454
+ showgrid=True
455
+ )
 
 
 
 
 
 
 
 
 
 
456
  fig.update_yaxes(title_text=f"<b>{total_label}</b>", secondary_y=True)
457
 
458
  # Update layout
 
712
  value=pd.DataFrame(columns=[col[0] for col in LEADERBOARD_COLUMNS]), # Empty initially
713
  datatype=LEADERBOARD_COLUMNS,
714
  search_columns=["Assistant", "Website"],
 
715
  filter_columns=[
716
  ColumnFilter(
717
  "Issue Resolved Rate (%)",
 
761
  discussion_metrics_plot = gr.Plot()
762
 
763
  with gr.Column():
764
+ gr.Markdown("*Wanted issue volume and resolved rate over time*")
765
  wanted_metrics_plot = gr.Plot()
766
 
767
  # Load monthly metrics when app starts
msr.py CHANGED
@@ -1050,20 +1050,25 @@ def calculate_monthly_metrics_by_discussions(all_discussions_dict, assistants):
1050
  }
1051
 
1052
 
1053
- def calculate_monthly_metrics_by_wanted_issues(wanted_resolved_dict, assistants):
1054
- """Calculate monthly metrics for resolved wanted issues for all assistants for visualization.
 
 
 
 
1055
  """
1056
  identifier_to_name = {assistant.get('github_identifier'): assistant.get('name') for assistant in assistants if assistant.get('github_identifier')}
1057
 
1058
- if not wanted_resolved_dict:
1059
  return {'assistants': [], 'months': [], 'data': {}}
1060
 
1061
  # Calculate the valid date range for filtering
1062
  current_time = datetime.now(timezone.utc)
1063
  start_date = current_time - timedelta(days=LEADERBOARD_TIME_FRAME_DAYS)
1064
  start_month_key = f"{start_date.year}-{start_date.month:02d}"
 
1065
 
1066
- agent_month_data = defaultdict(lambda: defaultdict(list))
1067
 
1068
  # Process resolved wanted issues - group by closed_at date
1069
  for agent_identifier, metadata_list in (wanted_resolved_dict or {}).items():
@@ -1083,11 +1088,17 @@ def calculate_monthly_metrics_by_wanted_issues(wanted_resolved_dict, assistants)
1083
  continue
1084
 
1085
  month_key = f"{dt.year}-{dt.month:02d}"
1086
- agent_month_data[agent_name][month_key].append(issue_meta)
1087
  except Exception as e:
1088
  print(f"Warning: Could not parse wanted issue date '{closed_at}': {e}")
1089
  continue
1090
 
 
 
 
 
 
 
1091
  all_months = set()
1092
  for agent_data in agent_month_data.values():
1093
  all_months.update(agent_data.keys())
@@ -1097,13 +1108,26 @@ def calculate_monthly_metrics_by_wanted_issues(wanted_resolved_dict, assistants)
1097
 
1098
  result_data = {}
1099
  for agent_name, month_dict in agent_month_data.items():
 
 
1100
  resolved_wanted_list = []
1101
 
1102
  for month in months:
1103
- resolved_count = len(month_dict.get(month, []))
 
 
 
 
 
 
 
 
 
1104
  resolved_wanted_list.append(resolved_count)
1105
 
1106
  result_data[agent_name] = {
 
 
1107
  'resolved_wanted': resolved_wanted_list
1108
  }
1109
 
@@ -1288,7 +1312,7 @@ def mine_all_agents():
1288
  agent_discussions, assistants
1289
  )
1290
  wanted_issue_monthly_metrics = calculate_monthly_metrics_by_wanted_issues(
1291
- wanted_resolved, assistants
1292
  )
1293
  save_leaderboard_data_to_hf(
1294
  leaderboard_dict, issue_monthly_metrics, wanted_open, discussion_monthly_metrics, wanted_issue_monthly_metrics
 
1050
  }
1051
 
1052
 
1053
+ def calculate_monthly_metrics_by_wanted_issues(wanted_resolved_dict, wanted_in_progress_dict, assistants):
1054
+ """Calculate monthly metrics for wanted issues for all assistants for visualization.
1055
+
1056
+ Resolved wanted issues are grouped by their closing date.
1057
+ In-progress wanted issues are counted in the current month (representing ongoing work).
1058
+ Only months within LEADERBOARD_TIME_FRAME_DAYS are included.
1059
  """
1060
  identifier_to_name = {assistant.get('github_identifier'): assistant.get('name') for assistant in assistants if assistant.get('github_identifier')}
1061
 
1062
+ if not wanted_resolved_dict and not wanted_in_progress_dict:
1063
  return {'assistants': [], 'months': [], 'data': {}}
1064
 
1065
  # Calculate the valid date range for filtering
1066
  current_time = datetime.now(timezone.utc)
1067
  start_date = current_time - timedelta(days=LEADERBOARD_TIME_FRAME_DAYS)
1068
  start_month_key = f"{start_date.year}-{start_date.month:02d}"
1069
+ current_month_key = f"{current_time.year}-{current_time.month:02d}"
1070
 
1071
+ agent_month_data = defaultdict(lambda: defaultdict(lambda: {'resolved': 0, 'in_progress': 0}))
1072
 
1073
  # Process resolved wanted issues - group by closed_at date
1074
  for agent_identifier, metadata_list in (wanted_resolved_dict or {}).items():
 
1088
  continue
1089
 
1090
  month_key = f"{dt.year}-{dt.month:02d}"
1091
+ agent_month_data[agent_name][month_key]['resolved'] += 1
1092
  except Exception as e:
1093
  print(f"Warning: Could not parse wanted issue date '{closed_at}': {e}")
1094
  continue
1095
 
1096
+ # Process in-progress wanted issues - count in current month (ongoing work)
1097
+ for agent_identifier, metadata_list in (wanted_in_progress_dict or {}).items():
1098
+ agent_name = identifier_to_name.get(agent_identifier, agent_identifier)
1099
+ # All in-progress issues count towards the current month
1100
+ agent_month_data[agent_name][current_month_key]['in_progress'] += len(metadata_list)
1101
+
1102
  all_months = set()
1103
  for agent_data in agent_month_data.values():
1104
  all_months.update(agent_data.keys())
 
1108
 
1109
  result_data = {}
1110
  for agent_name, month_dict in agent_month_data.items():
1111
+ resolved_rates = []
1112
+ total_wanted_list = []
1113
  resolved_wanted_list = []
1114
 
1115
  for month in months:
1116
+ month_data = month_dict.get(month, {'resolved': 0, 'in_progress': 0})
1117
+ resolved_count = month_data['resolved']
1118
+ in_progress_count = month_data['in_progress']
1119
+ total_count = resolved_count + in_progress_count
1120
+
1121
+ # Resolved rate = resolved / total * 100
1122
+ resolved_rate = (resolved_count / total_count * 100) if total_count > 0 else None
1123
+
1124
+ resolved_rates.append(resolved_rate)
1125
+ total_wanted_list.append(total_count)
1126
  resolved_wanted_list.append(resolved_count)
1127
 
1128
  result_data[agent_name] = {
1129
+ 'resolved_rates': resolved_rates,
1130
+ 'total_wanted': total_wanted_list,
1131
  'resolved_wanted': resolved_wanted_list
1132
  }
1133
 
 
1312
  agent_discussions, assistants
1313
  )
1314
  wanted_issue_monthly_metrics = calculate_monthly_metrics_by_wanted_issues(
1315
+ wanted_resolved, wanted_in_progress, assistants
1316
  )
1317
  save_leaderboard_data_to_hf(
1318
  leaderboard_dict, issue_monthly_metrics, wanted_open, discussion_monthly_metrics, wanted_issue_monthly_metrics