zhimin-z commited on
Commit ·
d64c999
1
Parent(s): 53e82b5
add
Browse files
msr.py
CHANGED
|
@@ -1050,16 +1050,22 @@ def calculate_monthly_metrics_by_discussions(all_discussions_dict, assistants):
|
|
| 1050 |
}
|
| 1051 |
|
| 1052 |
|
| 1053 |
-
def calculate_monthly_metrics_by_wanted_issues(wanted_resolved_dict,
|
| 1054 |
-
"""Calculate monthly metrics for wanted issues for all assistants for visualization.
|
|
|
|
| 1055 |
identifier_to_name = {assistant.get('github_identifier'): assistant.get('name') for assistant in assistants if assistant.get('github_identifier')}
|
| 1056 |
|
| 1057 |
-
if not wanted_resolved_dict
|
| 1058 |
return {'assistants': [], 'months': [], 'data': {}}
|
| 1059 |
|
| 1060 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1061 |
|
| 1062 |
-
# Process resolved wanted issues
|
| 1063 |
for agent_identifier, metadata_list in (wanted_resolved_dict or {}).items():
|
| 1064 |
for issue_meta in metadata_list:
|
| 1065 |
closed_at = issue_meta.get('closed_at')
|
|
@@ -1071,62 +1077,34 @@ def calculate_monthly_metrics_by_wanted_issues(wanted_resolved_dict, wanted_in_p
|
|
| 1071 |
|
| 1072 |
try:
|
| 1073 |
dt = datetime.fromisoformat(closed_at.replace('Z', '+00:00'))
|
| 1074 |
-
month_key = f"{dt.year}-{dt.month:02d}"
|
| 1075 |
-
agent_month_data[agent_name][month_key]['resolved'].append(issue_meta)
|
| 1076 |
-
except Exception as e:
|
| 1077 |
-
print(f"Warning: Could not parse wanted issue date '{closed_at}': {e}")
|
| 1078 |
-
continue
|
| 1079 |
|
| 1080 |
-
|
| 1081 |
-
|
| 1082 |
-
|
| 1083 |
-
created_at = issue_meta.get('created_at')
|
| 1084 |
-
|
| 1085 |
-
if not created_at or created_at == 'N/A':
|
| 1086 |
-
continue
|
| 1087 |
-
|
| 1088 |
-
agent_name = identifier_to_name.get(agent_identifier, agent_identifier)
|
| 1089 |
|
| 1090 |
-
try:
|
| 1091 |
-
dt = datetime.fromisoformat(created_at.replace('Z', '+00:00'))
|
| 1092 |
month_key = f"{dt.year}-{dt.month:02d}"
|
| 1093 |
-
agent_month_data[agent_name][month_key]
|
| 1094 |
except Exception as e:
|
| 1095 |
-
print(f"Warning: Could not parse wanted issue date '{
|
| 1096 |
continue
|
| 1097 |
|
| 1098 |
all_months = set()
|
| 1099 |
for agent_data in agent_month_data.values():
|
| 1100 |
all_months.update(agent_data.keys())
|
| 1101 |
-
|
|
|
|
|
|
|
| 1102 |
|
| 1103 |
result_data = {}
|
| 1104 |
for agent_name, month_dict in agent_month_data.items():
|
| 1105 |
-
resolved_rates = []
|
| 1106 |
-
total_wanted_list = []
|
| 1107 |
resolved_wanted_list = []
|
| 1108 |
-
in_progress_wanted_list = []
|
| 1109 |
|
| 1110 |
for month in months:
|
| 1111 |
-
|
| 1112 |
-
|
| 1113 |
-
resolved_count = len(month_data['resolved'])
|
| 1114 |
-
in_progress_count = len(month_data['in_progress'])
|
| 1115 |
-
total_count = resolved_count + in_progress_count
|
| 1116 |
-
|
| 1117 |
-
# Resolved rate = resolved / total * 100
|
| 1118 |
-
resolved_rate = (resolved_count / total_count * 100) if total_count > 0 else None
|
| 1119 |
-
|
| 1120 |
-
resolved_rates.append(resolved_rate)
|
| 1121 |
-
total_wanted_list.append(total_count)
|
| 1122 |
resolved_wanted_list.append(resolved_count)
|
| 1123 |
-
in_progress_wanted_list.append(in_progress_count)
|
| 1124 |
|
| 1125 |
result_data[agent_name] = {
|
| 1126 |
-
'
|
| 1127 |
-
'total_wanted': total_wanted_list,
|
| 1128 |
-
'resolved_wanted': resolved_wanted_list,
|
| 1129 |
-
'in_progress_wanted': in_progress_wanted_list
|
| 1130 |
}
|
| 1131 |
|
| 1132 |
agents_list = sorted(list(agent_month_data.keys()))
|
|
@@ -1310,7 +1288,7 @@ def mine_all_agents():
|
|
| 1310 |
agent_discussions, assistants
|
| 1311 |
)
|
| 1312 |
wanted_issue_monthly_metrics = calculate_monthly_metrics_by_wanted_issues(
|
| 1313 |
-
wanted_resolved,
|
| 1314 |
)
|
| 1315 |
save_leaderboard_data_to_hf(
|
| 1316 |
leaderboard_dict, issue_monthly_metrics, wanted_open, discussion_monthly_metrics, wanted_issue_monthly_metrics
|
|
|
|
| 1050 |
}
|
| 1051 |
|
| 1052 |
|
| 1053 |
+
def calculate_monthly_metrics_by_wanted_issues(wanted_resolved_dict, assistants):
|
| 1054 |
+
"""Calculate monthly metrics for resolved wanted issues for all assistants for visualization.
|
| 1055 |
+
"""
|
| 1056 |
identifier_to_name = {assistant.get('github_identifier'): assistant.get('name') for assistant in assistants if assistant.get('github_identifier')}
|
| 1057 |
|
| 1058 |
+
if not wanted_resolved_dict:
|
| 1059 |
return {'assistants': [], 'months': [], 'data': {}}
|
| 1060 |
|
| 1061 |
+
# Calculate the valid date range for filtering
|
| 1062 |
+
current_time = datetime.now(timezone.utc)
|
| 1063 |
+
start_date = current_time - timedelta(days=LEADERBOARD_TIME_FRAME_DAYS)
|
| 1064 |
+
start_month_key = f"{start_date.year}-{start_date.month:02d}"
|
| 1065 |
+
|
| 1066 |
+
agent_month_data = defaultdict(lambda: defaultdict(list))
|
| 1067 |
|
| 1068 |
+
# Process resolved wanted issues - group by closed_at date
|
| 1069 |
for agent_identifier, metadata_list in (wanted_resolved_dict or {}).items():
|
| 1070 |
for issue_meta in metadata_list:
|
| 1071 |
closed_at = issue_meta.get('closed_at')
|
|
|
|
| 1077 |
|
| 1078 |
try:
|
| 1079 |
dt = datetime.fromisoformat(closed_at.replace('Z', '+00:00'))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1080 |
|
| 1081 |
+
# Skip if closed before the timeframe window
|
| 1082 |
+
if dt < start_date:
|
| 1083 |
+
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1084 |
|
|
|
|
|
|
|
| 1085 |
month_key = f"{dt.year}-{dt.month:02d}"
|
| 1086 |
+
agent_month_data[agent_name][month_key].append(issue_meta)
|
| 1087 |
except Exception as e:
|
| 1088 |
+
print(f"Warning: Could not parse wanted issue date '{closed_at}': {e}")
|
| 1089 |
continue
|
| 1090 |
|
| 1091 |
all_months = set()
|
| 1092 |
for agent_data in agent_month_data.values():
|
| 1093 |
all_months.update(agent_data.keys())
|
| 1094 |
+
|
| 1095 |
+
# Filter to only include months within the timeframe
|
| 1096 |
+
months = sorted([m for m in all_months if m >= start_month_key])
|
| 1097 |
|
| 1098 |
result_data = {}
|
| 1099 |
for agent_name, month_dict in agent_month_data.items():
|
|
|
|
|
|
|
| 1100 |
resolved_wanted_list = []
|
|
|
|
| 1101 |
|
| 1102 |
for month in months:
|
| 1103 |
+
resolved_count = len(month_dict.get(month, []))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1104 |
resolved_wanted_list.append(resolved_count)
|
|
|
|
| 1105 |
|
| 1106 |
result_data[agent_name] = {
|
| 1107 |
+
'resolved_wanted': resolved_wanted_list
|
|
|
|
|
|
|
|
|
|
| 1108 |
}
|
| 1109 |
|
| 1110 |
agents_list = sorted(list(agent_month_data.keys()))
|
|
|
|
| 1288 |
agent_discussions, assistants
|
| 1289 |
)
|
| 1290 |
wanted_issue_monthly_metrics = calculate_monthly_metrics_by_wanted_issues(
|
| 1291 |
+
wanted_resolved, assistants
|
| 1292 |
)
|
| 1293 |
save_leaderboard_data_to_hf(
|
| 1294 |
leaderboard_dict, issue_monthly_metrics, wanted_open, discussion_monthly_metrics, wanted_issue_monthly_metrics
|