add

.gitignore

@@ -1,3 +1,4 @@
+*.vscode
 *.claude
 *.env
 *.venv

app.py

@@ -3,7 +3,6 @@ from gradio_leaderboard import Leaderboard, ColumnFilter
 import json
 import os
 import time
-import subprocess
 import requests
 from huggingface_hub import HfApi, hf_hub_download
 from huggingface_hub.errors import HfHubHTTPError
@@ -18,7 +17,7 @@ from apscheduler.triggers.cron import CronTrigger
 from datetime import datetime, timezone
 
 # Load environment variables
-load_dotenv()
+load_dotenv(override=True)
 
 # =============================================================================
 # CONFIGURATION
@@ -37,7 +36,9 @@ LEADERBOARD_COLUMNS = [
     ("Total Discussions", "number"),
     ("Issue Resolved Rate (%)", "number"),
     ("Discussion Resolved Rate (%)", "number"),
+    ("Total Wanted Issues", "number"),
     ("Resolved Wanted Issues", "number"),
+    ("Wanted Issue Resolved Rate (%)", "number"),
     ("Resolved Issues", "number"),
     ("Resolved Discussions", "number"),
 ]
@@ -495,11 +496,12 @@ def get_leaderboard_dataframe():
     for identifier, data in cache_dict.items():
         total_issues = data.get('total_issues', 0)
         total_discussions = data.get('total_discussions', 0)
+        total_wanted_issues = data.get('total_wanted_issues', 0)
         resolved_wanted_issues = data.get('resolved_wanted_issues', 0)
-        print(f"   Assistant '{identifier}': {total_issues} issues, {total_discussions} discussions, {resolved_wanted_issues} wanted issues resolved")
+        wanted_issue_resolved_rate = data.get('wanted_issue_resolved_rate', 0.0)
 
-        # Filter out assistants with no activity (all three metrics are zero)
-        if total_issues == 0 and total_discussions == 0 and resolved_wanted_issues == 0:
+        # Filter out assistants with no activity (all four metrics are zero)
+        if total_issues == 0 and total_discussions == 0 and total_wanted_issues == 0:
             filtered_count += 1
             continue
 
@@ -511,7 +513,9 @@ def get_leaderboard_dataframe():
             total_discussions,  # Total Discussions
             data.get('resolved_rate', 0.0),  # Issue Resolved Rate (%)
             data.get('discussion_resolved_rate', 0.0),  # Discussion Resolved Rate (%)
+            total_wanted_issues,  # Total Wanted Issues
             resolved_wanted_issues,  # Resolved Wanted Issues
+            wanted_issue_resolved_rate,  # Wanted Issue Resolved Rate (%)
             data.get('resolved_issues', 0),  # Resolved Issues
             data.get('resolved_discussions', 0),  # Resolved Discussions
         ])
@@ -527,15 +531,16 @@ def get_leaderboard_dataframe():
     numeric_cols = [
         "Total Issues", "Total Discussions",
         "Issue Resolved Rate (%)", "Discussion Resolved Rate (%)",
-        "Resolved Issues", "Resolved Discussions", "Resolved Wanted Issues"
+        "Total Wanted Issues", "Resolved Wanted Issues", "Wanted Issue Resolved Rate (%)",
+        "Resolved Issues", "Resolved Discussions"
     ]
     for col in numeric_cols:
         if col in df.columns:
             df[col] = pd.to_numeric(df[col], errors='coerce').fillna(0)
 
-    # Sort by Total Issues descending
-    if "Total Issues" in df.columns and not df.empty:
-        df = df.sort_values(by="Total Issues", ascending=False).reset_index(drop=True)
+    # Sort by Total Wanted Issues descending (primary metric for this leaderboard)
+    if "Total Wanted Issues" in df.columns and not df.empty:
+        df = df.sort_values(by="Total Wanted Issues", ascending=False).reset_index(drop=True)
 
     print(f"Final DataFrame shape: {df.shape}")
     print("="*60 + "\n")
@@ -706,22 +711,16 @@ with gr.Blocks(title="SWE Assistant Issue & Discussion Leaderboard", theme=gr.th
                 value=pd.DataFrame(columns=[col[0] for col in LEADERBOARD_COLUMNS]),  # Empty initially
                 datatype=LEADERBOARD_COLUMNS,
                 search_columns=["Assistant", "Website"],
+                # Note: Slider filters with min=0 have boundary issues in gradio_leaderboard
+                # Using min=-1 as workaround to include records with 0% rates
                 filter_columns=[
                     ColumnFilter(
-                        "Issue Resolved Rate (%)",
+                        "Wanted Issue Resolved Rate (%)",
                         min=-1,
-                        max=101,
-                        default=[-1, 101],
+                        max=100,
+                        default=[-1, 100],
                         type="slider",
-                        label="Issue Resolved Rate (%)"
-                    ),
-                    ColumnFilter(
-                        "Discussion Resolved Rate (%)",
-                        min=-1,
-                        max=101,
-                        default=[-1, 101],
-                        type="slider",
-                        label="Discussion Resolved Rate (%)"
+                        label="Wanted Issue Resolved Rate (%)"
                     )
                 ]
             )

msr.py

@@ -1,6 +1,5 @@
 import json
 import os
-import time
 from datetime import datetime, timezone, timedelta
 from collections import defaultdict
 from huggingface_hub import HfApi, hf_hub_download
@@ -14,11 +13,10 @@ from apscheduler.schedulers.blocking import BlockingScheduler
 from apscheduler.triggers.cron import CronTrigger
 import logging
 import traceback
-import subprocess
 import re
 
 # Load environment variables
-load_dotenv()
+load_dotenv(override=True)
 
 # =============================================================================
 # CONFIGURATION
@@ -753,7 +751,8 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
     print(f"\n   Post-processing {len(all_issues)} wanted issues...")
 
     wanted_open = []
-    wanted_resolved = defaultdict(list)
+    wanted_resolved = defaultdict(list)  # agent_id -> [resolved wanted issues]
+    wanted_in_progress = defaultdict(list)  # agent_id -> [in-progress wanted issues (PR not merged)]
     current_time = datetime.now(timezone.utc)
 
     for issue_url, issue_meta in all_issues.items():
@@ -762,74 +761,90 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
         if not linked_prs:
             continue
 
-        # Check if any linked PR was merged AND created by an assistant
-        resolved_by = None
+        # Find all agents who have PRs for this issue and their merge status
+        agents_with_merged_pr = set()
+        agents_with_unmerged_pr = set()
+
         for pr_url in linked_prs:
+            pr_creator = pr_creators.get(pr_url)
+            if pr_creator not in identifier_set:
+                continue
+
             merged_at = pr_merged_at.get(pr_url)
             if merged_at:  # PR was merged
-                pr_creator = pr_creators.get(pr_url)
-                if pr_creator in identifier_set:
-                    resolved_by = pr_creator
+                agents_with_merged_pr.add(pr_creator)
+            else:  # PR not merged (in progress or rejected)
+                agents_with_unmerged_pr.add(pr_creator)
+
+        # Skip if no agent has a PR for this issue
+        if not agents_with_merged_pr and not agents_with_unmerged_pr:
+            continue
+
+        # Check if issue qualifies as a "wanted" issue (has patch label)
+        issue_labels = issue_meta.get('labels', [])
+        has_patch_label = False
+        for issue_label in issue_labels:
+            for wanted_label in PATCH_WANTED_LABELS:
+                if wanted_label.lower() in issue_label:
+                    has_patch_label = True
                     break
+            if has_patch_label:
+                break
+
+        if not has_patch_label:
+            continue
+
+        # Check if long-standing (open 30+ days)
+        created_at_str = issue_meta.get('created_at')
+        if not created_at_str or created_at_str == 'N/A':
+            continue
 
-        if not resolved_by:
+        try:
+            created_dt = datetime.fromisoformat(created_at_str.replace('Z', '+00:00'))
+        except:
             continue
 
         # Process based on issue state
         if issue_meta['state'] == 'open':
-            # For open issues: check if labels match PATCH_WANTED_LABELS
-            issue_labels = issue_meta.get('labels', [])
-            has_patch_label = False
-            for issue_label in issue_labels:
-                for wanted_label in PATCH_WANTED_LABELS:
-                    if wanted_label.lower() in issue_label:
-                        has_patch_label = True
-                        break
-                if has_patch_label:
-                    break
-
-            if not has_patch_label:
+            days_open = (current_time - created_dt).days
+            if days_open < LONGSTANDING_GAP_DAYS:
                 continue
 
-            # Check if long-standing
-            created_at_str = issue_meta.get('created_at')
-            if created_at_str and created_at_str != 'N/A':
-                try:
-                    created_dt = datetime.fromisoformat(created_at_str.replace('Z', '+00:00'))
-                    days_open = (current_time - created_dt).days
-                    if days_open >= LONGSTANDING_GAP_DAYS:
-                        wanted_open.append(issue_meta)
-                except:
-                    pass
+            # Add to global wanted_open list
+            wanted_open.append(issue_meta)
+
+            # Track in-progress for agents with unmerged PRs (excluding those who already resolved it)
+            for agent_id in agents_with_unmerged_pr - agents_with_merged_pr:
+                wanted_in_progress[agent_id].append(issue_meta)
 
         elif issue_meta['state'] == 'closed':
-            # For closed issues: must be closed within time frame AND open 30+ days
             closed_at_str = issue_meta.get('closed_at')
-            created_at_str = issue_meta.get('created_at')
-
-            if closed_at_str and closed_at_str != 'N/A' and created_at_str and created_at_str != 'N/A':
-                try:
-                    closed_dt = datetime.fromisoformat(closed_at_str.replace('Z', '+00:00'))
-                    created_dt = datetime.fromisoformat(created_at_str.replace('Z', '+00:00'))
+            if not closed_at_str or closed_at_str == 'N/A':
+                continue
 
-                    # Calculate how long the issue was open
-                    days_open = (closed_dt - created_dt).days
+            try:
+                closed_dt = datetime.fromisoformat(closed_at_str.replace('Z', '+00:00'))
+                days_open = (closed_dt - created_dt).days
 
-                    # Only include if closed within timeframe AND was open 30+ days
-                    if start_date <= closed_dt <= end_date and days_open >= LONGSTANDING_GAP_DAYS:
-                        wanted_resolved[resolved_by].append(issue_meta)
-                except:
-                    pass
+                # Only include if closed within timeframe AND was open 30+ days
+                if start_date <= closed_dt <= end_date and days_open >= LONGSTANDING_GAP_DAYS:
+                    # Track resolved for agents with merged PRs
+                    for agent_id in agents_with_merged_pr:
+                        wanted_resolved[agent_id].append(issue_meta)
+            except:
+                pass
 
     print(f"   ✓ Found {sum(len(issues) for issues in agent_issues.values())} assistant-assigned issues across {len(agent_issues)} assistants")
     print(f"   ✓ Found {len(wanted_open)} long-standing open wanted issues")
     print(f"   ✓ Found {sum(len(issues) for issues in wanted_resolved.values())} resolved wanted issues across {len(wanted_resolved)} assistants")
+    print(f"   ✓ Found {sum(len(issues) for issues in wanted_in_progress.values())} in-progress wanted issues across {len(wanted_in_progress)} assistants")
     print(f"   ✓ Found {sum(len(discussions) for discussions in discussions_by_agent.values())} discussions across {len(discussions_by_agent)} assistants")
 
     return {
         'agent_issues': dict(agent_issues),
         'wanted_open': wanted_open,
         'wanted_resolved': dict(wanted_resolved),
+        'wanted_in_progress': dict(wanted_in_progress),
         'agent_discussions': dict(discussions_by_agent)
     }
 
@@ -1049,13 +1064,14 @@ def calculate_monthly_metrics_by_agent_discussions(all_discussions_dict, assista
     }
 
 
-def construct_leaderboard_from_metadata(all_metadata_dict, assistants, wanted_resolved_dict=None, discussions_dict=None):
+def construct_leaderboard_from_metadata(all_metadata_dict, assistants, wanted_resolved_dict=None, wanted_in_progress_dict=None, discussions_dict=None):
     """Construct leaderboard from in-memory issue metadata and discussion metadata.
 
     Args:
         all_metadata_dict: Dictionary mapping assistant ID to list of issue metadata (assistant-assigned issues)
         assistants: List of assistant metadata
         wanted_resolved_dict: Optional dictionary mapping assistant ID to list of resolved wanted issues
+        wanted_in_progress_dict: Optional dictionary mapping assistant ID to list of in-progress wanted issues
         discussions_dict: Optional dictionary mapping assistant ID to list of discussion metadata
     """
     if not assistants:
@@ -1065,6 +1081,9 @@ def construct_leaderboard_from_metadata(all_metadata_dict, assistants, wanted_re
     if wanted_resolved_dict is None:
         wanted_resolved_dict = {}
 
+    if wanted_in_progress_dict is None:
+        wanted_in_progress_dict = {}
+
     if discussions_dict is None:
         discussions_dict = {}
 
@@ -1077,8 +1096,11 @@ def construct_leaderboard_from_metadata(all_metadata_dict, assistants, wanted_re
         bot_data = all_metadata_dict.get(identifier, [])
         stats = calculate_issue_stats_from_metadata(bot_data)
 
-        # Add wanted issues count
+        # Add wanted issues stats
         resolved_wanted = len(wanted_resolved_dict.get(identifier, []))
+        in_progress_wanted = len(wanted_in_progress_dict.get(identifier, []))
+        total_wanted = resolved_wanted + in_progress_wanted
+        wanted_resolved_rate = (resolved_wanted / total_wanted * 100) if total_wanted > 0 else 0.0
 
         # Add discussion stats
         discussion_metadata = discussions_dict.get(identifier, [])
@@ -1089,7 +1111,9 @@ def construct_leaderboard_from_metadata(all_metadata_dict, assistants, wanted_re
             'website': assistant.get('website', 'N/A'),
             'github_identifier': identifier,
             **stats,
+            'total_wanted_issues': total_wanted,
             'resolved_wanted_issues': resolved_wanted,
+            'wanted_issue_resolved_rate': round(wanted_resolved_rate, 2),
             **discussion_stats
         }
 
@@ -1191,6 +1215,7 @@ def mine_all_agents():
         agent_issues = results['agent_issues']
         wanted_open = results['wanted_open']
         wanted_resolved = results['wanted_resolved']
+        wanted_in_progress = results['wanted_in_progress']
         agent_discussions = results['agent_discussions']
     except Exception as e:
         print(f"Error during DuckDB fetch: {str(e)}")
@@ -1203,7 +1228,7 @@ def mine_all_agents():
 
     try:
         leaderboard_dict = construct_leaderboard_from_metadata(
-            agent_issues, assistants, wanted_resolved, agent_discussions
+            agent_issues, assistants, wanted_resolved, wanted_in_progress, agent_discussions
         )
         issue_monthly_metrics = calculate_monthly_metrics_by_agent(agent_issues, assistants)
         discussion_monthly_metrics = calculate_monthly_metrics_by_agent_discussions(