refine

msr.py

@@ -283,8 +283,23 @@ def get_duckdb_connection():
     """
     Initialize DuckDB connection with OPTIMIZED memory settings.
     Uses persistent database and reduced memory footprint.
+    Automatically removes cache file if lock conflict is detected.
     """
-    conn = duckdb.connect(DUCKDB_CACHE_FILE)
+    try:
+        conn = duckdb.connect(DUCKDB_CACHE_FILE)
+    except Exception as e:
+        # Check if it's a locking error
+        error_msg = str(e)
+        if "lock" in error_msg.lower() or "conflicting" in error_msg.lower():
+            print(f"   ⚠ Lock conflict detected, removing {DUCKDB_CACHE_FILE}...")
+            if os.path.exists(DUCKDB_CACHE_FILE):
+                os.remove(DUCKDB_CACHE_FILE)
+                print(f"   ✓ Cache file removed, retrying connection...")
+            # Retry connection after removing cache
+            conn = duckdb.connect(DUCKDB_CACHE_FILE)
+        else:
+            # Re-raise if it's not a locking error
+            raise
 
     # OPTIMIZED SETTINGS
     conn.execute(f"SET threads TO {DUCKDB_THREADS};")
@@ -383,16 +398,14 @@ def fetch_all_issue_metadata_streaming(conn, identifiers, start_date, end_date):
         # Build file patterns SQL for THIS BATCH
         file_patterns_sql = '[' + ', '.join([f"'{fp}'" for fp in file_patterns]) + ']'
 
-        # Query for this batch - IssuesEvent filtered by assignee only
+        # Query for this batch - IssuesEvent filtered by assignee (singular) and assignees (array)
         query = f"""
-        WITH issue_events AS (
+        WITH raw_events AS (
             SELECT
-                CONCAT(
-                    REPLACE(repo.url, 'api.github.com/repos/', 'github.com/'),
-                    '/issues/',
-                    CAST(payload.issue.number AS VARCHAR)
-                ) as url,
-                payload.issue.assignee.login as assignee,
+                repo.url,
+                payload.issue.number,
+                payload.issue.assignee.login as single_assignee,
+                payload.issue.assignees as assignees_array,
                 created_at as event_time,
                 payload.issue.created_at as issue_created_at,
                 payload.issue.closed_at as issue_closed_at,
@@ -402,7 +415,41 @@ def fetch_all_issue_metadata_streaming(conn, identifiers, start_date, end_date):
                 type = 'IssuesEvent'
                 AND payload.issue.number IS NOT NULL
                 AND payload.issue.pull_request IS NULL
-                AND payload.issue.assignee.login IN ({identifier_list})
+        ),
+        -- Extract assignees from both single assignee field and assignees array
+        issue_events AS (
+            -- Get from single assignee field
+            SELECT
+                CONCAT(
+                    REPLACE(url, 'api.github.com/repos/', 'github.com/'),
+                    '/issues/',
+                    CAST(number AS VARCHAR)
+                ) as url,
+                single_assignee as assignee,
+                event_time,
+                issue_created_at,
+                issue_closed_at,
+                state_reason
+            FROM raw_events
+            WHERE single_assignee IN ({identifier_list})
+
+            UNION ALL
+
+            -- Get from assignees array
+            SELECT
+                CONCAT(
+                    REPLACE(url, 'api.github.com/repos/', 'github.com/'),
+                    '/issues/',
+                    CAST(number AS VARCHAR)
+                ) as url,
+                assignee_data.login as assignee,
+                event_time,
+                issue_created_at,
+                issue_closed_at,
+                state_reason
+            FROM raw_events,
+            UNNEST(assignees_array) as assignee_data
+            WHERE assignee_data.login IN ({identifier_list})
         ),
         issue_timeline AS (
             SELECT