Update app.py

app.py

@@ -6,9 +6,7 @@ import time
 import os
 
 def get_contribution_count(username, github_token, days=30):
-    """
-    Get accurate contribution count using GitHub GraphQL API
-    """
+    """Get accurate contribution count using GitHub GraphQL API"""
     if not github_token:
         return 0
     
@@ -42,7 +40,8 @@ def get_contribution_count(username, github_token, days=30):
         response = requests.post(
             'https://api.github.com/graphql',
             json={'query': query, 'variables': variables},
-            headers=headers
+            headers=headers,
+            timeout=10
         )
         
         if response.status_code == 200:
@@ -54,131 +53,151 @@ def get_contribution_count(username, github_token, days=30):
     
     return 0
 
-def fetch_github_users(github_token=None, max_users=200, min_followers=10, days=30):
+def fetch_github_users(github_token=None, max_users=200, min_followers=0, days=30):
     """
-    Fetch GitHub users from UAE directly using GitHub API
-    Sorted by contributions (primary) and followers (secondary)
+    Fetch GitHub users from UAE using committers.top methodology:
+    1. Search by location + sort by followers
+    2. Get top N users by followers
+    3. Get their contributions
+    4. Re-sort by contributions
     """
     if not github_token:
         github_token = os.getenv('GITHUB_TOKEN')
+    
+    if not github_token:
+        return pd.DataFrame(), "❌ GitHub token is REQUIRED for GraphQL API (contribution counting)"
 
     headers = {
         'Accept': 'application/vnd.github.v3+json',
+        'Authorization': f'token {github_token}'
     }
 
-    if github_token:
-        headers['Authorization'] = f'token {github_token}'
-
-    all_users = []
+    all_users = {}  # Use dict to avoid duplicates
+    status_updates = []
+    api_calls = 0
 
-    # Search locations in UAE - KEEP THE ORIGINAL WORKING APPROACH
+    # UAE location search terms
     locations = [
-        'Dubai', 'Abu Dhabi', 'Sharjah', 'Ajman',
         'United Arab Emirates', 'UAE',
+        'Dubai', 'Abu Dhabi', 'Sharjah', 'Ajman',
         'Ras Al Khaimah', 'Fujairah', 'Umm Al Quwain'
     ]
 
-    status_updates = []
-
     try:
+        status_updates.append(f"🔍 Step 1: Searching for UAE users (sorted by followers)...")
+        
+        # STEP 1: Get users sorted by FOLLOWERS (like committers.top does)
         for location in locations:
-            status_updates.append(f"🔍 Searching users in {location}...")
-
-            # Search users by location and followers
-            search_url = f'https://api.github.com/search/users?q=location:{location}+followers:>={min_followers}&sort=followers&order=desc&per_page=100'
-
+            if api_calls >= 100:
+                break
+            
+            # Search by followers DESC
+            search_url = f'https://api.github.com/search/users?q=location:"{location}"+followers:>={min_followers}&sort=followers&order=desc&per_page=100'
+            
             response = requests.get(search_url, headers=headers)
-
+            api_calls += 1
+            
             if response.status_code == 200:
                 data = response.json()
                 users = data.get('items', [])
-
+                status_updates.append(f"  📍 {location}: {len(users)} users")
+                
                 for user in users:
-                    if not any(u['login'] == user['login'] for u in all_users):
-                        # Fetch detailed user info
-                        user_url = user['url']
-                        user_response = requests.get(user_url, headers=headers)
-
+                    if user['login'] not in all_users and api_calls < 100:
+                        # Get detailed user info
+                        user_response = requests.get(user['url'], headers=headers)
+                        api_calls += 1
+                        
                         if user_response.status_code == 200:
                             user_data = user_response.json()
                             
-                            # Verify location contains UAE-related keywords (case insensitive)
+                            # Verify UAE location (case insensitive)
                             user_location = (user_data.get('location') or '').lower()
-                            # Keywords are already lowercase for comparison
-                            uae_keywords = ['dubai', 'abu dhabi', 'sharjah', 'ajman', 'uae', 'u.a.e', 
-                                           'united arab emirates', 'ras al khaimah', 'fujairah', 
-                                           'umm al quwain', 'emirates', 'abudhabi', 'rak']
+                            uae_keywords = ['dubai', 'abu dhabi', 'abudhabi', 'sharjah', 'ajman', 
+                                           'uae', 'u.a.e', 'united arab emirates', 'ras al khaimah', 
+                                           'fujairah', 'umm al quwain', 'emirates', 'rak']
                             
-                            # Check if any UAE keyword exists in the lowercased location
-                            if not any(keyword in user_location for keyword in uae_keywords):
-                                continue
-
-                            # Get accurate contribution count using GraphQL (FIX: Use GraphQL instead of Events API)
-                            contributions = get_contribution_count(user['login'], github_token, days)
-
-                            all_users.append({
-                                'login': user_data.get('login', ''),
-                                'name': user_data.get('name', user_data.get('login', '')),
-                                'avatar': user_data.get('avatar_url', ''),
-                                'followers': user_data.get('followers', 0),
-                                'public_repos': user_data.get('public_repos', 0),
-                                'contributions': contributions,
-                                'location': user_data.get('location', ''),
-                                'bio': user_data.get('bio', ''),
-                                'company': user_data.get('company', ''),
-                            })
-
-                            time.sleep(0.5)  # Rate limiting
-
-                        if len(all_users) >= max_users * 2:  # Fetch more to ensure we get top contributors
-                            break
-
-                status_updates.append(f"✅ Found {len(users)} users in {location}")
-                time.sleep(1)  # Rate limiting between searches
-
+                            if any(kw in user_location for kw in uae_keywords):
+                                all_users[user['login']] = {
+                                    'login': user_data.get('login', ''),
+                                    'name': user_data.get('name') or user_data.get('login', ''),
+                                    'avatar': user_data.get('avatar_url', ''),
+                                    'followers': user_data.get('followers', 0),
+                                    'public_repos': user_data.get('public_repos', 0),
+                                    'location': user_data.get('location', ''),
+                                    'contributions': 0  # Will fill later
+                                }
+                        
+                        time.sleep(0.5)
+                
+                time.sleep(1)
+            
             elif response.status_code == 403:
-                status_updates.append(f"⚠️ Rate limit reached. Please add a GitHub token.")
+                status_updates.append(f"⚠️ Rate limit hit")
                 break
-            else:
-                status_updates.append(f"❌ Error searching {location}: {response.status_code}")
-
-            if len(all_users) >= max_users * 2:
+        
+        status_updates.append(f"✅ Found {len(all_users)} unique UAE users")
+        
+        # STEP 2: Get contributions for collected users
+        status_updates.append(f"📊 Step 2: Fetching contribution counts...")
+        
+        count = 0
+        for login, user in all_users.items():
+            if api_calls >= 200:
                 break
-
-        # Sort by CONTRIBUTIONS FIRST (descending), then by followers (descending)
-        all_users.sort(key=lambda x: (x['contributions'], x['followers']), reverse=True)
-
-        # Take top contributors based on max_users
-        top_users = all_users[:max_users]
-
-        # Add rank based on contribution order
+            
+            contributions = get_contribution_count(login, github_token, days)
+            user['contributions'] = contributions
+            
+            count += 1
+            if count % 20 == 0:
+                status_updates.append(f"  ⏳ Processed {count}/{len(all_users)} users...")
+            
+            time.sleep(0.5)
+        
+        # STEP 3: Sort by CONTRIBUTIONS (like committers.top final step)
+        users_list = list(all_users.values())
+        users_list.sort(key=lambda x: (x['contributions'], x['followers']), reverse=True)
+        
+        # Take top N
+        top_users = users_list[:max_users]
+        
+        # Add rank
         for i, user in enumerate(top_users, 1):
             user['rank'] = i
-
-        # Convert to DataFrame
+        
+        # Create DataFrame
         df = pd.DataFrame(top_users)
-
+        
         if not df.empty:
             display_df = df[['rank', 'name', 'login', 'contributions', 'followers', 'public_repos', 'location']].copy()
             display_df.columns = ['Rank', 'Name', 'Username', 'Contributions', 'Followers', 'Public Repos', 'Location']
             display_df['GitHub Profile'] = df['login'].apply(lambda x: f"https://github.com/{x}")
-
-            status_message = f"✅ Successfully fetched top {len(df)} contributors (last {days} days)\n" + "\n".join(status_updates[-5:])
+            
+            status_message = f"""✅ Success! Top {len(df)} UAE contributors (last {days} days)
+📊 Total UAE users found: {len(all_users)}
+🔧 API calls used: {api_calls}
+
+Methodology (like committers.top):
+1. Search by location + sort by followers
+2. Get top users by followers  
+3. Fetch contribution counts
+4. Re-sort by contributions
+
+""" + "\n".join(status_updates[-5:])
+            
             return display_df, status_message
         else:
-            return pd.DataFrame(), "⚠️ No users found\n" + "\n".join(status_updates)
-
+            return pd.DataFrame(), "⚠️ No users found"
+    
     except Exception as e:
-        return pd.DataFrame(), f"❌ Error: {str(e)}\n" + "\n".join(status_updates)
+        return pd.DataFrame(), f"❌ Error: {str(e)}"
 
 def search_users(df, search_term):
-    """Filter users based on search term"""
-    if df is None or df.empty:
+    """Filter users"""
+    if df is None or df.empty or not search_term:
         return df
-
-    if not search_term:
-        return df
-
+    
     search_term = search_term.lower()
     mask = (
         df['Name'].str.lower().str.contains(search_term, na=False) |
@@ -186,25 +205,27 @@ def search_users(df, search_term):
     )
     return df[mask]
 
-# Create Gradio interface
+# Gradio interface
 with gr.Blocks(theme=gr.themes.Soft(), title="GitHub UAE Top Contributors") as app:
 
     gr.Markdown("""
-    # 🏆 Top 200 GitHub Contributors in UAE (Last 30 Days)
-    ### Ranked by Contributions Using GitHub GraphQL API
-
-    **This version uses the WORKING search from v1 + ACCURATE GraphQL contribution counting**
+    # 🇦🇪 GitHub UAE Top Contributors
+    ### Using committers.top Methodology
+    
+    **Approach:** Search by location → Sort by followers → Get contributions → Re-sort by contributions
+    
+    **Reference:** [committers.top/uae](https://committers.top/uae.html) found 31,463 UAE users!
     """)
 
     with gr.Row():
         token_input = gr.Textbox(
-            label="GitHub Personal Access Token (REQUIRED for GraphQL)",
-            placeholder="ghp_xxxxxxxxxxxx or leave empty to use GITHUB_TOKEN env var",
+            label="GitHub Token (REQUIRED - needs 'read:user' scope)",
+            placeholder="ghp_xxxx or set GITHUB_TOKEN env var",
             type="password",
             scale=3
         )
         max_users_input = gr.Slider(
-            label="Max Contributors",
+            label="Max Users to Display",
             minimum=10,
             maximum=500,
             value=200,
@@ -214,37 +235,34 @@ with gr.Blocks(theme=gr.themes.Soft(), title="GitHub UAE Top Contributors") as a
 
     with gr.Row():
         min_followers_input = gr.Slider(
-            label="Minimum Followers (for initial search)",
+            label="Min Followers (committers.top uses 34)",
             minimum=0,
             maximum=100,
-            value=10,
+            value=0,
             step=1
         )
         days_input = gr.Slider(
-            label="Days to Count (30 = last 30 days)",
+            label="Days to Count",
             minimum=7,
             maximum=90,
             value=30,
             step=1
         )
 
-    with gr.Row():
-        fetch_btn = gr.Button("🚀 Fetch Top Contributors from GitHub", variant="primary", size="lg")
-
-    status_msg = gr.Textbox(label="Status", interactive=False, lines=3)
+    fetch_btn = gr.Button("🚀 Fetch Top Contributors", variant="primary", size="lg")
+    
+    status_msg = gr.Textbox(label="Status", interactive=False, lines=6)
 
     with gr.Row():
         search_box = gr.Textbox(
-            label="🔍 Search by Name or Username",
-            placeholder="Type to search...",
+            label="🔍 Filter Results",
+            placeholder="Search by name or username...",
             scale=4
         )
         clear_btn = gr.Button("Clear", scale=1)
 
-    # Store the full dataframe
     full_data = gr.State(value=pd.DataFrame())
-
-    # Display dataframe
+    
     data_display = gr.Dataframe(
         headers=["Rank", "Name", "Username", "Contributions", "Followers", "Public Repos", "Location", "GitHub Profile"],
         datatype=["number", "str", "str", "number", "number", "number", "str", "str"],
@@ -254,40 +272,43 @@ with gr.Blocks(theme=gr.themes.Soft(), title="GitHub UAE Top Contributors") as a
 
     gr.Markdown("""
     ---
-    **What Changed from Original:**
-    - ✅ Kept the WORKING search approach (searches Dubai, Abu Dhabi, Sharjah, UAE, etc.)
-    - ✅ Fixed contribution counting: Now uses GitHub GraphQL API for ACCURATE counts
-    - ✅ GraphQL returns the EXACT number from user's contribution graph (not estimated from events)
+    ## How This Works:
     
-    **Original Problem:**
-    - Events API only returns ~100 events and misses many contributions
-    - GraphQL API returns the actual total contributions for the date range
+    **Methodology (same as committers.top):**
+    ```
+    1. Search: location:UAE + sort:followers
+    2. Filter: Verify location has UAE keywords
+    3. Collect: Top N users by followers
+    4. Count: Get contributions via GraphQL
+    5. Sort: Re-rank by contributions
+    ```
     
-    **Ranking:**
-    - Primary: Contributions in last N days (accurate from GraphQL)
-    - Secondary: Followers count
+    **Why This Finds Everyone:**
+    - Searches multiple UAE locations
+    - Sorts by followers first (GitHub's default)
+    - Gets top users who are likely active
+    - Then accurately counts contributions
+    - Re-sorts by actual contributions
     
-    **Token Setup (REQUIRED):**
-    1. Go to [GitHub Settings > Tokens](https://github.com/settings/tokens)
+    **Token Setup:**
+    1. Go to: https://github.com/settings/tokens
     2. Generate new token (classic)
-    3. Select scope: `read:user` (REQUIRED for GraphQL)
-    4. Copy and paste above OR set `GITHUB_TOKEN` env var
+    3. **Must have `read:user` scope** (for GraphQL)
+    4. Copy and paste above
+    
+    **Expected Results:**
+    - Should find: tschm (~10k), zning1994 (~9k), YASSERRMD (~788), nkapila6 (~446)
+    - All users from committers.top list should appear
     
-    **Why Token is Required:**
-    - REST API works without token but with rate limits
-    - GraphQL API REQUIRES authentication
-    - Without token: can search users but contributions will be 0
+    **Note:** Takes 2-5 minutes. Be patient!
     """)
 
-    # Event handlers
     def fetch_and_display(token, max_users, min_followers, days):
-        df, msg = fetch_github_users(token if token else None, int(max_users), int(min_followers), int(days))
+        df, msg = fetch_github_users(token or None, int(max_users), int(min_followers), int(days))
         return df, df, msg
 
     def filter_data(df, search):
-        if df is None or df.empty:
-            return df
-        return search_users(df, search)
+        return search_users(df, search) if df is not None else df
 
     def clear_search(df):
         return "", df
@@ -310,6 +331,5 @@ with gr.Blocks(theme=gr.themes.Soft(), title="GitHub UAE Top Contributors") as a
         outputs=[search_box, data_display]
     )
 
-# Launch the app
 if __name__ == "__main__":
     app.launch()