Update app.py

app.py

@@ -5,24 +5,39 @@ from datetime import datetime, timedelta
 import time
 import os
 
-def get_contribution_count(username, github_token, days=30):
+def get_contribution_count_graphql(username, github_token, days=30):
     """
-    Get accurate contribution count using GitHub GraphQL API
+    Get ACCURATE contribution count using GitHub GraphQL API with proper date range
     """
     if not github_token:
-        return 0
+        return 0, "No token provided"
     
-    # Calculate date range
+    # Calculate exact date range
     end_date = datetime.now()
     start_date = end_date - timedelta(days=days)
     
+    # Format dates in ISO format with timezone
+    from_date = start_date.strftime('%Y-%m-%dT%H:%M:%SZ')
+    to_date = end_date.strftime('%Y-%m-%dT%H:%M:%SZ')
+    
+    # GraphQL query - get detailed contribution data
     query = """
     query($username: String!, $from: DateTime!, $to: DateTime!) {
       user(login: $username) {
         contributionsCollection(from: $from, to: $to) {
           contributionCalendar {
             totalContributions
+            weeks {
+              contributionDays {
+                contributionCount
+                date
+              }
+            }
           }
+          totalCommitContributions
+          totalIssueContributions
+          totalPullRequestContributions
+          totalPullRequestReviewContributions
         }
       }
     }
@@ -30,8 +45,8 @@ def get_contribution_count(username, github_token, days=30):
     
     variables = {
         "username": username,
-        "from": start_date.isoformat(),
-        "to": end_date.isoformat()
+        "from": from_date,
+        "to": to_date
     }
     
     headers = {
@@ -43,17 +58,39 @@ def get_contribution_count(username, github_token, days=30):
         response = requests.post(
             'https://api.github.com/graphql',
             json={'query': query, 'variables': variables},
-            headers=headers
+            headers=headers,
+            timeout=10
         )
         
         if response.status_code == 200:
             data = response.json()
-            if 'data' in data and data['data']['user']:
-                return data['data']['user']['contributionsCollection']['contributionCalendar']['totalContributions']
+            
+            # Check for errors in response
+            if 'errors' in data:
+                return 0, f"GraphQL Error: {data['errors']}"
+            
+            if 'data' in data and data['data'] and data['data']['user']:
+                contrib_collection = data['data']['user']['contributionsCollection']
+                
+                # Get the total contributions from the calendar
+                total = contrib_collection['contributionCalendar']['totalContributions']
+                
+                # Detailed breakdown (for debugging)
+                commits = contrib_collection.get('totalCommitContributions', 0)
+                issues = contrib_collection.get('totalIssueContributions', 0)
+                prs = contrib_collection.get('totalPullRequestContributions', 0)
+                reviews = contrib_collection.get('totalPullRequestReviewContributions', 0)
+                
+                debug_info = f"Total: {total} (Commits: {commits}, Issues: {issues}, PRs: {prs}, Reviews: {reviews})"
+                
+                return total, debug_info
+            else:
+                return 0, "User not found or no data"
+        else:
+            return 0, f"HTTP Error {response.status_code}: {response.text[:200]}"
+            
     except Exception as e:
-        print(f"Error fetching contributions for {username}: {e}")
-    
-    return 0
+        return 0, f"Exception: {str(e)}"
 
 def is_uae_location(location):
     """
@@ -104,16 +141,16 @@ def is_uae_location(location):
     
     return False
 
-def fetch_github_users(github_token=None, max_users=200, min_followers=1, days=30):
+def fetch_github_users(github_token=None, max_users=200, min_followers=0, days=30):
     """
-    Fetch GitHub users from UAE - Country-focused search
+    Fetch GitHub users from UAE with ACCURATE contribution counts
     """
     # Read token from environment variable if not provided
     if not github_token:
         github_token = os.getenv('GITHUB_TOKEN')
     
     if not github_token:
-        return pd.DataFrame(), "❌ GitHub token is required for accurate contribution counts. Please provide a token."
+        return pd.DataFrame(), "❌ GitHub token is REQUIRED. Please provide a token with 'read:user' scope."
 
     headers = {
         'Accept': 'application/vnd.github.v3+json',
@@ -123,35 +160,34 @@ def fetch_github_users(github_token=None, max_users=200, min_followers=1, days=3
     all_users = {}  # Use dict to avoid duplicates by login
     status_updates = []
     total_api_calls = 0
+    contribution_errors = []
 
     try:
-        # PRIORITY 1: Country-level searches (most reliable)
+        # Country-level searches
         country_searches = [
             'United+Arab+Emirates',
             'UAE',
-            'U.A.E',
             '"United Arab Emirates"',
-            '"UAE"',
-            'Emirates',
         ]
         
-        status_updates.append("🔍 Phase 1: Searching by country (United Arab Emirates)...")
+        status_updates.append(f"🔍 Phase 1: Country-level search (UAE)...")
+        status_updates.append(f"📅 Counting contributions from last {days} days")
         
         for search_term in country_searches:
-            if total_api_calls >= 200:
+            if total_api_calls >= 150:
                 break
             
-            # Search with different sorting methods to catch different users
+            # Multiple sort methods to get diverse users
             sort_methods = [
-                ('followers', 'desc'),  # Most followed
-                ('repositories', 'desc'),  # Most repos
-                ('joined', 'desc'),  # Recently joined
+                ('followers', 'desc'),
+                ('repositories', 'desc'),
+                ('joined', 'desc'),
             ]
             
             for sort_by, order in sort_methods:
-                if total_api_calls >= 200:
+                if total_api_calls >= 150:
                     break
-                    
+                
                 search_url = f'https://api.github.com/search/users?q=location:{search_term}+followers:>={min_followers}&sort={sort_by}&order={order}&per_page=100'
                 
                 response = requests.get(search_url, headers=headers)
@@ -161,31 +197,35 @@ def fetch_github_users(github_token=None, max_users=200, min_followers=1, days=3
                     data = response.json()
                     users = data.get('items', [])
                     
-                    status_updates.append(f"  📍 Found {len(users)} users with search '{search_term}' sorted by {sort_by}")
+                    status_updates.append(f"  📍 Search '{search_term}' by {sort_by}: {len(users)} users")
                     
                     for user in users:
                         if user['login'] not in all_users:
-                            if total_api_calls >= 200:
+                            if total_api_calls >= 150:
                                 break
-                                
+                            
                             # Fetch detailed user info
-                            user_url = user['url']
-                            user_response = requests.get(user_url, headers=headers)
+                            user_response = requests.get(user['url'], headers=headers)
                             total_api_calls += 1
                             
                             if user_response.status_code == 200:
                                 user_data = user_response.json()
-                                
-                                # Verify location is UAE-related
                                 user_location = user_data.get('location', '')
                                 
                                 if is_uae_location(user_location):
-                                    # Get accurate contribution count
-                                    contributions = get_contribution_count(user['login'], github_token, days)
+                                    # Get ACCURATE contribution count via GraphQL
+                                    contributions, debug_info = get_contribution_count_graphql(
+                                        user['login'], 
+                                        github_token, 
+                                        days
+                                    )
+                                    
+                                    if contributions == 0 and "Error" in debug_info:
+                                        contribution_errors.append(f"{user['login']}: {debug_info}")
                                     
                                     all_users[user['login']] = {
                                         'login': user_data.get('login', ''),
-                                        'name': user_data.get('name', user_data.get('login', '')),
+                                        'name': user_data.get('name') or user_data.get('login', ''),
                                         'avatar': user_data.get('avatar_url', ''),
                                         'followers': user_data.get('followers', 0),
                                         'public_repos': user_data.get('public_repos', 0),
@@ -193,64 +233,56 @@ def fetch_github_users(github_token=None, max_users=200, min_followers=1, days=3
                                         'location': user_data.get('location', ''),
                                         'bio': user_data.get('bio', ''),
                                         'company': user_data.get('company', ''),
+                                        'debug_info': debug_info
                                     }
                                     
-                                    if len(all_users) % 10 == 0:
-                                        status_updates.append(f"  ✅ Collected {len(all_users)} unique UAE users so far...")
+                                    if len(all_users) % 20 == 0:
+                                        status_updates.append(f"  ✅ Processed {len(all_users)} unique UAE users...")
                                 
-                                time.sleep(0.5)  # Rate limiting
+                                time.sleep(0.8)  # Rate limiting - be more conservative
                     
-                    time.sleep(1)  # Rate limiting between searches
+                    time.sleep(1.5)
                     
                 elif response.status_code == 403:
-                    status_updates.append(f"⚠️ Rate limit reached after {total_api_calls} calls")
+                    status_updates.append(f"⚠️ Rate limit hit after {total_api_calls} calls")
                     break
                 elif response.status_code == 422:
-                    status_updates.append(f"  ⚠️ Search term '{search_term}' invalid, skipping...")
                     continue
                 else:
-                    status_updates.append(f"  ❌ Error with search '{search_term}': {response.status_code}")
+                    status_updates.append(f"  ❌ Search error: {response.status_code}")
         
-        # PRIORITY 2: City-level searches as supplementary (if we need more users)
-        if len(all_users) < max_users * 1.5 and total_api_calls < 180:
-            status_updates.append(f"🔍 Phase 2: Supplementary city-level search...")
+        # Phase 2: Major cities
+        if len(all_users) < max_users * 1.5 and total_api_calls < 130:
+            status_updates.append(f"🔍 Phase 2: Major city searches...")
             
-            city_searches = [
-                'Dubai',
-                'Abu+Dhabi',
-                'Sharjah',
-                'Ajman',
-            ]
+            cities = ['Dubai', 'Abu+Dhabi', 'Sharjah']
             
-            for city in city_searches:
-                if total_api_calls >= 200 or len(all_users) >= max_users * 2:
+            for city in cities:
+                if total_api_calls >= 150:
                     break
                 
-                search_url = f'https://api.github.com/search/users?q=location:{city}+followers:>={min_followers}&sort=followers&order=desc&per_page=100'
-                
+                search_url = f'https://api.github.com/search/users?q=location:{city}+followers:>={min_followers}&sort=followers&order=desc&per_page=50'
                 response = requests.get(search_url, headers=headers)
                 total_api_calls += 1
                 
                 if response.status_code == 200:
-                    data = response.json()
-                    users = data.get('items', [])
+                    users = response.json().get('items', [])
                     
-                    for user in users[:30]:  # Limit to top 30 per city
-                        if user['login'] not in all_users and total_api_calls < 200:
-                            user_url = user['url']
-                            user_response = requests.get(user_url, headers=headers)
+                    for user in users[:25]:
+                        if user['login'] not in all_users and total_api_calls < 150:
+                            user_response = requests.get(user['url'], headers=headers)
                             total_api_calls += 1
                             
                             if user_response.status_code == 200:
                                 user_data = user_response.json()
-                                user_location = user_data.get('location', '')
-                                
-                                if is_uae_location(user_location):
-                                    contributions = get_contribution_count(user['login'], github_token, days)
+                                if is_uae_location(user_data.get('location', '')):
+                                    contributions, debug_info = get_contribution_count_graphql(
+                                        user['login'], github_token, days
+                                    )
                                     
                                     all_users[user['login']] = {
                                         'login': user_data.get('login', ''),
-                                        'name': user_data.get('name', user_data.get('login', '')),
+                                        'name': user_data.get('name') or user_data.get('login', ''),
                                         'avatar': user_data.get('avatar_url', ''),
                                         'followers': user_data.get('followers', 0),
                                         'public_repos': user_data.get('public_repos', 0),
@@ -258,28 +290,27 @@ def fetch_github_users(github_token=None, max_users=200, min_followers=1, days=3
                                         'location': user_data.get('location', ''),
                                         'bio': user_data.get('bio', ''),
                                         'company': user_data.get('company', ''),
+                                        'debug_info': debug_info
                                     }
                                 
-                                time.sleep(0.5)
-                    
-                    time.sleep(1)
+                                time.sleep(0.8)
+                    time.sleep(1.5)
 
-        # Convert dict to list
+        # Convert and sort
         users_list = list(all_users.values())
+        status_updates.append(f"📊 Found {len(users_list)} unique UAE developers")
         
-        status_updates.append(f"📊 Total unique UAE users found: {len(users_list)}")
-        
-        # Sort by CONTRIBUTIONS FIRST (descending), then by followers (descending)
+        # Sort by contributions (primary), then followers (secondary)
         users_list.sort(key=lambda x: (x['contributions'], x['followers']), reverse=True)
         
-        # Take top contributors based on max_users
+        # Take top N
         top_users = users_list[:max_users]
         
-        # Add rank based on contribution order
+        # Add rank
         for i, user in enumerate(top_users, 1):
             user['rank'] = i
         
-        # Convert to DataFrame
+        # Create DataFrame
         df = pd.DataFrame(top_users)
         
         if not df.empty:
@@ -290,8 +321,15 @@ def fetch_github_users(github_token=None, max_users=200, min_followers=1, days=3
             status_message = f"""✅ Successfully fetched top {len(df)} contributors (last {days} days)
 📊 Total unique users found: {len(all_users)}
 🔧 API calls made: {total_api_calls}
-🌍 Search strategy: Country-first (UAE/United Arab Emirates)
-""" + "\n".join(status_updates[-8:])
+📅 Date range: {datetime.now() - timedelta(days=days)} to {datetime.now()}
+
+Recent status:
+""" + "\n".join(status_updates[-6:])
+            
+            if contribution_errors:
+                status_message += f"\n\n⚠️ Contribution fetch errors: {len(contribution_errors)}"
+                status_message += f"\n(First 3: {', '.join(contribution_errors[:3])})"
+            
             return display_df, status_message
         else:
             return pd.DataFrame(), "⚠️ No users found\n" + "\n".join(status_updates)
@@ -303,7 +341,6 @@ def search_users(df, search_term):
     """Filter users based on search term"""
     if df is None or df.empty:
         return df
-    
     if not search_term:
         return df
     
@@ -319,17 +356,18 @@ def search_users(df, search_term):
 with gr.Blocks(theme=gr.themes.Soft(), title="GitHub UAE Top Contributors") as app:
     
     gr.Markdown("""
-    # 🇦🇪 Top 200 GitHub Contributors in UAE
-    ### Country-Focused Search with Accurate Contribution Counts
+    # 🇦🇪 GitHub UAE Top Contributors Leaderboard
+    ### Accurate Contribution Counting via GitHub GraphQL API
+    
+    **This tool ranks UAE-based developers by their actual contribution count from GitHub's official API.**
     
-    **Search Strategy:** Prioritizes "United Arab Emirates" and "UAE" country-level searches, 
-    supplemented by major city searches. This ensures comprehensive coverage.
+    ⚠️ **IMPORTANT:** You MUST provide a GitHub Personal Access Token with `read:user` scope for accurate contribution counts.
     """)
     
     with gr.Row():
         token_input = gr.Textbox(
-            label="GitHub Personal Access Token (Required)",
-            placeholder="ghp_xxxxxxxxxxxx or set GITHUB_TOKEN env var",
+            label="GitHub Personal Access Token (REQUIRED - needs 'read:user' scope)",
+            placeholder="ghp_xxxxxxxxxxxx or set GITHUB_TOKEN environment variable",
             type="password",
             scale=3
         )
@@ -344,14 +382,14 @@ with gr.Blocks(theme=gr.themes.Soft(), title="GitHub UAE Top Contributors") as a
     
     with gr.Row():
         min_followers_input = gr.Slider(
-            label="Minimum Followers (0 = most comprehensive)",
+            label="Minimum Followers (0 = everyone)",
             minimum=0,
             maximum=50,
             value=0,
             step=1
         )
         days_input = gr.Slider(
-            label="Days to Count Contributions",
+            label="Days to Count (30 = last 30 days)",
             minimum=7,
             maximum=90,
             value=30,
@@ -359,22 +397,20 @@ with gr.Blocks(theme=gr.themes.Soft(), title="GitHub UAE Top Contributors") as a
         )
     
     with gr.Row():
-        fetch_btn = gr.Button("🚀 Fetch Top Contributors from UAE", variant="primary", size="lg")
+        fetch_btn = gr.Button("🚀 Fetch Top Contributors", variant="primary", size="lg")
     
-    status_msg = gr.Textbox(label="Status & Progress", interactive=False, lines=6)
+    status_msg = gr.Textbox(label="Status & Debug Info", interactive=False, lines=8)
     
     with gr.Row():
         search_box = gr.Textbox(
-            label="🔍 Search by Name, Username, or Location",
-            placeholder="Type to search...",
+            label="🔍 Search Results",
+            placeholder="Search by name, username, or location...",
             scale=4
         )
         clear_btn = gr.Button("Clear", scale=1)
     
-    # Store the full dataframe
     full_data = gr.State(value=pd.DataFrame())
     
-    # Display dataframe
     data_display = gr.Dataframe(
         headers=["Rank", "Name", "Username", "Contributions", "Followers", "Public Repos", "Location", "GitHub Profile"],
         datatype=["number", "str", "str", "number", "number", "number", "str", "str"],
@@ -384,41 +420,43 @@ with gr.Blocks(theme=gr.themes.Soft(), title="GitHub UAE Top Contributors") as a
     
     gr.Markdown("""
     ---
-    **Improved Search Strategy:**
-    
-    ✅ **Phase 1: Country-Level Search (Primary)**
-    - Searches: "United Arab Emirates", "UAE", "U.A.E", "Emirates"
-    - Multiple sort orders: followers, repositories, recently joined
-    - Catches users regardless of city
-    
-    ✅ **Phase 2: City-Level Search (Supplementary)**  
-    - Searches: Dubai, Abu Dhabi, Sharjah, Ajman
-    - Only if needed to reach target count
-    
-    ✅ **Smart Location Verification**
-    - Validates location field contains UAE/Emirates/city keywords
-    - Filters out false positives
-    
-    ✅ **Comprehensive Coverage**
-    - Multiple search variations and sort methods
-    - Automatic deduplication
-    - Up to 200 API calls for thorough search
-    
-    **Ranking:**
-    - Primary: Contributions in last N days (accurate count from GitHub GraphQL)
-    - Secondary: Follower count
-    
-    **Token Setup:**
-    1. [Generate token](https://github.com/settings/tokens) with `read:user` scope
-    2. Paste above or set `GITHUB_TOKEN` environment variable
-    
-    **Tips:**
-    - Set "Minimum Followers" to 0 for most comprehensive results
-    - Increase "Max Contributors" to 300-500 for broader coverage
-    - Takes 2-5 minutes depending on settings
+    ## How It Works:
+    
+    **Data Source:** GitHub GraphQL API (v4) - same data shown on user profiles
+    
+    **Contribution Types Counted:**
+    - Commits to any branch
+    - Pull requests opened
+    - Issues opened  
+    - Pull request reviews
+    - Repository creation and more
+    
+    **Search Strategy:**
+    1. **Country search:** "United Arab Emirates", "UAE" (primary)
+    2. **City search:** Dubai, Abu Dhabi, Sharjah (supplementary)
+    3. **Location verification:** Confirms UAE location in profile
+    4. **Multiple sort orders:** Followers, repos, recently joined
+    
+    **Ranking:** Primary by contributions (last N days), secondary by followers
+    
+    ---
+    
+    ## Setup GitHub Token:
+    
+    1. Go to: https://github.com/settings/tokens
+    2. Click "Generate new token (classic)"
+    3. Name it: "UAE Contributors Tracker"
+    4. **Check the `read:user` scope** (REQUIRED for GraphQL API)
+    5. Generate and copy the token
+    6. Paste above OR set environment variable: `export GITHUB_TOKEN="ghp_..."`
+    
+    ---
+    
+    **Processing Time:** 2-5 minutes depending on settings
+    
+    **Note:** If you see 0 contributions for highly active users, there may be a GraphQL API error (check status output).
     """)
     
-    # Event handlers
     def fetch_and_display(token, max_users, min_followers, days):
         df, msg = fetch_github_users(token if token else None, int(max_users), int(min_followers), int(days))
         return df, df, msg
@@ -449,6 +487,5 @@ with gr.Blocks(theme=gr.themes.Soft(), title="GitHub UAE Top Contributors") as a
         outputs=[search_box, data_display]
     )
 
-# Launch the app
 if __name__ == "__main__":
     app.launch()