from pymongo import MongoClient from pymongo.server_api import ServerApi from curl_cffi import requests from pandas import DataFrame from pandas import concat as pd_concat from numpy import where as np_where from numpy import nan as np_nan from datetime import datetime from datetime import date from datetime import timedelta from pytz import timezone as pytz_timezone import time import random uri = "mongodb+srv://multichem:Xr1q5wZdXPbxdUmJ@testcluster.lgwtp5i.mongodb.net/?retryWrites=true&w=majority&appName=TestCluster" # Try to connect with error handling try: # First attempt: Using SRV format client = MongoClient(uri, server_api=ServerApi('1')) # Test the connection client.admin.command('ping') print("Pinged your deployment. You successfully connected to MongoDB!") except Exception as e: print(f"SRV connection failed: {e}") try: # Second attempt: Try direct connection format direct_uri = uri.replace('mongodb+srv://', 'mongodb://') client = MongoClient(direct_uri, server_api=ServerApi('1')) # Test the connection client.admin.command('ping') print("Pinged your deployment using direct connection!") except Exception as e: print(f"Direct connection failed: {e}") raise def grab_contest_ids(): db = client['Contest_Information'] for sport in ['MLB', 'NBA', 'NHL', 'NFL', 'GOLF', 'MMA', 'CFB', 'LOL', 'CS', 'COD', 'SOC', 'F1', 'NAS', 'UFL', 'TEN', 'WNBA']: # for sport in ['MLB', 'NFL']: for contest_type_var in ['Classic', 'Showdown Captain Mode']: if sport != 'WNBA': response = requests.get(f'https://www.draftkings.com/lobby/getcontests?sport={sport}', impersonate="chrome") else: response = requests.get(f'https://www.draftkings.com/lobby/getcontests?sport=NBA', impersonate="chrome") data = response.json() contest_import = None player_import = None payout_import = None if contest_type_var == 'Classic': contest_collection = db[f'{sport}_reg_contest_info'] player_collection = db[f'{sport}_reg_player_info'] payout_collection = db[f'{sport}_reg_payout_info'] cursor = contest_collection.find() try: contest_import = DataFrame(list(cursor)).drop('_id', axis=1) except Exception as e: print(f"Classic Contest Importing Error: {e}") cursor = player_collection.find() try: player_import = DataFrame(list(cursor)).drop('_id', axis=1) except Exception as e: print(f"Classic Player Importing Error: {e}") cursor = payout_collection.find() try: payout_import = DataFrame(list(cursor)).drop('_id', axis=1) except Exception as e: print(f"Classic Payout Importing Error: {e}") instances = 25 elif contest_type_var == 'Showdown Captain Mode': contest_collection = db[f'{sport}_sd_contest_info'] player_collection = db[f'{sport}_showdown_player_info'] payout_collection = db[f'{sport}_showdown_payout_info'] cursor = contest_collection.find() try: contest_import = DataFrame(list(cursor)).drop('_id', axis=1) except Exception as e: print(f"Showdown Contest Importing Error: {e}") cursor = player_collection.find() try: player_import = DataFrame(list(cursor)).drop('_id', axis=1) except Exception as e: print(f"Showdown Player Importing Error: {e}") cursor = payout_collection.find() try: payout_import = DataFrame(list(cursor)).drop('_id', axis=1) except Exception as e: print(f"Showdown Payout Importing Error: {e}") instances = 25 contest_overall = [] payouts_overall = DataFrame(columns=['Date', 'Contest Date', 'Contest ID', 'Entry Fee', 'minPosition', 'maxPosition', 'value']) player_overall = DataFrame(columns=['Date', 'Contest Date', 'Contest ID', 'draftableId', 'First Name', 'Last Name', 'Display Name', 'Position', 'Team', 'Opp', 'Salary', 'Avg FPTS', 'Sport']) try: if sport != 'WNBA': if sport == 'TEN': classic_contests = [contest for contest in data['Contests'] if contest.get('gameType') == 'Single Match' or contest.get('gameType') == 'Short Slate' and '-Player' not in contest.get('n') and '50-50' not in contest.get('n')] else: classic_contests = [contest for contest in data['Contests'] if contest.get('gameType') == 'Classic' and '-Player' not in contest.get('n') and '50-50' not in contest.get('n')] else: classic_contests = [contest for contest in data['Contests'] if contest.get('gameType') == 'WNBA' and '-Player' not in contest.get('n') and '50-50' not in contest.get('n')] # Sort classic contests by 'po' element in descending order (highest first) classic_contests.sort(key=lambda x: x.get('po', 0), reverse=True) # Filter out contests with "-Player" afterwards classic_contests = [contest for contest in classic_contests if '-Player' not in contest.get('n') and '50-50' not in contest.get('n') and 'Winner Take All' not in contest.get('n')] except Exception as e: print(f"Classic Contest Error: {e}") if not classic_contests: print('No Classic Contests Found') try: if sport != 'WNBA': showdown_contests = [contest for contest in data['Contests'] if contest.get('gameType') == 'Showdown Captain Mode' or contest.get('gameType') == 'Showdown' and '-Player' not in contest.get('n') and '50-50' not in contest.get('n')] else: showdown_contests = [contest for contest in data['Contests'] if 'WNBA' in contest.get('n', '') and '-Player' not in contest.get('n') and '50-50' not in contest.get('n') and contest.get('gameType') == 'Showdown Captain Mode'] # Sort showdown contests by 'po' element in descending order (highest first) showdown_contests.sort(key=lambda x: x.get('po', 0), reverse=True) # Filter out contests with "-Player" afterwards showdown_contests = [contest for contest in showdown_contests if '-Player' not in contest.get('n') and '50-50' not in contest.get('n') and 'Winner Take All' not in contest.get('n')] except Exception as e: print(f"Showdown Contest Error: {e}") if not showdown_contests: print('No Showdown Contests Found') for contest_num in range(0,instances): try: contest_info = [] if contest_type_var == 'Classic': # Then get the nth contest (if it exists) if contest_num < len(classic_contests): contests = classic_contests[contest_num] else: break # No more contests to process elif contest_type_var == 'Showdown Captain Mode': # Then get the nth contest (if it exists) if contest_num < len(showdown_contests): contests = showdown_contests[contest_num] else: break # No more contests to process # Get current time in EST est = pytz_timezone('US/Eastern') current_time = datetime.now(est) # If before 8pm EST, use today, otherwise use tomorrow if current_time.hour < 20: current_date = str(current_time.date()).replace('-', '') else: tomorrow = current_time + timedelta(days=1) current_date = str(tomorrow.date()).replace('-', '') contest_name = contests.get('n') contest_type = contests.get('gameType') contest_id = contests.get('id') contest_date = str((datetime.fromtimestamp(int(contests.get('sd')[6:-2]) / 1000)).strftime('%Y-%m-%d')).replace('-', '') contest_url = 'https://dh5nxc6yx3kwy.cloudfront.net/contests/'+str(sport.lower())+'/'+str(contest_date)+'/'+str(contest_id)+'/data/' for data_type in [current_date, contest_date, contest_name, contest_type, contest_id, contest_url, sport]: contest_info.append(data_type) contest_overall.append(contest_info) time.sleep(.5 + random.random()) contest_specs_url = 'https://api.draftkings.com/contests/v1/contests/'+str(contest_id)+'?format=json' response = requests.get(contest_specs_url, impersonate="chrome") contest_specs_data = response.json() contests = contest_specs_data['contestDetail'] draftGroup = contests.get('draftGroupId') entry_fee = contests.get('entryFee') payouts_data = contests.get('payoutSummary') payouts = [ { 'Date': current_date, 'Contest Date': contest_date, 'Contest ID': contest_id, 'Entry Fee': entry_fee, 'minPosition': payout['minPosition'], 'maxPosition': payout['maxPosition'], 'value': payout.get('payoutDescriptions', [{}])[0].get('value', 0) if payout.get('payoutDescriptions') else 0 } for payout in payouts_data ] payout_info = DataFrame(payouts) payouts_overall = pd_concat([payouts_overall, payout_info], ignore_index=True) time.sleep(.5 + random.random()) DK_URL = "https://api.draftkings.com/draftgroups/v1/draftgroups/"+str(draftGroup)+"/draftables" # Make a request to the API response = requests.get(DK_URL, impersonate="chrome") draftables_data = response.json() # Extracting the required fields draftables = draftables_data['draftables'] try: draftables_data = [ { 'Date': current_date, 'Contest Date': contest_date, 'Contest ID': contest_id, 'draftableId': draftable.get('draftableId'), 'First Name': draftable.get('firstName'), 'Last Name': draftable.get('lastName'), 'Display Name': draftable.get('displayName'), 'Position': draftable.get('position'), 'Team': draftable.get('teamAbbreviation'), 'Opp': np_where(draftable.get('teamAbbreviation') == draftable['competition']['nameDisplay'][0]['value'], draftable['competition']['nameDisplay'][2]['value'], draftable['competition']['nameDisplay'][0]['value']).item(), 'Salary': draftable.get('salary'), 'Avg FPTS': draftable.get('draftStatAttributes', [{}])[0].get('value', 0), 'Sport': sport, } for draftable in draftables ] except: draftables_data = [ { 'Date': current_date, 'Contest Date': contest_date, 'Contest ID': contest_id, 'draftableId': draftable.get('draftableId'), 'First Name': draftable.get('firstName'), 'Last Name': draftable.get('lastName'), 'Display Name': draftable.get('displayName'), 'Position': draftable.get('position'), 'Team': draftable.get('teamAbbreviation'), 'Opp': np_nan, 'Salary': draftable.get('salary'), 'Avg FPTS': draftable.get('draftStatAttributes', [{}])[0].get('value', 0), 'Sport': sport, } for draftable in draftables ] player_info = DataFrame(draftables_data) player_overall = pd_concat([player_overall, player_info], ignore_index=True) except Exception as e: print(f"Error: {e}") pass payout_info_frame = DataFrame(payouts_overall, columns=['Date', 'Contest Date', 'Contest ID', 'Entry Fee', 'minPosition', 'maxPosition', 'value']) contest_info_frame = DataFrame(contest_overall, columns=['Date', 'Contest Date', 'Contest Name', 'Contest Type', 'Contest ID', 'URL', 'Sport']) print(contest_info_frame) try: contest_export = pd_concat([contest_import, contest_info_frame], ignore_index=True) except: contest_export = contest_info_frame if len(contest_export) > 0: contest_export = contest_export.drop_duplicates(subset=['Contest Date', 'Contest ID']) contest_export = contest_export.reset_index(drop=True) chunk_size = 10000 contest_collection.drop() for i in range(0, len(contest_export), chunk_size): for _ in range(5): try: df_chunk = contest_export.iloc[i:i + chunk_size] contest_collection.insert_many(df_chunk.to_dict('records'), ordered=False) break except Exception as e: print(f"Retry due to error: {e}") time.sleep(1) print(player_overall) try: player_export = pd_concat([player_import, player_overall], ignore_index=True) except: player_export = player_overall if len(player_export) > 0: player_export = player_export.drop_duplicates(subset=['Contest Date', 'draftableId', 'Display Name']) player_export = player_export.reset_index(drop=True) chunk_size = 10000 player_collection.drop() for i in range(0, len(player_export), chunk_size): for _ in range(5): try: df_chunk = player_export.iloc[i:i + chunk_size] player_collection.insert_many(df_chunk.to_dict('records'), ordered=False) break except Exception as e: print(f"Retry due to error: {e}") time.sleep(1) print(payout_info_frame) try: payout_export = pd_concat([payout_import, payout_info_frame], ignore_index=True) except: payout_export = payout_info_frame if len(payout_export) > 0: payout_export = payout_export.drop_duplicates(subset=['Contest Date', 'Contest ID', 'Entry Fee', 'minPosition', 'maxPosition']) payout_export = payout_export.reset_index(drop=True) chunk_size = 10000 payout_collection.drop() for i in range(0, len(payout_export), chunk_size): for _ in range(5): try: df_chunk = payout_export.iloc[i:i + chunk_size] payout_collection.insert_many(df_chunk.to_dict('records'), ordered=False) break except Exception as e: print(f"Retry due to error: {e}") time.sleep(1) grab_contest_ids()