global_func/grab_pricing.py

from pymongo import MongoClient
from pymongo.server_api import ServerApi
from curl_cffi import requests
from pandas import DataFrame
from pandas import concat as pd_concat
from numpy import where as np_where
from numpy import nan as np_nan
from datetime import datetime
from datetime import date
from datetime import timedelta
from pytz import timezone as pytz_timezone
import time
import random

uri = "mongodb+srv://multichem:Xr1q5wZdXPbxdUmJ@testcluster.lgwtp5i.mongodb.net/?retryWrites=true&w=majority&appName=TestCluster"

# Try to connect with error handling
try:
    # First attempt: Using SRV format
    client = MongoClient(uri, server_api=ServerApi('1'))
    
    # Test the connection
    client.admin.command('ping')
    print("Pinged your deployment. You successfully connected to MongoDB!")
    
except Exception as e:
    print(f"SRV connection failed: {e}")
    try:
        # Second attempt: Try direct connection format
        direct_uri = uri.replace('mongodb+srv://', 'mongodb://')
        client = MongoClient(direct_uri, server_api=ServerApi('1'))
        
        # Test the connection
        client.admin.command('ping')
        print("Pinged your deployment using direct connection!")
        
    except Exception as e:
        print(f"Direct connection failed: {e}")
        raise

def grab_contest_ids():
    db = client['Contest_Information']

    for sport in ['MLB', 'NBA', 'NHL', 'NFL', 'GOLF', 'MMA', 'CFB', 'LOL', 'CS', 'COD', 'SOC', 'F1', 'NAS', 'UFL', 'TEN', 'WNBA']:
    # for sport in ['MLB', 'NFL']:

        for contest_type_var in ['Classic', 'Showdown Captain Mode']:
            if sport != 'WNBA':
                response = requests.get(f'https://www.draftkings.com/lobby/getcontests?sport={sport}', impersonate="chrome")
            else:
                response = requests.get(f'https://www.draftkings.com/lobby/getcontests?sport=NBA', impersonate="chrome")

            data = response.json()
            contest_import = None
            player_import = None
            payout_import = None

            if contest_type_var == 'Classic':
                contest_collection = db[f'{sport}_reg_contest_info']
                player_collection = db[f'{sport}_reg_player_info']
                payout_collection = db[f'{sport}_reg_payout_info']

                cursor = contest_collection.find()
                try:
                    contest_import = DataFrame(list(cursor)).drop('_id', axis=1)
                except Exception as e:
                    print(f"Classic Contest Importing Error: {e}")

                cursor = player_collection.find()
                try:
                    player_import = DataFrame(list(cursor)).drop('_id', axis=1)
                except Exception as e:
                    print(f"Classic Player Importing Error: {e}")

                cursor = payout_collection.find()
                try:
                    payout_import = DataFrame(list(cursor)).drop('_id', axis=1)
                except Exception as e:
                    print(f"Classic Payout Importing Error: {e}")
                
                instances = 25

            elif contest_type_var == 'Showdown Captain Mode':
                contest_collection = db[f'{sport}_sd_contest_info']
                player_collection = db[f'{sport}_showdown_player_info']
                payout_collection = db[f'{sport}_showdown_payout_info']

                cursor = contest_collection.find()
                try:
                    contest_import = DataFrame(list(cursor)).drop('_id', axis=1)
                except Exception as e:
                    print(f"Showdown Contest Importing Error: {e}")

                cursor = player_collection.find()
                try:
                    player_import = DataFrame(list(cursor)).drop('_id', axis=1)
                except Exception as e:
                    print(f"Showdown Player Importing Error: {e}")

                cursor = payout_collection.find()
                try:
                    payout_import = DataFrame(list(cursor)).drop('_id', axis=1)
                except Exception as e:
                    print(f"Showdown Payout Importing Error: {e}")
                
                instances = 25

            contest_overall = []
            payouts_overall = DataFrame(columns=['Date', 'Contest Date', 'Contest ID', 'Entry Fee', 'minPosition', 'maxPosition', 'value'])
            player_overall = DataFrame(columns=['Date', 'Contest Date', 'Contest ID', 'draftableId', 'First Name', 'Last Name', 'Display Name', 'Position', 'Team', 'Opp', 'Salary', 'Avg FPTS', 'Sport'])

            try:
                if sport != 'WNBA':
                    if sport == 'TEN':
                        classic_contests = [contest for contest in data['Contests'] 
                                            if contest.get('gameType') == 'Single Match' or contest.get('gameType') == 'Short Slate' and '-Player' not in contest.get('n') and '50-50' not in contest.get('n')]
                    else:
                        classic_contests = [contest for contest in data['Contests'] 
                                            if contest.get('gameType') == 'Classic' and '-Player' not in contest.get('n') and '50-50' not in contest.get('n')]
                else:
                    classic_contests = [contest for contest in data['Contests'] 
                                        if contest.get('gameType') == 'WNBA' and '-Player' not in contest.get('n') and '50-50' not in contest.get('n')]
                        
                
                # Sort classic contests by 'po' element in descending order (highest first)
                classic_contests.sort(key=lambda x: x.get('po', 0), reverse=True)

                # Filter out contests with "-Player" afterwards
                classic_contests = [contest for contest in classic_contests if '-Player' not in contest.get('n') and '50-50' not in contest.get('n') and 'Winner Take All' not in contest.get('n')]
                
            except Exception as e:
                print(f"Classic Contest Error: {e}")
                
            if not classic_contests:
                print('No Classic Contests Found')
            
            try:
                if sport != 'WNBA':
                    showdown_contests = [contest for contest in data['Contests'] 
                                        if contest.get('gameType') == 'Showdown Captain Mode' or contest.get('gameType') == 'Showdown' and '-Player' not in contest.get('n') and '50-50' not in contest.get('n')]
                else:
                    showdown_contests = [contest for contest in data['Contests'] 
                                        if 'WNBA' in contest.get('n', '') and '-Player' not in contest.get('n') and '50-50' not in contest.get('n') and contest.get('gameType') == 'Showdown Captain Mode']
                
                # Sort showdown contests by 'po' element in descending order (highest first)
                showdown_contests.sort(key=lambda x: x.get('po', 0), reverse=True)

                # Filter out contests with "-Player" afterwards
                showdown_contests = [contest for contest in showdown_contests if '-Player' not in contest.get('n') and '50-50' not in contest.get('n') and 'Winner Take All' not in contest.get('n')]
                
            except Exception as e:
                print(f"Showdown Contest Error: {e}")
                
            if not showdown_contests:
                print('No Showdown Contests Found')

            for contest_num in range(0,instances):
                try:
                    contest_info = []
                    if contest_type_var == 'Classic':
                        
                        # Then get the nth contest (if it exists)
                        if contest_num < len(classic_contests):
                            contests = classic_contests[contest_num]
                        else:
                            break  # No more contests to process
                        
                    elif contest_type_var == 'Showdown Captain Mode':
                        
                        # Then get the nth contest (if it exists)
                        if contest_num < len(showdown_contests):
                            contests = showdown_contests[contest_num]
                        else:
                            break  # No more contests to process

                    # Get current time in EST
                    est = pytz_timezone('US/Eastern')
                    current_time = datetime.now(est)
                    
                    # If before 8pm EST, use today, otherwise use tomorrow
                    if current_time.hour < 20:
                        current_date = str(current_time.date()).replace('-', '')
                    else:
                        tomorrow = current_time + timedelta(days=1)
                        current_date = str(tomorrow.date()).replace('-', '')
                    contest_name = contests.get('n')
                    contest_type = contests.get('gameType')
                    contest_id = contests.get('id')
                    contest_date = str((datetime.fromtimestamp(int(contests.get('sd')[6:-2]) / 1000)).strftime('%Y-%m-%d')).replace('-', '')
                    contest_url = 'https://dh5nxc6yx3kwy.cloudfront.net/contests/'+str(sport.lower())+'/'+str(contest_date)+'/'+str(contest_id)+'/data/'

                    for data_type in [current_date, contest_date, contest_name, contest_type, contest_id, contest_url, sport]:
                        contest_info.append(data_type)
                    
                    contest_overall.append(contest_info)

                    time.sleep(.5 + random.random())

                    contest_specs_url = 'https://api.draftkings.com/contests/v1/contests/'+str(contest_id)+'?format=json'

                    response = requests.get(contest_specs_url, impersonate="chrome")
                    contest_specs_data = response.json()

                    contests = contest_specs_data['contestDetail']
                    draftGroup = contests.get('draftGroupId')
                    entry_fee = contests.get('entryFee')
                    payouts_data = contests.get('payoutSummary')

                    payouts = [
                        {
                            'Date': current_date,
                            'Contest Date': contest_date,
                            'Contest ID': contest_id,
                            'Entry Fee': entry_fee,
                            'minPosition': payout['minPosition'],
                            'maxPosition': payout['maxPosition'],
                            'value': payout.get('payoutDescriptions', [{}])[0].get('value', 0) if payout.get('payoutDescriptions') else 0
                        }
                        for payout in payouts_data
                    ]

                    payout_info = DataFrame(payouts)

                    payouts_overall = pd_concat([payouts_overall, payout_info], ignore_index=True)

                    time.sleep(.5 + random.random())

                    DK_URL = "https://api.draftkings.com/draftgroups/v1/draftgroups/"+str(draftGroup)+"/draftables"

                    # Make a request to the API
                    response = requests.get(DK_URL, impersonate="chrome")
                    draftables_data = response.json()

                    # Extracting the required fields
                    draftables = draftables_data['draftables']
                    try:
                        draftables_data = [
                            {
                                'Date': current_date,
                                'Contest Date': contest_date,
                                'Contest ID': contest_id,
                                'draftableId': draftable.get('draftableId'),
                                'First Name': draftable.get('firstName'),
                                'Last Name': draftable.get('lastName'),
                                'Display Name': draftable.get('displayName'),
                                'Position': draftable.get('position'),
                                'Team': draftable.get('teamAbbreviation'),
                                'Opp': np_where(draftable.get('teamAbbreviation') == draftable['competition']['nameDisplay'][0]['value'], draftable['competition']['nameDisplay'][2]['value'], draftable['competition']['nameDisplay'][0]['value']).item(),
                                'Salary': draftable.get('salary'),
                                'Avg FPTS': draftable.get('draftStatAttributes', [{}])[0].get('value', 0),
                                'Sport': sport, 
                                
                            }
                            for draftable in draftables
                        ]
                    except:
                        draftables_data = [
                            {
                                'Date': current_date,
                                'Contest Date': contest_date,
                                'Contest ID': contest_id,
                                'draftableId': draftable.get('draftableId'),
                                'First Name': draftable.get('firstName'),
                                'Last Name': draftable.get('lastName'),
                                'Display Name': draftable.get('displayName'),
                                'Position': draftable.get('position'),
                                'Team': draftable.get('teamAbbreviation'),
                                'Opp': np_nan,
                                'Salary': draftable.get('salary'),
                                'Avg FPTS': draftable.get('draftStatAttributes', [{}])[0].get('value', 0),
                                'Sport': sport, 
                                
                            }
                            for draftable in draftables
                        ]

                    player_info = DataFrame(draftables_data)
                        
                    player_overall = pd_concat([player_overall, player_info], ignore_index=True)
                except Exception as e:
                    print(f"Error: {e}")
                    pass

            payout_info_frame = DataFrame(payouts_overall, columns=['Date', 'Contest Date', 'Contest ID', 'Entry Fee', 'minPosition', 'maxPosition', 'value'])
            contest_info_frame = DataFrame(contest_overall, columns=['Date', 'Contest Date', 'Contest Name', 'Contest Type', 'Contest ID', 'URL', 'Sport'])

            print(contest_info_frame)
            try:
                contest_export = pd_concat([contest_import, contest_info_frame], ignore_index=True)
            except:
                contest_export = contest_info_frame

            if len(contest_export) > 0:
                contest_export = contest_export.drop_duplicates(subset=['Contest Date', 'Contest ID'])
                contest_export = contest_export.reset_index(drop=True)

                chunk_size = 10000
                contest_collection.drop()
                for i in range(0, len(contest_export), chunk_size):
                    for _ in range(5):
                        try:
                            df_chunk = contest_export.iloc[i:i + chunk_size]
                            contest_collection.insert_many(df_chunk.to_dict('records'), ordered=False)
                            break
                        except Exception as e:
                            print(f"Retry due to error: {e}")
                            time.sleep(1)
            
            print(player_overall)
            try:
                player_export = pd_concat([player_import, player_overall], ignore_index=True)
            except:
                player_export = player_overall
            
            if len(player_export) > 0:
                player_export = player_export.drop_duplicates(subset=['Contest Date', 'draftableId', 'Display Name'])
                player_export = player_export.reset_index(drop=True)

                chunk_size = 10000
                player_collection.drop()
                for i in range(0, len(player_export), chunk_size):
                    for _ in range(5):
                        try:
                            df_chunk = player_export.iloc[i:i + chunk_size]
                            player_collection.insert_many(df_chunk.to_dict('records'), ordered=False)
                            break
                        except Exception as e:
                            print(f"Retry due to error: {e}")
                            time.sleep(1)
            
            print(payout_info_frame)
            try:
                payout_export = pd_concat([payout_import, payout_info_frame], ignore_index=True)
            except:
                payout_export = payout_info_frame
            
            if len(payout_export) > 0:
                payout_export = payout_export.drop_duplicates(subset=['Contest Date', 'Contest ID', 'Entry Fee', 'minPosition', 'maxPosition'])
                payout_export = payout_export.reset_index(drop=True)

                chunk_size = 10000
                payout_collection.drop()
                for i in range(0, len(payout_export), chunk_size):
                    for _ in range(5):
                        try:
                            df_chunk = payout_export.iloc[i:i + chunk_size]
                            payout_collection.insert_many(df_chunk.to_dict('records'), ordered=False)
                            break
                        except Exception as e:
                            print(f"Retry due to error: {e}")
                            time.sleep(1)

grab_contest_ids()