Upload api_scraper.py with huggingface_hub

api_scraper.py

@@ -100,20 +100,25 @@ class MLB_Scrape:
         # Make API call to retrieve game schedule
         game_call = requests.get(url=f'https://statsapi.mlb.com/api/v1/schedule/?sportId={sport_id_str}&gameTypes={game_type_str}&season={year_input_str}&hydrate=lineup,players').json()
         try:
-            # Extract relevant data from the API response
-            game_list = [item for sublist in [[y['gamePk'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            time_list = [item for sublist in [[y['gameDate'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            date_list = [item for sublist in [[y['officialDate'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            away_team_list = [item for sublist in [[y['teams']['away']['team']['name'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            away_team_id_list = [item for sublist in [[y['teams']['away']['team']['id'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            home_team_list = [item for sublist in [[y['teams']['home']['team']['name'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            home_team_id_list = [item for sublist in [[y['teams']['home']['team']['id'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            state_list = [item for sublist in [[y['status']['codedGameState'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            venue_id = [item for sublist in [[y['venue']['id'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            venue_name = [item for sublist in [[y['venue']['name'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            gameday_type = [item for sublist in [[y['gamedayType'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            # Create a Polars DataFrame with the extracted data
-
+            def safe_get(d, keys, default=np.nan):
+                """Safely retrieve nested dictionary values."""
+                for key in keys:
+                    d = d.get(key, {})
+                    if not isinstance(d, dict):
+                        return d  # Return value if it's not a dict
+                return default  # Return default if keys don't exist
+
+            game_list = [item for sublist in [[y.get('gamePk', np.nan) for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            time_list = [item for sublist in [[y.get('gameDate', np.nan) for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            date_list = [item for sublist in [[y.get('officialDate', np.nan) for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            away_team_list = [item for sublist in [[safe_get(y, ['teams', 'away', 'team', 'name'], "") for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            away_team_id_list = [item for sublist in [[safe_get(y, ['teams', 'away', 'team', 'id'], np.nan) for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            home_team_list = [item for sublist in [[safe_get(y, ['teams', 'home', 'team', 'name'], "") for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            home_team_id_list = [item for sublist in [[safe_get(y, ['teams', 'home', 'team', 'id'], np.nan) for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            state_list = [item for sublist in [[safe_get(y, ['status', 'codedGameState'], "") for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            venue_id = [item for sublist in [[safe_get(y, ['venue', 'id'], np.nan) for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            venue_name = [item for sublist in [[safe_get(y, ['venue', 'name'], "") for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            gameday_type = [item for sublist in [[safe_get(y, ['gamedayType'], "") for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
 
             # Create a Polars DataFrame with the extracted data
             game_df = pl.DataFrame(data={'game_id': game_list,
@@ -154,41 +159,30 @@ class MLB_Scrape:
         return game_df
     
 
-    # def get_data(self, game_list_input: list):
-    #     """
-    #     Retrieves live game data for a list of game IDs in parallel.
+    def get_data(self, game_list_input: list):
+        """
+        Retrieves live game data for a list of game IDs in parallel.
         
-    #     Parameters:
-    #     - game_list_input (list): A list of game IDs for which to retrieve live data.
+        Parameters:
+        - game_list_input (list): A list of game IDs for which to retrieve live data.
         
-    #     Returns:
-    #     - data_total (list): A list of JSON responses containing live game data for each game ID.
-    #     """
-    #     data_total = []
-    #     print('This May Take a While. Progress Bar shows Completion of Data Retrieval.')
+        Returns:
+        - data_total (list): A list of JSON responses containing live game data for each game ID.
+        """
+        data_total = []
+        print('This May Take a While. Progress Bar shows Completion of Data Retrieval.')
         
-    #     def fetch_data(game_id):
-    #         r = requests.get(f'https://statsapi.mlb.com/api/v1.1/game/{game_id}/feed/live')
-    #         return r.json()
+        def fetch_data(game_id):
+            r = requests.get(f'https://statsapi.mlb.com/api/v1.1/game/{game_id}/feed/live')
+            return r.json()
         
-    #     with ThreadPoolExecutor() as executor:
-    #         futures = {executor.submit(fetch_data, game_id): game_id for game_id in game_list_input}
-    #         for future in tqdm(as_completed(futures), total=len(futures), desc="Processing", unit="iteration"):
-    #             data_total.append(future.result())
+        with ThreadPoolExecutor() as executor:
+            futures = {executor.submit(fetch_data, game_id): game_id for game_id in game_list_input}
+            for future in tqdm(as_completed(futures), total=len(futures), desc="Processing", unit="iteration"):
+                data_total.append(future.result())
         
-    #     return data_total
-
-
-    def get_data(self,game_list_input = [748540]):
-        data_total = []
-        #n_count = 0
-        print('This May Take a While. Progress Bar shows Completion of Data Retrieval.')
-        for i in tqdm(range(len(game_list_input)), desc="Processing", unit="iteration"):
-            r = requests.get(f'https://statsapi.mlb.com/api/v1.1/game/{game_list_input[i]}/feed/live')
-            data_total.append(r.json())
         return data_total
 
-
     def get_data_df(self, data_list):
         """
         Converts a list of game data JSON objects into a Polars DataFrame.