Upload api_scraper.py with huggingface_hub

api_scraper.py

@@ -100,20 +100,25 @@ class MLB_Scrape:
         # Make API call to retrieve game schedule
         game_call = requests.get(url=f'https://statsapi.mlb.com/api/v1/schedule/?sportId={sport_id_str}&gameTypes={game_type_str}&season={year_input_str}&hydrate=lineup,players').json()
         try:
-            # Extract relevant data from the API response
-            game_list = [item for sublist in [[y['gamePk'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            time_list = [item for sublist in [[y['gameDate'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            date_list = [item for sublist in [[y['officialDate'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            away_team_list = [item for sublist in [[y['teams']['away']['team']['name'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            away_team_id_list = [item for sublist in [[y['teams']['away']['team']['id'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            home_team_list = [item for sublist in [[y['teams']['home']['team']['name'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            home_team_id_list = [item for sublist in [[y['teams']['home']['team']['id'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            state_list = [item for sublist in [[y['status']['codedGameState'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            venue_id = [item for sublist in [[y['venue']['id'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            venue_name = [item for sublist in [[y['venue']['name'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            gameday_type = [item for sublist in [[y['gamedayType'] for y in x['games']] for x in game_call['dates']] for item in sublist]
-            # Create a Polars DataFrame with the extracted data
-
+            def safe_get(d, keys, default=np.nan):
+                """Safely retrieve nested dictionary values."""
+                for key in keys:
+                    d = d.get(key, {})
+                    if not isinstance(d, dict):
+                        return d  # Return value if it's not a dict
+                return default  # Return default if keys don't exist
+
+            game_list = [item for sublist in [[y.get('gamePk', np.nan) for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            time_list = [item for sublist in [[y.get('gameDate', np.nan) for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            date_list = [item for sublist in [[y.get('officialDate', np.nan) for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            away_team_list = [item for sublist in [[safe_get(y, ['teams', 'away', 'team', 'name'], "") for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            away_team_id_list = [item for sublist in [[safe_get(y, ['teams', 'away', 'team', 'id'], np.nan) for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            home_team_list = [item for sublist in [[safe_get(y, ['teams', 'home', 'team', 'name'], "") for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            home_team_id_list = [item for sublist in [[safe_get(y, ['teams', 'home', 'team', 'id'], np.nan) for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            state_list = [item for sublist in [[safe_get(y, ['status', 'codedGameState'], "") for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            venue_id = [item for sublist in [[safe_get(y, ['venue', 'id'], np.nan) for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            venue_name = [item for sublist in [[safe_get(y, ['venue', 'name'], "") for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
+            gameday_type = [item for sublist in [[safe_get(y, ['gamedayType'], "") for y in x.get('games', [])] for x in game_call.get('dates', [])] for item in sublist]
 
             # Create a Polars DataFrame with the extracted data
             game_df = pl.DataFrame(data={'game_id': game_list,
@@ -154,40 +159,51 @@ class MLB_Scrape:
         return game_df
     
 
-    # def get_data(self, game_list_input: list):
-    #     """
-    #     Retrieves live game data for a list of game IDs in parallel.
-        
-    #     Parameters:
-    #     - game_list_input (list): A list of game IDs for which to retrieve live data.
-        
-    #     Returns:
-    #     - data_total (list): A list of JSON responses containing live game data for each game ID.
-    #     """
-    #     data_total = []
-    #     print('This May Take a While. Progress Bar shows Completion of Data Retrieval.')
-        
-    #     def fetch_data(game_id):
-    #         r = requests.get(f'https://statsapi.mlb.com/api/v1.1/game/{game_id}/feed/live')
-    #         return r.json()
+    def get_data(self, game_list_input: list):
+        """
+        Retrieves live game data for a list of game IDs.
         
-    #     with ThreadPoolExecutor() as executor:
-    #         futures = {executor.submit(fetch_data, game_id): game_id for game_id in game_list_input}
-    #         for future in tqdm(as_completed(futures), total=len(futures), desc="Processing", unit="iteration"):
-    #             data_total.append(future.result())
+        Parameters:
+        - game_list_input (list): A list of game IDs for which to retrieve live data.
         
-    #     return data_total
-
-
-    def get_data(self,game_list_input = [748540]):
+        Returns:
+        - data_total (list): A list of JSON responses containing live game data for each game ID.
+        """
         data_total = []
-        #n_count = 0
         print('This May Take a While. Progress Bar shows Completion of Data Retrieval.')
+        
+        # Iterate over the list of game IDs with a progress bar
         for i in tqdm(range(len(game_list_input)), desc="Processing", unit="iteration"):
+            # Make a GET request to the MLB API for each game ID
             r = requests.get(f'https://statsapi.mlb.com/api/v1.1/game/{game_list_input[i]}/feed/live')
+            # Append the JSON response to the data_total list
             data_total.append(r.json())
+        
         return data_total
 
+    def get_data_new(self, game_list_input: list):
+        """
+        Retrieves live game data for a list of game IDs in parallel.
+        
+        Parameters:
+        - game_list_input (list): A list of game IDs for which to retrieve live data.
+        
+        Returns:
+        - data_total (list): A list of JSON responses containing live game data for each game ID.
+        """
+        data_total = []
+        print('This May Take a While. Progress Bar shows Completion of Data Retrieval.')
+        
+        def fetch_data(game_id):
+            r = requests.get(f'https://statsapi.mlb.com/api/v1.1/game/{game_id}/feed/live')
+            return r.json()
+        
+        with ThreadPoolExecutor() as executor:
+            futures = {executor.submit(fetch_data, game_id): game_id for game_id in game_list_input}
+            for future in tqdm(as_completed(futures), total=len(futures), desc="Processing", unit="iteration"):
+                data_total.append(future.result())
+        
+        return data_total
 
     def get_data_df(self, data_list):
         """
@@ -232,6 +248,7 @@ class MLB_Scrape:
         strikes_after = []
         balls_after = []
         outs_after = []
+        inning = []
 
         start_speed = []
         end_speed = []
@@ -339,7 +356,8 @@ class MLB_Scrape:
                             else:
                                 is_swing.append(None)
                                 is_whiff.append(None)
-    
+
+                            inning.append(ab_list['about']['inning'] if 'inning' in ab_list['about'] else None)
                             is_ball.append(ab_list['playEvents'][n]['details']['isOut'] if 'isOut' in ab_list['playEvents'][n]['details'] else None)
                             is_review.append(ab_list['playEvents'][n]['details']['hasReview'] if 'hasReview' in ab_list['playEvents'][n]['details'] else None)
                             pitch_type.append(ab_list['playEvents'][n]['details']['type']['code'] if 'type' in ab_list['playEvents'][n]['details'] else None)
@@ -431,6 +449,7 @@ class MLB_Scrape:
                                 vb.append(None)
                                 ivb.append(None)
                                 hb.append(None)
+                                
     
                             if 'hitData' in ab_list['playEvents'][n]:
                                 launch_speed.append(ab_list['playEvents'][n]['hitData']['launchSpeed'] if 'launchSpeed' in ab_list['playEvents'][n]['hitData'] else None)
@@ -514,6 +533,7 @@ class MLB_Scrape:
                             is_review.append(None)
                             pitch_type.append(None)
                             pitch_description.append(None)
+                            inning.append(None)
                             strikes.append(ab_list['playEvents'][n]['count']['balls'] if 'balls' in ab_list['playEvents'][n]['count'] else None)
                             balls.append(ab_list['playEvents'][n]['count']['strikes'] if 'strikes' in ab_list['playEvents'][n]['count'] else None)
                             outs.append(ab_list['playEvents'][n]['count']['outs'] if 'outs' in ab_list['playEvents'][n]['count'] else None)
@@ -592,6 +612,7 @@ class MLB_Scrape:
             'pitcher_team':pitcher_team,
             'pitcher_team_id':pitcher_team_id,
             'ab_number':ab_number,
+            'inning':inning,
             'play_description':play_description,
             'play_code':play_code,
             'in_play':in_play,