Refactor DataFrame optimization in app.py to enhance memory efficiency. Disable categorical conversion for specific columns to avoid issues with exposure_spread, while implementing smarter handling for numeric types. Introduce a new function to prepare DataFrames for exposure_spread, ensuring compatibility and improved performance during calculations.

app.py

@@ -131,13 +131,29 @@ def chunk_name_matching(portfolio_names, csv_names, chunk_size=1000):
 
 def optimize_dataframe_dtypes(df):
     """Optimize DataFrame data types for memory efficiency"""
+    # For now, disable categorical conversion entirely to avoid issues with exposure_spread and other operations
+    # This maintains compatibility while still providing other memory optimizations
+    # Future enhancement: implement smarter categorical handling that preserves mutability
+    
+    # Only optimize numeric columns to more efficient dtypes
     for col in df.columns:
-        if df[col].dtype == 'object':
-            # Only convert to category if there are many duplicates AND it's not a player column
-            # Player columns need to stay as object for mapping operations
-            excluded_cols = ['salary', 'median', 'Own', 'Finish_percentile', 'Dupes', 'Stack', 'Size', 'Win%', 'Lineup Edge', 'Weighted Own', 'Geomean', 'Diversity']
-            if col not in excluded_cols and df[col].nunique() / len(df) < 0.3:
-                df[col] = df[col].astype('category')
+        if df[col].dtype == 'float64':
+            # Convert float64 to float32 if possible without significant precision loss
+            try:
+                if df[col].max() < 3.4e+38 and df[col].min() > -3.4e+38:  # float32 range
+                    df[col] = df[col].astype('float32')
+            except:
+                pass
+        elif df[col].dtype == 'int64':
+            # Convert int64 to smaller int types if possible
+            try:
+                if df[col].max() <= 32767 and df[col].min() >= -32768:
+                    df[col] = df[col].astype('int16')
+                elif df[col].max() <= 2147483647 and df[col].min() >= -2147483648:
+                    df[col] = df[col].astype('int32')
+            except:
+                pass
+    
     return df
 
 def create_memory_efficient_mappings(projections_df, site_var, type_var, sport_var):
@@ -284,6 +300,11 @@ def calculate_lineup_metrics(df, player_columns, map_dict, type_var, sport_var,
     """Centralized function to calculate salary, median, and ownership efficiently"""
     df = df.copy()  # Work on a copy to avoid modifying original
     
+    # Ensure player columns are object type to avoid categorical issues with exposure_spread
+    for col in player_columns:
+        if df[col].dtype.name == 'category':
+            df[col] = df[col].astype('object')
+    
     # Vectorized calculations
     df['salary'] = calculate_salary_vectorized(df[player_columns], player_columns, map_dict, type_var, sport_var)
     df['median'] = calculate_median_vectorized(df[player_columns], player_columns, map_dict, type_var, sport_var)
@@ -358,6 +379,17 @@ def create_team_filter_mask(df, player_columns, team_map, teams_to_filter, focus
     
     return mask
 
+def prepare_dataframe_for_exposure_spread(df, player_columns):
+    """Ensure DataFrame is ready for exposure_spread by converting player columns to object type"""
+    df_prepared = df.copy()
+    
+    # Convert any categorical player columns back to object type
+    for col in player_columns:
+        if col in df_prepared.columns and df_prepared[col].dtype.name == 'category':
+            df_prepared[col] = df_prepared[col].astype('object')
+    
+    return df_prepared
+
 def create_position_export_dict(column_name, csv_file, site_var, type_var, sport_var):
     try:
         # Remove any numbers from the column name to get the position
@@ -2037,7 +2069,9 @@ if selected_tab == 'Manage Portfolio':
                         exp_submitted = st.form_submit_button("Export")
                     if reg_submitted:
                         st.session_state['settings_base'] = False
-                        parsed_frame = exposure_spread(st.session_state['working_frame'], st.session_state['exposure_player'], exposure_target, ignore_stacks, remove_teams_exposure, specific_replacements, specific_columns, st.session_state['projections_df'], sport_var, type_var, salary_max, stacking_sports)
+                        # Prepare DataFrame for exposure_spread to avoid categorical issues
+                        working_frame_prepared = prepare_dataframe_for_exposure_spread(st.session_state['working_frame'], st.session_state['player_columns'])
+                        parsed_frame = exposure_spread(working_frame_prepared, st.session_state['exposure_player'], exposure_target, ignore_stacks, remove_teams_exposure, specific_replacements, specific_columns, st.session_state['projections_df'], sport_var, type_var, salary_max, stacking_sports)
                         
                         # Use consolidated calculation function
                         parsed_frame = calculate_lineup_metrics(
@@ -2056,7 +2090,9 @@ if selected_tab == 'Manage Portfolio':
                         st.session_state['export_merge'] = st.session_state['working_frame'].copy()
                     elif exp_submitted:
                         st.session_state['settings_base'] = False
-                        parsed_frame = exposure_spread(st.session_state['export_base'], st.session_state['exposure_player'], exposure_target, ignore_stacks, remove_teams_exposure, specific_replacements, specific_columns, st.session_state['projections_df'], sport_var, type_var, salary_max, stacking_sports)
+                        # Prepare DataFrame for exposure_spread to avoid categorical issues
+                        export_base_prepared = prepare_dataframe_for_exposure_spread(st.session_state['export_base'], st.session_state['player_columns'])
+                        parsed_frame = exposure_spread(export_base_prepared, st.session_state['exposure_player'], exposure_target, ignore_stacks, remove_teams_exposure, specific_replacements, specific_columns, st.session_state['projections_df'], sport_var, type_var, salary_max, stacking_sports)
                         
                         # Use consolidated calculation function for export
                         parsed_frame = calculate_lineup_metrics(