Add memory optimization functions and implement chunked processing for name matching in app.py

Introduced helper functions for chunked name matching and DataFrame type optimization to enhance memory efficiency. Updated the data loading process to utilize these functions, ensuring better performance and reduced memory usage during portfolio and projections handling.

app.py

@@ -105,6 +105,105 @@ st.markdown("""
 
 </style>""", unsafe_allow_html=True)
 
+# Memory optimization helper functions
+def chunk_name_matching(portfolio_names, csv_names, chunk_size=1000):
+    """Process name matching in chunks to reduce memory usage"""
+    portfolio_match_dict = {}
+    unmatched_names = []
+    
+    for i in range(0, len(portfolio_names), chunk_size):
+        chunk = portfolio_names[i:i+chunk_size]
+        for portfolio_name in chunk:
+            match = process.extractOne(
+                portfolio_name,
+                csv_names,
+                score_cutoff=87
+            )
+            if match:
+                portfolio_match_dict[portfolio_name] = match[0]
+                if match[1] < 100:
+                    st.write(f"{portfolio_name} matched from portfolio to site csv {match[0]} with a score of {match[1]}%")
+            else:
+                portfolio_match_dict[portfolio_name] = portfolio_name
+                unmatched_names.append(portfolio_name)
+    
+    return portfolio_match_dict, unmatched_names
+
+def optimize_dataframe_dtypes(df):
+    """Optimize DataFrame data types for memory efficiency"""
+    for col in df.columns:
+        if df[col].dtype == 'object':
+            # Try to convert to category if many duplicates
+            if df[col].nunique() / len(df) < 0.5:
+                df[col] = df[col].astype('category')
+    return df
+
+def create_memory_efficient_mappings(projections_df, site_var, type_var, sport_var):
+    """Create mappings with optimized data types"""
+    # Optimize projections data types first
+    projections_df = projections_df.copy()
+    
+    # Convert to more efficient data types
+    if 'position' in projections_df.columns:
+        projections_df['position'] = projections_df['position'].astype('category')
+    if 'team' in projections_df.columns:
+        projections_df['team'] = projections_df['team'].astype('category')
+    if 'salary' in projections_df.columns:
+        projections_df['salary'] = projections_df['salary'].astype('int32')
+    if 'median' in projections_df.columns:
+        projections_df['median'] = projections_df['median'].astype('float32')
+    if 'ownership' in projections_df.columns:
+        projections_df['ownership'] = projections_df['ownership'].astype('float32')
+    if 'captain ownership' in projections_df.columns:
+        projections_df['captain ownership'] = projections_df['captain ownership'].astype('float32')
+    
+    # Create base mappings
+    base_mappings = {
+        'pos_map': dict(zip(projections_df['player_names'], projections_df['position'])),
+        'team_map': dict(zip(projections_df['player_names'], projections_df['team'])),
+        'salary_map': dict(zip(projections_df['player_names'], projections_df['salary'])),
+        'proj_map': dict(zip(projections_df['player_names'], projections_df['median'])),
+        'own_map': dict(zip(projections_df['player_names'], projections_df['ownership'])),
+        'own_percent_rank': dict(zip(projections_df['player_names'], projections_df['ownership'].rank(pct=True).astype('float32')))
+    }
+    
+    # Add site/type specific mappings
+    if site_var == 'Draftkings':
+        if type_var == 'Classic':
+            if sport_var == 'CS2' or sport_var == 'LOL':
+                base_mappings.update({
+                    'cpt_salary_map': dict(zip(projections_df['player_names'], projections_df['salary'] * 1.5)),
+                    'cpt_proj_map': dict(zip(projections_df['player_names'], projections_df['median'] * 1.5)),
+                    'cpt_own_map': dict(zip(projections_df['player_names'], projections_df['captain ownership']))
+                })
+            else:
+                base_mappings.update({
+                    'cpt_salary_map': dict(zip(projections_df['player_names'], projections_df['salary'])),
+                    'cpt_proj_map': dict(zip(projections_df['player_names'], projections_df['median'] * 1.5)),
+                    'cpt_own_map': dict(zip(projections_df['player_names'], projections_df['captain ownership']))
+                })
+        elif type_var == 'Showdown':
+            if sport_var == 'GOLF':
+                base_mappings.update({
+                    'cpt_salary_map': dict(zip(projections_df['player_names'], projections_df['salary'])),
+                    'cpt_proj_map': dict(zip(projections_df['player_names'], projections_df['median'])),
+                    'cpt_own_map': dict(zip(projections_df['player_names'], projections_df['ownership']))
+                })
+            else:
+                base_mappings.update({
+                    'cpt_salary_map': dict(zip(projections_df['player_names'], projections_df['salary'] * 1.5)),
+                    'cpt_proj_map': dict(zip(projections_df['player_names'], projections_df['median'] * 1.5)),
+                    'cpt_own_map': dict(zip(projections_df['player_names'], projections_df['captain ownership']))
+                })
+    elif site_var == 'Fanduel':
+        base_mappings.update({
+            'cpt_salary_map': dict(zip(projections_df['player_names'], projections_df['salary'] * 1.5)),
+            'cpt_proj_map': dict(zip(projections_df['player_names'], projections_df['median'] * 1.5)),
+            'cpt_own_map': dict(zip(projections_df['player_names'], projections_df['captain ownership']))
+        })
+    
+    return base_mappings
+
 def create_position_export_dict(column_name, csv_file, site_var, type_var, sport_var):
     try:
         # Remove any numbers from the column name to get the position
@@ -339,9 +438,20 @@ if selected_tab == 'Data Load':
                 else:
                     stack_dict = None
                 if st.session_state['portfolio'] is not None:
+
+                    # Optimize data types early for memory efficiency
+                    st.session_state['portfolio'] = optimize_dataframe_dtypes(st.session_state['portfolio'])
                     
                     st.success('Portfolio file loaded successfully!')
-                    st.session_state['portfolio'] = st.session_state['portfolio'].apply(lambda x: x.replace(player_wrong_names_mlb, player_right_names_mlb))
+                    for col in st.session_state['portfolio'].select_dtypes(include=['object', 'category']).columns:
+                        if st.session_state['portfolio'][col].dtype == 'category':
+                            # Handle categorical columns
+                            st.session_state['portfolio'][col] = st.session_state['portfolio'][col].cat.rename_categories(
+                                lambda x: player_right_names_mlb.get(x, x) if x in player_wrong_names_mlb else x
+                            )
+                        else:
+                            # Handle object columns
+                            st.session_state['portfolio'][col] = st.session_state['portfolio'][col].replace(player_wrong_names_mlb)
                     st.dataframe(st.session_state['portfolio'].head(10))
 
     with col3:
@@ -371,6 +481,8 @@ if selected_tab == 'Data Load':
             export_projections, projections = load_file(projections_file, site_var, type_var, sport_var, 'projections')
             if projections is not None:
                 st.success('Projections file loaded successfully!')
+                
+                # Optimize projections data types early
                 try:
                     projections['salary'] = projections['salary'].str.replace(',', '').str.replace('$', '').str.replace(' ', '')
                     st.write('replaced salary symbols')
@@ -381,16 +493,25 @@ if selected_tab == 'Data Load':
                     st.write('replaced ownership symbols')
                 except:
                     pass
-                projections['salary'] = projections['salary'].dropna().astype(int)
-                projections['ownership'] = projections['ownership'].astype(float)
+                
+                # Convert to efficient data types
+                projections['salary'] = projections['salary'].dropna().astype('int32')
+                projections['ownership'] = projections['ownership'].astype('float32')
+                
                 if projections['captain ownership'].isna().all():
                     projections['CPT_Own_raw'] = (projections['ownership'] / 2) * ((100 - (100-projections['ownership']))/100)
                     cpt_own_var = 100 / projections['CPT_Own_raw'].sum()
                     projections['captain ownership'] = projections['CPT_Own_raw'] * cpt_own_var
                     projections = projections.drop(columns='CPT_Own_raw', axis=1)
-                            
-                projections = projections.apply(lambda x: x.replace(player_wrong_names_mlb, player_right_names_mlb))
-                ### if the position column is empty, set to sport_var appropriate position
+                
+                projections['captain ownership'] = projections['captain ownership'].astype('float32')
+                projections['median'] = projections['median'].astype('float32')
+                
+                # More efficient string replacement for projections
+                for col in projections.select_dtypes(include=['object']).columns:
+                    projections[col] = projections[col].replace(player_wrong_names_mlb)
+                
+                # Set position/team variables if needed
                 if position_var is not None:
                     projections['position'] = position_var
                 if team_var is not None:
@@ -402,7 +523,7 @@ if selected_tab == 'Data Load':
         if st.session_state['portfolio'] is not None and projections is not None:
         
             st.subheader("Name Matching Analysis")
-            # Initialize projections_df in session state if it doesn't exist
+            
             # Get unique names from portfolio
             portfolio_names = get_portfolio_names(st.session_state['portfolio'])
             try:
@@ -411,78 +532,35 @@ if selected_tab == 'Data Load':
                 csv_names = st.session_state['csv_file']['Nickname'].tolist()
             projection_names = projections['player_names'].tolist()
             
-            # Create match dictionary for portfolio names to projection names
-            portfolio_match_dict = {}
-            unmatched_names = []
-            for portfolio_name in portfolio_names:
-                match = process.extractOne(
-                    portfolio_name,
-                    csv_names,
-                    score_cutoff=87
-                )
-                if match:
-                    portfolio_match_dict[portfolio_name] = match[0]
-                    if match[1] < 100:
-                        st.write(f"{portfolio_name} matched from portfolio to site csv {match[0]} with a score of {match[1]}%")
-                else:
-                    portfolio_match_dict[portfolio_name] = portfolio_name
-                    unmatched_names.append(portfolio_name)
+            # Use chunked name matching for memory efficiency
+            portfolio_match_dict, unmatched_names = chunk_name_matching(portfolio_names, csv_names)
             
-            # Update portfolio with matched names
-            portfolio = st.session_state['portfolio'].copy()
-            player_columns = [col for col in portfolio.columns 
+            # Update portfolio with matched names (in-place to save memory)
+            player_columns = [col for col in st.session_state['portfolio'].columns 
                             if col not in ['salary', 'median', 'Own']]
             
             # For each player column, update names using the match dictionary
             for col in player_columns:
-                portfolio[col] = portfolio[col].map(lambda x: portfolio_match_dict.get(x, x))
-            st.session_state['portfolio'] = portfolio
+                st.session_state['portfolio'][col] = st.session_state['portfolio'][col].map(lambda x: portfolio_match_dict.get(x, x))
             
-            # Create match dictionary for portfolio names to projection names
-            projections_match_dict = {}
-            unmatched_proj_names = []
-            for projections_name in projection_names:
-                match = process.extractOne(
-                    projections_name,
-                    csv_names,
-                    score_cutoff=87
-                )
-                if match:
-                    projections_match_dict[projections_name] = match[0]
-                    if match[1] < 100:
-                        st.write(f"{projections_name} matched from projections to site csv {match[0]} with a score of {match[1]}%")
-                else:
-                    projections_match_dict[projections_name] = projections_name
-                    unmatched_proj_names.append(projections_name)
+            # Create match dictionary for projections to CSV names (chunked)
+            projections_match_dict, unmatched_proj_names = chunk_name_matching(projection_names, csv_names)
             
             # Update projections with matched names
             projections['player_names'] = projections['player_names'].map(lambda x: projections_match_dict.get(x, x))
             st.session_state['projections_df'] = projections
-
+            
+            # Second round of matching (projections to portfolio)
             projections_names = st.session_state['projections_df']['player_names'].tolist()
             portfolio_names = get_portfolio_names(st.session_state['portfolio'])
             
-            # Create match dictionary for portfolio names to projection names
-            projections_match_dict = {}
-            unmatched_proj_names = []
-            for projections_name in projection_names:
-                match = process.extractOne(
-                    projections_name,
-                    portfolio_names,
-                    score_cutoff=87
-                )
-                if match:
-                    projections_match_dict[projections_name] = match[0]
-                    if match[1] < 100:
-                        st.write(f"{projections_name} matched from portfolio to projections {match[0]} with a score of {match[1]}%")
-                else:
-                    projections_match_dict[projections_name] = projections_name
-                    unmatched_proj_names.append(projections_name)
+            projections_match_dict2, unmatched_proj_names2 = chunk_name_matching(projection_names, portfolio_names)
             
             # Update projections with matched names
-            projections['player_names'] = projections['player_names'].map(lambda x: projections_match_dict.get(x, x))
+            projections['player_names'] = projections['player_names'].map(lambda x: projections_match_dict2.get(x, x))
             st.session_state['projections_df'] = projections
 
+            # Handle stacking if needed
             if sport_var in stacking_sports:
                 team_dict = dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['team']))
                 st.session_state['portfolio']['Stack'] = st.session_state['portfolio'].apply(
@@ -502,78 +580,23 @@ if selected_tab == 'Data Load':
                 st.session_state['stack_dict'] = dict(zip(st.session_state['portfolio'].index, st.session_state['portfolio']['Stack']))
                 st.session_state['size_dict'] = dict(zip(st.session_state['portfolio'].index, st.session_state['portfolio']['Size']))
 
+            # Create export dictionary
             try:
                 st.session_state['export_dict'] = dict(zip(st.session_state['csv_file']['Name'], st.session_state['csv_file']['Name + ID']))
             except:
                 st.session_state['export_dict'] = dict(zip(st.session_state['csv_file']['Nickname'], st.session_state['csv_file']['Id']))
+            
+            # Create memory-efficient mappings
             if 'map_dict' not in st.session_state:
-                if site_var == 'Draftkings':
-                    if type_var == 'Classic':
-                        if sport_var == 'CS2' or sport_var == 'LOL':
-                            st.session_state['map_dict'] = {
-                                'pos_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['position'])),
-                                'team_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['team'])),
-                                'salary_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['salary'])),
-                                'proj_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['median'])),
-                                'own_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['ownership'])),
-                                'own_percent_rank':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['ownership'].rank(pct=True))),
-                                'cpt_salary_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['salary'] * 1.5)),
-                                'cpt_proj_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['median'] * 1.5)),
-                                'cpt_own_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['captain ownership']))
-                            }
-                        elif sport_var != 'CS2' and sport_var != 'LOL':
-                            st.session_state['map_dict'] = {
-                                'pos_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['position'])),
-                                'team_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['team'])),
-                                'salary_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['salary'])),
-                                'proj_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['median'])),
-                                'own_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['ownership'])),
-                                'own_percent_rank':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['ownership'].rank(pct=True))),
-                                'cpt_salary_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['salary'])),
-                                'cpt_proj_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['median'] * 1.5)),
-                                'cpt_own_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['captain ownership']))
-                            }
-                    elif type_var == 'Showdown':
-                        if sport_var == 'GOLF':
-                            st.session_state['map_dict'] = {
-                                'pos_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['position'])),
-                                'team_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['team'])),
-                                'salary_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['salary'])),
-                                'proj_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['median'])),
-                                'own_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['ownership'])),
-                                'own_percent_rank':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['ownership'].rank(pct=True))),
-                                'cpt_salary_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['salary'])),
-                                'cpt_proj_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['median'])),
-                                'cpt_own_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['ownership']))
-                            }
-                        if sport_var != 'GOLF':
-                            st.session_state['map_dict'] = {
-                                'pos_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['position'])),
-                                'team_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['team'])),
-                                'salary_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['salary'])),
-                                'proj_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['median'])),
-                                'own_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['ownership'])),
-                                'own_percent_rank':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['ownership'].rank(pct=True))),
-                                'cpt_salary_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['salary'] * 1.5)),
-                                'cpt_proj_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['median'] * 1.5)),
-                                'cpt_own_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['captain ownership']))
-                            }
-                elif site_var == 'Fanduel':
-                    st.session_state['map_dict'] = {
-                        'pos_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['position'])),
-                        'team_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['team'])),
-                        'salary_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['salary'])),
-                        'proj_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['median'])),
-                        'own_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['ownership'])),
-                        'own_percent_rank':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['ownership'].rank(pct=True))),
-                        'cpt_salary_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['salary'] * 1.5)),
-                        'cpt_proj_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['median'] * 1.5)),
-                        'cpt_own_map':dict(zip(st.session_state['projections_df']['player_names'], st.session_state['projections_df']['captain ownership']))
-                    }
+                st.session_state['map_dict'] = create_memory_efficient_mappings(st.session_state['projections_df'], site_var, type_var, sport_var)
+            
+            # Store portfolio in compressed format and clean up
             st.session_state['origin_portfolio'] = st.session_state['portfolio']
             buffer = io.BytesIO()
             st.session_state['portfolio'].to_parquet(buffer, compression='snappy')
-            st.session_state['origin_portfolio'] = buffer.getvalue()
+            st.session_state['origin_portfolio_compressed'] = buffer.getvalue()
+            
+            # Clear large objects from session state to free memory
             del st.session_state['portfolio'], st.session_state['export_portfolio']
 
 # with tab2: