Update app.py

app.py

@@ -38,7 +38,13 @@ def standardize_dataframe(df: pd.DataFrame) -> pd.DataFrame:
     - Creates an 'employee_name' column if missing but first_name and last_name exist.
     - Combines duplicate key columns (e.g., multiple 'salary' or 'tin' columns) into one.
     - Forces the key columns 'tin' and 'employee_name' to be strings.
+    - Drops any middle name columns.
     """
+    # Drop any column that appears to be a middle name
+    middle_name_cols = [col for col in df.columns if 'middle_name' in col.lower()]
+    if middle_name_cols:
+        df = df.drop(columns=middle_name_cols)
+    
     rename_map = {}
     for col in df.columns:
         col_lower = col.lower()
@@ -150,48 +156,65 @@ def read_excel_file(file) -> pd.DataFrame:
 
 def merge_with_master(processed_files):
     """
-    Merge multiple DataFrames by using the earnings schedule file as the master.
-    This modified logic looks for files whose names include 'earnings', 'paye', or 'template'.
-    The PAYE figures and the template info are merged onto the earnings sheet using the 'tin' key.
+    Merge multiple DataFrames using a two-step process:
+    1. Use the earnings file as the master. Drop its inaccurate 'tin' column.
+    2. Merge template info onto earnings via 'employee_name' (constructed from first and last names).
+    3. Then merge the resulting DataFrame with the PAYE file using the (correct) 'tin' key.
     """
-    master_file = None
+    earnings_file = None
     paye_file = None
     template_file = None
     for file_info in processed_files:
         lower_filename = file_info["filename"].lower()
         if "earnings" in lower_filename:
-            master_file = file_info
+            earnings_file = file_info
         elif "paye" in lower_filename:
             paye_file = file_info
         elif "template" in lower_filename:
             template_file = file_info
-    if not master_file:
+    if not earnings_file:
         st.warning("No earnings file found as master. Using the first file as master.")
-        master_file = processed_files[0]
+        earnings_file = processed_files[0]
     
-    master_df = master_file["df"]
-    merged_df = master_df.copy()
+    # Start with the earnings DataFrame as master
+    earnings_df = earnings_file["df"]
+    # Drop the inaccurate 'tin' column from earnings, if present
+    if 'tin' in earnings_df.columns:
+        earnings_df = earnings_df.drop(columns=['tin'])
+    # Ensure any middle_name column is dropped (already handled in standardization, but double-check)
+    if 'middle_name' in earnings_df.columns:
+        earnings_df = earnings_df.drop(columns=['middle_name'])
     
-    # Merge PAYE figures onto the earnings sheet
-    if paye_file is not None:
-        st.write(f"Merging PAYE figures from '{paye_file['filename']}' onto the earnings sheet using key 'tin'.")
-        if 'tin' in merged_df.columns and 'tin' in paye_file["df"].columns:
-            merged_df = merged_df.merge(paye_file["df"], on='tin', how='left', suffixes=('', '_paye'))
-        else:
-            st.warning("Column 'tin' missing in either the earnings or PAYE file. Skipping PAYE merge.")
-    else:
-        st.warning("No PAYE file detected.")
+    merged_df = earnings_df.copy()
     
-    # Merge template info onto the earnings sheet
+    # Merge template info onto earnings using 'employee_name' as key
     if template_file is not None:
-        st.write(f"Merging template info from '{template_file['filename']}' onto the earnings sheet using key 'tin'.")
-        if 'tin' in merged_df.columns and 'tin' in template_file["df"].columns:
-            merged_df = merged_df.merge(template_file["df"], on='tin', how='left', suffixes=('', '_template'))
+        st.write(f"Merging template info from '{template_file['filename']}' onto the earnings sheet using key 'employee_name'.")
+        template_df = template_file["df"]
+        # Drop any middle_name column from the template file
+        if 'middle_name' in template_df.columns:
+            template_df = template_df.drop(columns=['middle_name'])
+        # Ensure template has an 'employee_name' column (constructed if necessary)
+        if 'employee_name' not in template_df.columns and 'first_name' in template_df.columns and 'last_name' in template_df.columns:
+            template_df['employee_name'] = template_df['first_name'].astype(str).str.strip() + ' ' + template_df['last_name'].astype(str).str.strip()
+        if 'employee_name' in merged_df.columns and 'employee_name' in template_df.columns:
+            merged_df = merged_df.merge(template_df, on='employee_name', how='left', suffixes=('', '_template'))
         else:
-            st.warning("Column 'tin' missing in either the earnings or template file. Skipping template merge.")
+            st.warning("Column 'employee_name' missing in either the earnings or template file. Skipping template merge.")
     else:
         st.warning("No template file detected.")
     
+    # Merge PAYE figures onto the merged DataFrame using 'tin'
+    if paye_file is not None:
+        st.write(f"Merging PAYE figures from '{paye_file['filename']}' onto the merged sheet using key 'tin'.")
+        paye_df = paye_file["df"]
+        if 'tin' in merged_df.columns and 'tin' in paye_df.columns:
+            merged_df = merged_df.merge(paye_df, on='tin', how='left', suffixes=('', '_paye'))
+        else:
+            st.warning("Column 'tin' missing in either the merged or PAYE file. Skipping PAYE merge.")
+    else:
+        st.warning("No PAYE file detected.")
+    
     return merged_df
 
 def safe_display_df(df: pd.DataFrame) -> pd.DataFrame: