Update app.py

app.py

@@ -15,7 +15,7 @@ EXAMPLE_FILES = {
     "cashflow_base": os.path.join(EXAMPLE_DATA_DIR, "cashflows_seriatim_10K.xlsx"),
     "cashflow_lapse": os.path.join(EXAMPLE_DATA_DIR, "cashflows_seriatim_10K_lapse50.xlsx"),
     "cashflow_mort": os.path.join(EXAMPLE_DATA_DIR, "cashflows_seriatim_10K_mort15.xlsx"),
-    "policy_data": os.path.join(EXAMPLE_DATA_DIR, "model_point_table.xlsx"), # Assuming this is the correct path/name for the example
+    "policy_data": os.path.join(EXAMPLE_DATA_DIR, "model_point_table.xlsx"),
     "pv_base": os.path.join(EXAMPLE_DATA_DIR, "pv_seriatim_10K.xlsx"),
     "pv_lapse": os.path.join(EXAMPLE_DATA_DIR, "pv_seriatim_10K_lapse50.xlsx"),
     "pv_mort": os.path.join(EXAMPLE_DATA_DIR, "pv_seriatim_10K_mort15.xlsx"),
@@ -68,85 +68,60 @@ class Clusters:
     def compare_total(self, df, agg=None):
         """Aggregate df by columns"""
         if agg:
-            # cols = df.columns # Not used
-            op = {c: (agg[c] if c in agg else 'sum') for c in df.columns}
-            actual = df.agg(op)
+            # Calculate actual values using specified aggregation
+            actual_values = {}
+            for col in df.columns:
+                if agg.get(col, 'sum') == 'mean':
+                    actual_values[col] = df[col].mean()
+                else:  # sum
+                    actual_values[col] = df[col].sum()
+            actual = pd.Series(actual_values)
             
-            # For estimate, ensure aggregation ops are correctly applied *after* scaling
-            scaled_reps = self.extract_and_scale_reps(df, agg=op) # Pass op to ensure correct scaling for mean
+            # Calculate estimate values
+            reps_unscaled = self.extract_reps(df)
+            estimate_values = {}
             
-            # Corrected aggregation for estimate when 'mean' is involved
-            estimate_agg_ops = {}
-            for col_name, agg_type in op.items():
-                if agg_type == 'mean':
+            for col in df.columns:
+                if agg.get(col, 'sum') == 'mean':
                     # Weighted average for mean columns
-                    estimate_agg_ops[col_name] = lambda s, c=col_name: (s * self.policy_count.reindex(s.index)).sum() / self.policy_count.reindex(s.index).sum() if c in self.policy_count.name else s.mean()
-                else: # 'sum'
-                    estimate_agg_ops[col_name] = 'sum'
+                    weighted_sum = (reps_unscaled[col] * self.policy_count).sum()
+                    total_weight = self.policy_count.sum()
+                    estimate_values[col] = weighted_sum / total_weight if total_weight > 0 else 0
+                else:  # sum
+                    estimate_values[col] = (reps_unscaled[col] * self.policy_count).sum()
             
-            # Need to handle the case where extract_and_scale_reps already applied scaling for sum
-            # The logic in extract_and_scale_reps is:
-            # mult = pd.DataFrame({c: (self.policy_count if (c not in agg or agg[c] == 'sum') else 1) for c in cols})
-            # This means 'mean' columns are NOT multiplied by policy_count initially.
+            estimate = pd.Series(estimate_values)
             
-            # Let's re-think the estimate aggregation for 'mean'
-            estimate_scaled = self.extract_and_scale_reps(df, agg=op) # agg=op is important here
-            
-            final_estimate_ops = {}
-            for col, method in op.items():
-                if method == 'mean':
-                    # For mean, we need the sum of (value * policy_count) / sum(policy_count)
-                    # extract_and_scale_reps with agg=op should have scaled sum-columns by policy_count
-                    # and mean-columns by 1. So, for mean columns in estimate_scaled, we need to multiply by policy_count,
-                    # sum them up, and divide by total policy_count.
-                    # However, the current extract_and_scale_reps scales 'mean' columns by 1.
-                    # So we need to take the mean of these scaled (by 1) values, but it should be a weighted mean.
-
-                    # Let's try to be more direct:
-                    # Get the representative policies (unscaled for mean columns)
-                    reps_unscaled_for_mean = self.extract_reps(df)
-                    estimate_values = {}
-                    for c in df.columns:
-                        if op[c] == 'sum':
-                           estimate_values[c] = reps_unscaled_for_mean[c].mul(self.policy_count, axis=0).sum()
-                        elif op[c] == 'mean':
-                           weighted_sum = (reps_unscaled_for_mean[c] * self.policy_count).sum()
-                           total_weight = self.policy_count.sum()
-                           estimate_values[c] = weighted_sum / total_weight if total_weight else 0
-                    estimate = pd.Series(estimate_values)
-
-                else: # original 'sum' logic for all columns
-                    final_estimate_ops[col] = 'sum' # All columns in estimate_scaled are ready to be summed up
-                    estimate = estimate_scaled.agg(final_estimate_ops)
-
-
-        else: # Original logic if no agg is specified (all sum)
+        else:  # Original logic if no agg is specified (all sum)
             actual = df.sum()
             estimate = self.extract_and_scale_reps(df).sum()
         
-        return pd.DataFrame({'actual': actual, 'estimate': estimate, 'error': estimate / actual - 1})
+        # Calculate error, handling division by zero
+        error = np.where(actual != 0, estimate / actual - 1, 0)
+        
+        return pd.DataFrame({'actual': actual, 'estimate': estimate, 'error': error})
 
 
 def plot_cashflows_comparison(cfs_list, cluster_obj, titles):
     """Create cashflow comparison plots"""
     if not cfs_list or not cluster_obj or not titles:
-        return None # Or a placeholder image
+        return None
     num_plots = len(cfs_list)
     if num_plots == 0:
         return None
 
-    # Determine subplot layout (e.g., 2x2 or adapt)
+    # Determine subplot layout
     cols = 2
     rows = (num_plots + cols - 1) // cols
     
-    fig, axes = plt.subplots(rows, cols, figsize=(15, 5 * rows), squeeze=False) # Ensure axes is always 2D
+    fig, axes = plt.subplots(rows, cols, figsize=(15, 5 * rows), squeeze=False)
     axes = axes.flatten()
     
     for i, (df, title) in enumerate(zip(cfs_list, titles)):
         if i < len(axes):
             comparison = cluster_obj.compare_total(df)
             comparison[['actual', 'estimate']].plot(ax=axes[i], grid=True, title=title)
-            axes[i].set_xlabel('Time') # Assuming x-axis is time for cashflows
+            axes[i].set_xlabel('Time')
             axes[i].set_ylabel('Value')
     
     # Hide any unused subplots
@@ -155,10 +130,10 @@ def plot_cashflows_comparison(cfs_list, cluster_obj, titles):
         
     plt.tight_layout()
     buf = io.BytesIO()
-    plt.savefig(buf, format='png', dpi=100) # Lowered DPI slightly for potentially faster rendering
+    plt.savefig(buf, format='png', dpi=100)
     buf.seek(0)
     img = Image.open(buf)
-    plt.close(fig) # Ensure figure is closed
+    plt.close(fig)
     return img
 
 def plot_scatter_comparison(df_compare_output, title):
@@ -175,7 +150,7 @@ def plot_scatter_comparison(df_compare_output, title):
         plt.close(fig)
         return img
 
-    fig, ax = plt.subplots(figsize=(12, 8)) # Use a single Axes object
+    fig, ax = plt.subplots(figsize=(12, 8))
     
     if not isinstance(df_compare_output.index, pd.MultiIndex) or df_compare_output.index.nlevels < 2:
          gr.Warning("Scatter plot data is not in the expected multi-index format. Plotting raw actual vs estimate.")
@@ -187,10 +162,9 @@ def plot_scatter_comparison(df_compare_output, title):
         for item_level, color_val in zip(unique_levels, colors):
             subset = df_compare_output.xs(item_level, level=1)
             ax.scatter(subset['actual'], subset['estimate'], color=color_val, s=9, alpha=0.6, label=item_level)
-        if len(unique_levels) > 1 and len(unique_levels) <=10: # Add legend if not too many items
+        if len(unique_levels) > 1 and len(unique_levels) <= 10:
             ax.legend(title=df_compare_output.index.names[1])
 
-
     ax.set_xlabel('Actual')
     ax.set_ylabel('Estimate')
     ax.set_title(title)
@@ -201,7 +175,7 @@ def plot_scatter_comparison(df_compare_output, title):
         np.min([ax.get_xlim(), ax.get_ylim()]),
         np.max([ax.get_xlim(), ax.get_ylim()]),
     ]
-    if lims[0] != lims[1]: # Avoid issues if all data is zero or a single point
+    if lims[0] != lims[1]:
       ax.plot(lims, lims, 'r-', linewidth=0.5)
       ax.set_xlim(lims)
       ax.set_ylim(lims)
@@ -229,30 +203,24 @@ def process_files(cashflow_base_path, cashflow_lapse_path, cashflow_mort_path,
         if all(col in pol_data_full.columns for col in required_cols):
             pol_data = pol_data_full[required_cols]
         else:
-            # Fallback or error if columns are missing. For now, try to use as is or a subset.
             gr.Warning(f"Policy data might be missing required columns. Found: {pol_data_full.columns.tolist()}")
             pol_data = pol_data_full
 
-
         pvs = pd.read_excel(pv_base_path, index_col=0)
         pvs_lapse50 = pd.read_excel(pv_lapse_path, index_col=0)
         pvs_mort15 = pd.read_excel(pv_mort_path, index_col=0)
         
         cfs_list = [cfs, cfs_lapse50, cfs_mort15]
-        # pvs_list = [pvs, pvs_lapse50, pvs_mort15] # Not directly used for plotting in this structure
         scen_titles = ['Base', 'Lapse+50%', 'Mort+15%']
         
         results = {}
         
-        mean_attrs = {'age_at_entry':'mean', 'policy_term':'mean', 'duration_mth':'mean', 'sum_assured': 'sum'} # sum_assured is usually summed
+        mean_attrs = {'age_at_entry':'mean', 'policy_term':'mean', 'duration_mth':'mean', 'sum_assured': 'sum'}
 
         # --- 1. Cashflow Calibration ---
         cluster_cfs = Clusters(cfs)
         
         results['cf_total_base_table'] = cluster_cfs.compare_total(cfs)
-        # results['cf_total_lapse_table'] = cluster_cfs.compare_total(cfs_lapse50) # For full detail if needed
-        # results['cf_total_mort_table'] = cluster_cfs.compare_total(cfs_mort15)
-
         results['cf_policy_attrs_total'] = cluster_cfs.compare_total(pol_data, agg=mean_attrs)
         
         results['cf_pv_total_base'] = cluster_cfs.compare_total(pvs)
@@ -261,16 +229,14 @@ def process_files(cashflow_base_path, cashflow_lapse_path, cashflow_mort_path,
         
         results['cf_cashflow_plot'] = plot_cashflows_comparison(cfs_list, cluster_cfs, scen_titles)
         results['cf_scatter_cashflows_base'] = plot_scatter_comparison(cluster_cfs.compare(cfs), 'Cashflow Calib. - Cashflows (Base)')
-        # results['cf_scatter_policy_attrs'] = plot_scatter_comparison(cluster_cfs.compare(pol_data, agg=mean_attrs), 'Cashflow Calib. - Policy Attributes')
-        # results['cf_scatter_pvs_base'] = plot_scatter_comparison(cluster_cfs.compare(pvs), 'Cashflow Calib. - PVs (Base)')
 
         # --- 2. Policy Attribute Calibration ---
         # Standardize policy attributes
-        if not pol_data.empty and (pol_data.max() - pol_data.min()).all() != 0 : # Avoid division by zero if a column is constant
+        if not pol_data.empty and (pol_data.max() - pol_data.min()).all() != 0:
              loc_vars_attrs = (pol_data - pol_data.min()) / (pol_data.max() - pol_data.min())
         else:
             gr.Warning("Policy data for attribute calibration is empty or has no variance. Skipping attribute calibration plots.")
-            loc_vars_attrs = pol_data # or handle as an error/skip
+            loc_vars_attrs = pol_data
         
         if not loc_vars_attrs.empty:
             cluster_attrs = Clusters(loc_vars_attrs)
@@ -279,16 +245,13 @@ def process_files(cashflow_base_path, cashflow_lapse_path, cashflow_mort_path,
             results['attr_total_pv_base'] = cluster_attrs.compare_total(pvs)
             results['attr_cashflow_plot'] = plot_cashflows_comparison(cfs_list, cluster_attrs, scen_titles)
             results['attr_scatter_cashflows_base'] = plot_scatter_comparison(cluster_attrs.compare(cfs), 'Policy Attr. Calib. - Cashflows (Base)')
-            # results['attr_scatter_policy_attrs'] = plot_scatter_comparison(cluster_attrs.compare(pol_data, agg=mean_attrs), 'Policy Attr. Calib. - Policy Attributes')
-
-        else: # Fill with None if skipped
+        else:
             results['attr_total_cf_base'] = pd.DataFrame()
             results['attr_policy_attrs_total'] = pd.DataFrame()
             results['attr_total_pv_base'] = pd.DataFrame()
             results['attr_cashflow_plot'] = None
             results['attr_scatter_cashflows_base'] = None
 
-
         # --- 3. Present Value Calibration ---
         cluster_pvs = Clusters(pvs)
         
@@ -301,67 +264,63 @@ def process_files(cashflow_base_path, cashflow_lapse_path, cashflow_mort_path,
         
         results['pv_cashflow_plot'] = plot_cashflows_comparison(cfs_list, cluster_pvs, scen_titles)
         results['pv_scatter_pvs_base'] = plot_scatter_comparison(cluster_pvs.compare(pvs), 'PV Calib. - PVs (Base)')
-        # results['pv_scatter_cashflows_base'] = plot_scatter_comparison(cluster_pvs.compare(cfs), 'PV Calib. - Cashflows (Base)')
-
 
         # --- Summary Comparison Plot Data ---
-        # Error metric: Mean Absolute Percentage Error for the 'TOTAL' net present value of cashflows (usually the 'PV_NetCF' column)
-        # Or sum of absolute errors if percentage is problematic (e.g. actual is zero)
-        # For simplicity, using mean of the 'error' column from compare_total for key metrics
+        # Error metric for key PV column or mean absolute error
         
         error_data = {}
         
+        # Function to safely get error value
+        def get_error_safe(compare_result, col_name=None):
+            if compare_result.empty:
+                return np.nan
+            if col_name and col_name in compare_result.index:
+                return abs(compare_result.loc[col_name, 'error'])
+            else:
+                # Use mean absolute error if specific column not found
+                return abs(compare_result['error']).mean()
+        
+        # Determine key PV column (try common names)
+        key_pv_col = None
+        for potential_col in ['PV_NetCF', 'pv_net_cf', 'net_cf_pv', 'PV_Net_CF']:
+            if potential_col in pvs.columns:
+                key_pv_col = potential_col
+                break
+        
         # Cashflow Calibration Errors
-        if 'PV_NetCF' in pvs.columns:
-            err_cf_cal_pv_base = cluster_cfs.compare_total(pvs).loc['PV_NetCF', 'error']
-            err_cf_cal_pv_lapse = cluster_cfs.compare_total(pvs_lapse50).loc['PV_NetCF', 'error']
-            err_cf_cal_pv_mort = cluster_cfs.compare_total(pvs_mort15).loc['PV_NetCF', 'error']
-            error_data['CF Calib. (PV NetCF)'] = [
-                abs(err_cf_cal_pv_base), abs(err_cf_cal_pv_lapse), abs(err_cf_cal_pv_mort)
-            ]
-        else: # Fallback if PV_NetCF is not present
-            error_data['CF Calib. (PV NetCF)'] = [
-                abs(cluster_cfs.compare_total(pvs)['error'].mean()),
-                abs(cluster_cfs.compare_total(pvs_lapse50)['error'].mean()),
-                abs(cluster_cfs.compare_total(pvs_mort15)['error'].mean())
-            ]
-
+        error_data['CF Calib.'] = [
+            get_error_safe(cluster_cfs.compare_total(pvs), key_pv_col),
+            get_error_safe(cluster_cfs.compare_total(pvs_lapse50), key_pv_col),
+            get_error_safe(cluster_cfs.compare_total(pvs_mort15), key_pv_col)
+        ]
 
         # Policy Attribute Calibration Errors
-        if not loc_vars_attrs.empty and 'PV_NetCF' in pvs.columns:
-            err_attr_cal_pv_base = cluster_attrs.compare_total(pvs).loc['PV_NetCF', 'error']
-            err_attr_cal_pv_lapse = cluster_attrs.compare_total(pvs_lapse50).loc['PV_NetCF', 'error']
-            err_attr_cal_pv_mort = cluster_attrs.compare_total(pvs_mort15).loc['PV_NetCF', 'error']
-            error_data['Attr Calib. (PV NetCF)'] = [
-                abs(err_attr_cal_pv_base), abs(err_attr_cal_pv_lapse), abs(err_attr_cal_pv_mort)
+        if not loc_vars_attrs.empty:
+            error_data['Attr Calib.'] = [
+                get_error_safe(cluster_attrs.compare_total(pvs), key_pv_col),
+                get_error_safe(cluster_attrs.compare_total(pvs_lapse50), key_pv_col),
+                get_error_safe(cluster_attrs.compare_total(pvs_mort15), key_pv_col)
             ]
         else:
-             error_data['Attr Calib. (PV NetCF)'] = [np.nan, np.nan, np.nan] # Placeholder if skipped
-
+            error_data['Attr Calib.'] = [np.nan, np.nan, np.nan]
 
         # Present Value Calibration Errors
-        if 'PV_NetCF' in pvs.columns:
-            err_pv_cal_pv_base = cluster_pvs.compare_total(pvs).loc['PV_NetCF', 'error']
-            err_pv_cal_pv_lapse = cluster_pvs.compare_total(pvs_lapse50).loc['PV_NetCF', 'error']
-            err_pv_cal_pv_mort = cluster_pvs.compare_total(pvs_mort15).loc['PV_NetCF', 'error']
-            error_data['PV Calib. (PV NetCF)'] = [
-                abs(err_pv_cal_pv_base), abs(err_pv_cal_pv_lapse), abs(err_pv_cal_pv_mort)
-            ]
-        else:
-            error_data['PV Calib. (PV NetCF)'] = [
-                abs(cluster_pvs.compare_total(pvs)['error'].mean()),
-                abs(cluster_pvs.compare_total(pvs_lapse50)['error'].mean()),
-                abs(cluster_pvs.compare_total(pvs_mort15)['error'].mean())
-            ]
+        error_data['PV Calib.'] = [
+            get_error_safe(cluster_pvs.compare_total(pvs), key_pv_col),
+            get_error_safe(cluster_pvs.compare_total(pvs_lapse50), key_pv_col),
+            get_error_safe(cluster_pvs.compare_total(pvs_mort15), key_pv_col)
+        ]
         
         # Create Summary Plot
         summary_df = pd.DataFrame(error_data, index=['Base', 'Lapse+50%', 'Mort+15%'])
         
         fig_summary, ax_summary = plt.subplots(figsize=(10, 6))
         summary_df.plot(kind='bar', ax=ax_summary, grid=True)
-        ax_summary.set_ylabel('Mean Absolute Error (of PV_NetCF)')
-        ax_summary.set_title('Calibration Method Comparison - Error in Total PV Net Cashflow')
+        ax_summary.set_ylabel('Absolute Error Rate')
+        title_suffix = f' ({key_pv_col})' if key_pv_col else ' (Mean Absolute Error)'
+        ax_summary.set_title(f'Calibration Method Comparison - Error in Total PV{title_suffix}')
         ax_summary.tick_params(axis='x', rotation=0)
+        ax_summary.legend(title='Calibration Method')
         plt.tight_layout()
         
         buf_summary = io.BytesIO()
@@ -384,7 +343,7 @@ def process_files(cashflow_base_path, cashflow_lapse_path, cashflow_mort_path,
 
 
 def create_interface():
-    with gr.Blocks(title="Cluster Model Points Analysis") as demo: # Removed theme
+    with gr.Blocks(title="Cluster Model Points Analysis") as demo:
         gr.Markdown("""
         # Cluster Model Points Analysis
         
@@ -422,7 +381,7 @@ def create_interface():
         
         with gr.Tabs():
             with gr.TabItem("📊 Summary"):
-                summary_plot_output = gr.Image(label="Calibration Methods Comparison (Error in Total PV Net Cashflow)")
+                summary_plot_output = gr.Image(label="Calibration Methods Comparison")
             
             with gr.TabItem("💸 Cashflow Calibration"):
                 gr.Markdown("### Results: Using Annual Cashflows as Calibration Variables")
@@ -479,16 +438,12 @@ def create_interface():
         
         # --- Action for Analyze Button ---
         def handle_analysis(f1, f2, f3, f4, f5, f6, f7):
-            # Ensure all files are provided (either by upload or example load)
             files = [f1, f2, f3, f4, f5, f6, f7]
-            # Gradio File objects have a .name attribute for the temp path
-            # If they are already strings (from example load), they are paths
             
             file_paths = []
             for i, f_obj in enumerate(files):
                 if f_obj is None:
                     gr.Error(f"Missing file input for argument {i+1}. Please upload all files or load examples.")
-                    # Return Nones for all output components
                     return [None] * len(get_all_output_components())
                 
                 # If f_obj is a Gradio FileData object (from direct upload)
@@ -501,11 +456,9 @@ def create_interface():
                     gr.Error(f"Invalid file input for argument {i+1}. Type: {type(f_obj)}")
                     return [None] * len(get_all_output_components())
 
-
             results = process_files(*file_paths)
             
             if "error" in results:
-                # Error already displayed by process_files or here
                 return [None] * len(get_all_output_components())
             
             return [
@@ -532,11 +485,10 @@ def create_interface():
 
         # --- Action for Load Example Data Button ---
         def load_example_files():
-            # Check if all example files exist
             missing_files = [fp for fp in EXAMPLE_FILES.values() if not os.path.exists(fp)]
             if missing_files:
                 gr.Error(f"Missing example data files in '{EXAMPLE_DATA_DIR}': {', '.join(missing_files)}. Please ensure they exist.")
-                return [None] * 7 # Return Nones for all file inputs
+                return [None] * 7
             
             gr.Info("Example data paths loaded. Click 'Analyze Dataset'.")
             return [
@@ -555,17 +507,10 @@ def create_interface():
     return demo
 
 if __name__ == "__main__":
-    # Create the eg_data directory if it doesn't exist (for testing, user should create it with files)
     if not os.path.exists(EXAMPLE_DATA_DIR):
         os.makedirs(EXAMPLE_DATA_DIR)
         print(f"Created directory '{EXAMPLE_DATA_DIR}'. Please place example Excel files there.")
-        # You might want to add dummy files here for basic testing if the real files aren't present
-        # For example:
-        # with open(os.path.join(EXAMPLE_DATA_DIR, "cashflows_seriatim_10K.xlsx"), "w") as f: f.write("") 
-        # ... and so on for other files, but they would be empty and cause errors in pd.read_excel.
-        # It's better to instruct the user to add the actual files.
         print(f"Expected files in '{EXAMPLE_DATA_DIR}': {list(EXAMPLE_FILES.values())}")
 
-
     demo_app = create_interface()
     demo_app.launch()