Upload report.py

report.py

@@ -23,7 +23,9 @@ def plot_heatmap(df, title):
 
 
 def plot_valid_data_percentage(original_df, cleaned_df, primary_key_column):
-    columns_to_plot = [col for col in original_df.columns if col != primary_key_column]
+    # Get columns present in both DataFrames, excluding the primary key
+    columns_to_plot = [col for col in original_df.columns if col in cleaned_df.columns and col != primary_key_column]
+
     original_valid = (original_df[columns_to_plot].notna().sum() / len(original_df)) * 100
     cleaned_valid = (cleaned_df[columns_to_plot].notna().sum() / len(cleaned_df)) * 100
 
@@ -52,6 +54,13 @@ def plot_valid_data_percentage(original_df, cleaned_df, primary_key_column):
     plt.savefig(os.path.join(REPORT_DIR, 'valid_data_percentage.png'))
     plt.close()
 
+    # Print information about removed columns
+    removed_columns = set(original_df.columns) - set(cleaned_df.columns)
+    if removed_columns:
+        print(f"The following columns were removed during the cleaning process: {', '.join(removed_columns)}")
+    else:
+        print("No columns were removed during the cleaning process.")
+
 
 def plot_column_schemas(df):
     schemas = df.dtypes.astype(str).value_counts()