refactor fetch_misclassified_dataframe to improve SQL queries and remove unused imports

prepare_pd_df.py

@@ -1,9 +1,7 @@
 # prepare_pd_dataframe.py
 
-import os
-import random
 import pandas as pd
-from sqlalchemy import create_engine, text
+from sqlalchemy import text
 
 
 def fetch_misclassified_dataframe(label_column: str, 
@@ -34,7 +32,6 @@ def fetch_misclassified_dataframe(label_column: str,
     
     # define conditions based on column
     miscond = f"mc.correct_{label_column} IS NOT NULL AND mc.model_predicted_{label_column} IS DISTINCT FROM mc.correct_{label_column}"
-    corrcond = f"mc.correct_{label_column} IS NOT NULL AND mc.model_predicted_{label_column} = mc.correct_{label_column}"
     
     # SQL to fetch misclassified records
     sql_mis = text(f"""
@@ -56,16 +53,17 @@ def fetch_misclassified_dataframe(label_column: str,
     n_mis = len(df_mis)
     n_correct = int(n_mis * correct_ratio)
     
-    # SQL to fetch correct records
+    # SQL to fetch correct records from complaints table NOT in misclassified_complaints
     sql_corr = text(f"""
-        SELECT c.message AS grievance,
-               mc.correct_department AS department,
-               mc.correct_urgency AS urgency
-        FROM misclassified_complaints mc
-        JOIN complaints c ON c.id = mc.complaint_id
-        WHERE mc.reviewed = TRUE
-          AND {corrcond}
+        SELECT c.id AS complaint_id,
+               c.message AS grievance,
+               c.department AS department,
+               c.urgency AS urgency
+        FROM complaints c
+        WHERE c.id NOT IN (SELECT complaint_id FROM misclassified_complaints)
+          AND c.{label_column} IS NOT NULL
     """)
+    
     with engine.connect() as conn:
         df_corr_all = pd.read_sql(sql_corr, conn)
     
@@ -83,12 +81,13 @@ def fetch_misclassified_dataframe(label_column: str,
     
     return df_combined
 
-# If this file is run directly, simple test:
-if __name__ == "__main__":
-    # Quick sanity test for department label
-    df_test = fetch_misclassified_dataframe(label_column="department", correct_ratio=0.5)
-    print("Rows fetched:", len(df_test))
-    print(df_test.head())
-    # Basic assertion: if rows>0 then none of grievances should be null
-    if len(df_test) > 0:
-        assert df_test['grievance'].isna().sum() == 0, "Some grievances are null"
+# # If this file is run directly, simple test:
+# if __name__ == "__main__":
+#     # Quick sanity test for department label
+#     df_test = fetch_misclassified_dataframe(label_column="department", 
+#                                             correct_ratio=0.5)
+#     print("Rows fetched:", len(df_test))
+#     print(df_test.head())
+#     # Basic assertion: if rows>0 then none of grievances should be null
+#     if len(df_test) > 0:
+#         assert df_test['grievance'].isna().sum() == 0, "Some grievances are null"