create and import CorrelationFilter class

A5/CorrelationFilter.py

@@ -0,0 +1,21 @@
+from sklearn.base import BaseEstimator, TransformerMixin
+
+# Finds similar features that are highly correlated and remove it
+class CorrelationFilter(BaseEstimator, TransformerMixin):
+    def __init__(self, threshold=0.99):
+        self.threshold = threshold
+        self.keep_cols_ = None
+
+    def fit(self, X, y=None):
+        Xdf = pd.DataFrame(X) if not isinstance(X, pd.DataFrame) else X
+        # calculates the correlation matrix and takes absolutte values since negative values are also calculated
+        corr = Xdf.corr(numeric_only=True).abs()
+        upper = corr.where(np.triu(np.ones(corr.shape), k=1).astype(bool))
+        to_drop = [col for col in upper.columns if any(upper[col] >= self.threshold)]
+        self.keep_cols_ = [c for c in Xdf.columns if c not in to_drop]
+        return self
+
+    # Applies transformation ad return result as pd dataframe
+    def transform(self, X):
+        Xdf = pd.DataFrame(X) if not isinstance(X, pd.DataFrame) else X
+        return Xdf[self.keep_cols_].copy()

app.py

@@ -2,6 +2,8 @@ import gradio as gr
 import pandas as pd
 import pickle
 import os
+from A5.CorrelationFilter import CorrelationFilter
+
 
 # Get directory where this script is located
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -9,11 +11,11 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 # Local paths - models loaded from A4/models/ directory
 MODEL_PATH = os.path.join(
     SCRIPT_DIR,
-    "A4/models/gDriveVersion/champion_model_final_2.pkl"
+    "A5/models/aimoscores_improved_A4.pkl"
 )
 CLASSIFICATION_MODEL_PATH = os.path.join(
     SCRIPT_DIR,
-    "A4/models/gDriveVersion/final_champion_model_A3.pkl"
+    "A5/models/weaklink_classifier_rfc_A4.pkl"
 )
 DATA_PATH = os.path.join(
     SCRIPT_DIR,
@@ -86,10 +88,6 @@ def load_classification_model():
     return False
 
 
-load_champion_model()
-load_classification_model()
-
-
 def predict_score(*feature_values):
     if model is None:
         return "Error", "Model not loaded", ""
@@ -415,7 +413,11 @@ def create_interface():
     return demo
 
 
-demo = create_interface()
-
 if __name__ == "__main__":
+    # load the pickled models
+    load_champion_model()
+    load_classification_model()
+
+    # create the interface
+    demo = create_interface()
     demo.launch(share=False, server_name="0.0.0.0", server_port=7860)