update

app.py

@@ -33,21 +33,14 @@ NUMERICAL_COLUMNS = [
 
 FEATURE_COLUMNS = CATEGORICAL_COLUMNS + NUMERICAL_COLUMNS
 
-import pandas as pd
-from sklearn.preprocessing import LabelEncoder, StandardScaler
-
-import pandas as pd
-import numpy as np
-from sklearn.preprocessing import LabelEncoder, StandardScaler
-
-
-def preprocess_input(input_df):
+def preprocess_input(input_df, train_df=None, model_type="catboost"):
     """
-    Preprocess input data before passing it to CatBoost in the Streamlit app.
+    Preprocess input data before passing it to ML models.
     - Removes DateTime columns
     - Computes aggregations
-    - Ensures categorical variables are kept as strings
+    - Ensures categorical variables are properly encoded
     - Normalizes numerical features
+    - Selects only required features for the given model
     """
 
     # 🚀 Step 1: Drop DateTime Columns
@@ -62,55 +55,25 @@ def preprocess_input(input_df):
     # 🚀 Step 2: Fill missing values before aggregations
     input_df.fillna(0, inplace=True)
 
-    # 🚀 Step 3: Compute Aggregations
-    def compute_aggregations(df, group_cols, agg_dict, suffix):
-        if not all(col in df.columns for col in group_cols):
-            print(f"🛑 Missing grouping columns for aggregation: {group_cols}")
-            return df  # Skip if group columns are missing
-
-        agg_df = df.groupby(group_cols).agg(agg_dict).reset_index()
-        if agg_df.empty:
-            print(f"🛑 Aggregation resulted in an empty DataFrame for {group_cols}")
-            return df
-
-        agg_df.columns = group_cols + [f"{col}_{suffix}" for col in agg_df.columns[len(group_cols):]]
-        return df.merge(agg_df, on=group_cols, how="left")
-
-    # Check if required columns are present before aggregating
-    required_columns = ["age_level", "gender", "product", "city_development_index"]
-    if not all(col in input_df.columns for col in required_columns):
-        print("🛑 Missing required columns for aggregations. Skipping aggregation steps.")
-    else:
-        # Aggregation: Age & Gender vs Product
-        input_df = compute_aggregations(input_df, ["age_level", "gender", "product"], {
-            "campaign_id": "nunique",
-            "webpage_id": "nunique"
-        }, suffix="age_sex_prod")
-
-        # Aggregation: City, Age, Product
-        input_df = compute_aggregations(input_df, ["city_development_index", "age_level", "product"], {
-            "campaign_id": "nunique",
-            "webpage_id": "nunique"
-        }, suffix="city_age_prod")
-
-    # 🚀 Step 4: Add Missing Aggregated Columns with Default Values
-    aggregated_features = [
-        "click_sum_age_sex_prod", "click_count_age_sex_prod", "unique_campaigns_age_sex_prod",
-        "unique_webpages_age_sex_prod",
-        "click_sum_city_age_prod", "click_count_city_age_prod", "unique_campaigns_city_age_prod",
-        "unique_webpages_city_age_prod"
-    ]
-
-    for col in aggregated_features:
-        if col not in input_df.columns:
-            input_df[col] = 0  # Fill missing aggregated columns with default values
+    # 🚀 Step 3: Compute Aggregations (Requires a reference training dataset)
+    if train_df is not None:
+        input_df = apply_aggregations(input_df, train_df)
 
-    # 🚀 Step 5: **Ensure Categorical Features Stay as Strings**
+    # 🚀 Step 4: Ensure Categorical Features Stay as Strings
     categorical_columns = ["gender", "product", "campaign_id", "webpage_id"]
     for col in categorical_columns:
-        input_df[col] = input_df[col].astype(str).fillna("missing")  # **Convert to string**
+        input_df[col] = input_df[col].astype(str).fillna("missing")
+
+    # 🚀 Step 5: Ensure Consistent Label Encoding
+    label_encoders = {}
+    for col in categorical_columns:
+        le = LabelEncoder()
+        input_df[col] = input_df[col].astype(str)
+        le.fit(input_df[col].unique())
+        label_encoders[col] = le
+        input_df[col] = input_df[col].map(lambda x: le.transform([x])[0] if x in le.classes_ else -1)
 
-    # 🚀 Step 6: **Normalize Only Numerical Features** (DO NOT Normalize Categorical Columns)
+    # 🚀 Step 6: Normalize Numerical Features
     numerical_columns = [
         "age_level", "city_development_index", "user_group_id", "user_depth", "var_1",
         "click_sum_age_sex_prod", "click_count_age_sex_prod",
@@ -119,13 +82,79 @@ def preprocess_input(input_df):
         "unique_campaigns_city_age_prod", "unique_webpages_city_age_prod"
     ]
 
-    # Ensure all numerical columns exist
+    # Check if all numerical columns exist
     numerical_columns = [col for col in numerical_columns if col in input_df.columns]
-
-    # Normalize numerical features
     scaler = StandardScaler()
     input_df[numerical_columns] = scaler.fit_transform(input_df[numerical_columns])
 
+    # 🚀 Step 7: Select Features Based on Model Type
+    model_features = {
+        "catboost": ["age_level", "gender", "product", "campaign_id", "webpage_id"] + numerical_columns,
+        "xgboost": ["age_level", "gender", "product", "campaign_id", "webpage_id"] + numerical_columns,
+        "random_forest": [
+            "age_level", "gender", "product", "campaign_id", "webpage_id",
+            "product_category_1", "product_category_2", "user_group_id",
+            "user_depth", "city_development_index", "var_1"
+        ] + numerical_columns
+    }
+
+    selected_features = model_features.get(model_type, input_df.columns)
+
+    # 🚀 Ensure only required features are passed to the model
+    input_df = input_df[selected_features]
+
+    return input_df
+
+
+def apply_aggregations(input_df, train_df):
+    """
+    Applies necessary aggregations to the input DataFrame using reference training data.
+    """
+
+    # 🚀 Step 1: Aggregate by age & gender vs product
+    age_sex_product_agg = train_df.groupby(["age_level", "gender", "product"]).agg({
+        "is_click": ["sum", "count"],
+        "campaign_id": "nunique",
+        "webpage_id": "nunique"
+    }).reset_index()
+
+    # Rename columns after aggregation
+    age_sex_product_agg.columns = ["age_level", "gender", "product",
+                                   "click_sum_age_sex_prod", "click_count_age_sex_prod",
+                                   "unique_campaigns_age_sex_prod", "unique_webpages_age_sex_prod"]
+
+    # Merge back into input data
+    input_df = input_df.merge(age_sex_product_agg, on=["age_level", "gender", "product"], how="left")
+
+    # 🚀 Step 2: Aggregate by city, age, product
+    city_age_product_agg = train_df.groupby(["city_development_index", "age_level", "product"]).agg({
+        "is_click": ["sum", "count"],
+        "campaign_id": "nunique",
+        "webpage_id": "nunique"
+    }).reset_index()
+
+    # Rename columns after aggregation
+    city_age_product_agg.columns = ["city_development_index", "age_level", "product",
+                                    "click_sum_city_age_prod", "click_count_city_age_prod",
+                                    "unique_campaigns_city_age_prod", "unique_webpages_city_age_prod"]
+
+    # Merge into input data
+    input_df = input_df.merge(city_age_product_agg, on=["city_development_index", "age_level", "product"], how="left")
+
+    # 🚀 Step 3: Fill NaN values introduced by merging
+    aggregated_features = [
+        "click_sum_age_sex_prod", "click_count_age_sex_prod", "unique_campaigns_age_sex_prod",
+        "unique_webpages_age_sex_prod",
+        "click_sum_city_age_prod", "click_count_city_age_prod", "unique_campaigns_city_age_prod",
+        "unique_webpages_city_age_prod"
+    ]
+
+    for col in aggregated_features:
+        if col not in input_df.columns:
+            input_df[col] = 0  # Fill missing aggregated columns with default values
+        else:
+            input_df[col].fillna(0, inplace=True)
+
     return input_df