{ "cells": [ { "cell_type": "markdown", "id": "c0cdf0b4", "metadata": {}, "source": [ "# Import Library & Configuration" ] }, { "cell_type": "code", "execution_count": 25, "id": "d37034b9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Libraries loaded\n", "Time: 2026-02-14 11:47:50\n" ] } ], "source": [ "import os\n", "os.environ['PYTHONHASHSEED'] = '0'\n", "os.environ['TF_DETERMINISTIC_OPS'] = '1'\n", "os.environ['TF_CUDNN_DETERMINISM'] = '1'\n", "\n", "import random\n", "random.seed(42)\n", "\n", "import math\n", "import pandas as pd\n", "import numpy as np\n", "np.random.seed(42)\n", "\n", "from datetime import datetime\n", "import pickle\n", "import warnings\n", "warnings.filterwarnings('ignore')\n", "\n", "\"\"\"\n", "Deep Learning Framework\n", "TensorFlow & Keras: Autoencoder architecture with Dense, BatchNormalization, Dropout layers\n", "\"\"\"\n", "import tensorflow as tf\n", "tf.random.set_seed(42)\n", "from tensorflow.keras import layers, Model, optimizers, callbacks\n", "from tensorflow.keras.layers import Input, Dense, Dropout, BatchNormalization\n", "\n", "\"\"\"\n", "Machine Learning & Metrics\n", "- KMeans: K-means clustering for shoe recommendation groups\n", "- StandardScaler/MinMaxScaler: Feature normalization for ML models\n", "- Clustering Metrics: Silhouette, Davies-Bouldin, Calinski-Harabasz indices\n", "- Similarity: Cosine similarity for recommendation ranking\n", "\"\"\"\n", "from sklearn.cluster import KMeans\n", "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n", "from sklearn.metrics import (\n", " silhouette_score, davies_bouldin_score, calinski_harabasz_score,\n", " adjusted_rand_score\n", ")\n", "from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances\n", "from sklearn.ensemble import RandomForestClassifier\n", "\n", "\"\"\"\n", "Data Visualization\n", "Matplotlib & Seaborn for statistical plots and cluster visualization\n", "\"\"\"\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "sns.set_style('whitegrid')\n", "plt.rcParams['figure.figsize'] = (14, 6)\n", "\n", "np.random.seed(42)\n", "tf.random.set_seed(42)\n", "\n", "print('Libraries loaded')\n", "print(f'Time: {datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')" ] }, { "cell_type": "markdown", "id": "4b97aa73", "metadata": {}, "source": [ "# Load Data" ] }, { "cell_type": "code", "execution_count": 26, "id": "ea0a9377", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loaded: 428 shoes × 32 columns\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
brandnamelightweightrockerremovable_insolepace_daily_runningpace_tempopace_competitionarch_neutralarch_stability...stiffness_scaledtorsional_rigidityheel_stiffplate_rock_plateplate_carbon_plateheel_lab_mmforefoot_lab_mmseason_summerseason_winterseason_all
0brookslaunch 910111010...5510032.423.0000
1brookslevitate 600110010...5330034.326.6101
2adidas4dfwd00110010...5110033.324.4001
3adidas4dfwd 200110010...5130031.821.2001
4adidas4dfwd 300110010...3110032.622.7001
\n", "

5 rows × 32 columns

\n", "
" ], "text/plain": [ " brand name lightweight rocker removable_insole \\\n", "0 brooks launch 9 1 0 1 \n", "1 brooks levitate 6 0 0 1 \n", "2 adidas 4dfwd 0 0 1 \n", "3 adidas 4dfwd 2 0 0 1 \n", "4 adidas 4dfwd 3 0 0 1 \n", "\n", " pace_daily_running pace_tempo pace_competition arch_neutral \\\n", "0 1 1 0 1 \n", "1 1 0 0 1 \n", "2 1 0 0 1 \n", "3 1 0 0 1 \n", "4 1 0 0 1 \n", "\n", " arch_stability ... stiffness_scaled torsional_rigidity heel_stiff \\\n", "0 0 ... 5 5 1 \n", "1 0 ... 5 3 3 \n", "2 0 ... 5 1 1 \n", "3 0 ... 5 1 3 \n", "4 0 ... 3 1 1 \n", "\n", " plate_rock_plate plate_carbon_plate heel_lab_mm forefoot_lab_mm \\\n", "0 0 0 32.4 23.0 \n", "1 0 0 34.3 26.6 \n", "2 0 0 33.3 24.4 \n", "3 0 0 31.8 21.2 \n", "4 0 0 32.6 22.7 \n", "\n", " season_summer season_winter season_all \n", "0 0 0 0 \n", "1 1 0 1 \n", "2 0 0 1 \n", "3 0 0 1 \n", "4 0 0 1 \n", "\n", "[5 rows x 32 columns]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "file = '../../data/road_dataset.csv'\n", "\n", "try:\n", " df = pd.read_csv(file)\n", " print(f'Loaded: {df.shape[0]} shoes × {df.shape[1]} columns')\n", " display(df.head())\n", "except FileNotFoundError:\n", " print(f\"WARNING: '{file}' not found.\")\n", " print(\"Please upload the correct dataset file to run with actual data.\")" ] }, { "cell_type": "code", "execution_count": 27, "id": "ff806013", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 428 entries, 0 to 427\n", "Data columns (total 32 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 brand 428 non-null str \n", " 1 name 428 non-null str \n", " 2 lightweight 428 non-null int64 \n", " 3 rocker 428 non-null int64 \n", " 4 removable_insole 428 non-null int64 \n", " 5 pace_daily_running 428 non-null int64 \n", " 6 pace_tempo 428 non-null int64 \n", " 7 pace_competition 428 non-null int64 \n", " 8 arch_neutral 428 non-null int64 \n", " 9 arch_stability 428 non-null int64 \n", " 10 weight_lab_oz 428 non-null float64\n", " 11 drop_lab_mm 428 non-null float64\n", " 12 strike_heel 428 non-null int64 \n", " 13 strike_mid 428 non-null int64 \n", " 14 strike_forefoot 428 non-null int64 \n", " 15 midsole_softness 428 non-null int64 \n", " 16 toebox_durability 428 non-null int64 \n", " 17 heel_durability 428 non-null int64 \n", " 18 outsole_durability 428 non-null int64 \n", " 19 breathability_scaled 428 non-null int64 \n", " 20 width_fit 428 non-null int64 \n", " 21 toebox_width 428 non-null int64 \n", " 22 stiffness_scaled 428 non-null int64 \n", " 23 torsional_rigidity 428 non-null int64 \n", " 24 heel_stiff 428 non-null int64 \n", " 25 plate_rock_plate 428 non-null int64 \n", " 26 plate_carbon_plate 428 non-null int64 \n", " 27 heel_lab_mm 428 non-null float64\n", " 28 forefoot_lab_mm 428 non-null float64\n", " 29 season_summer 428 non-null int64 \n", " 30 season_winter 428 non-null int64 \n", " 31 season_all 428 non-null int64 \n", "dtypes: float64(4), int64(26), str(2)\n", "memory usage: 107.1 KB\n" ] } ], "source": [ "df.info()" ] }, { "cell_type": "markdown", "id": "74a2caea", "metadata": {}, "source": [ "# Preprocessing" ] }, { "cell_type": "markdown", "id": "d94fbce6", "metadata": {}, "source": [ "## Feature Engineering\n", "Separates numeric features into two categories for different preprocessing strategies." ] }, { "cell_type": "code", "execution_count": 28, "id": "9f7f5da2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Features: 30 total\n", " Binary : 16\n", " Continuous : 14\n" ] } ], "source": [ "numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()\n", "\n", "binary_cols = [col for col in numeric_cols if set(df[col].unique()).issubset({0, 1})]\n", "continuous_cols = [col for col in numeric_cols if col not in binary_cols]\n", "\n", "print(f'Features: {len(numeric_cols)} total')\n", "print(f' Binary : {len(binary_cols)}')\n", "print(f' Continuous : {len(continuous_cols)}')" ] }, { "cell_type": "markdown", "id": "5fee0af0", "metadata": {}, "source": [ "## Normalization\n", "- Binary features: kept as-is (0-1 range)\n", "- Continuous features: MinMaxScaler to [0, 1]\n", "- Combined array: binary + continuous scaled features\n", "This ensures neural network compatibility and distance metric compatibility." ] }, { "cell_type": "code", "execution_count": 29, "id": "c04a0228", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Neural input shape: (428, 30)\n", "Range: [0.000000, 1.000000]\n" ] } ], "source": [ "feature_cols = numeric_cols.copy()\n", "X = df[feature_cols]\n", "\n", "X_binary = X[binary_cols].values\n", "X_continuous = X[continuous_cols].values\n", "\n", "scaler_continuous = MinMaxScaler()\n", "X_continuous_scaled = scaler_continuous.fit_transform(X_continuous)\n", "\n", "X_combined = np.concatenate([X_binary, X_continuous_scaled], axis=1)\n", "\n", "scaler_standard = StandardScaler()\n", "X_standard = scaler_standard.fit_transform(X)\n", "\n", "print(f'Neural input shape: {X_combined.shape}')\n", "print(f'Range: [{X_combined.min():.6f}, {X_combined.max():.6f}]')" ] }, { "cell_type": "markdown", "id": "05980d55", "metadata": {}, "source": [ "# Auto-Encoder" ] }, { "cell_type": "markdown", "id": "9be84505", "metadata": {}, "source": [ "## Modelling\n", "- Purpose: Dimensionality reduction (high-D features → 8D latent space)\n", "- Architecture: Encoder [input → 32 → 16 → 8] + Decoder [8 → 16 → 32 → reconstructed]\n", "- Regularization: BatchNormalization + Dropout(0.2) at each dense layer\n", "- Loss: MSE (reconstruction error) | Optimizer: Adam(lr=0.001)" ] }, { "cell_type": "code", "execution_count": 30, "id": "8e286cb1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Autoencoder architecture:\n" ] }, { "data": { "text/html": [ "
Model: \"functional_2\"\n",
       "
\n" ], "text/plain": [ "\u001b[1mModel: \"functional_2\"\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃ Layer (type)                    ┃ Output Shape           ┃       Param # ┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ input_layer_1 (InputLayer)      │ (None, 30)             │             0 │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_6 (Dense)                 │ (None, 32)             │           992 │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ batch_normalization_5           │ (None, 32)             │           128 │\n",
       "│ (BatchNormalization)            │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_5 (Dropout)             │ (None, 32)             │             0 │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_7 (Dense)                 │ (None, 16)             │           528 │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ batch_normalization_6           │ (None, 16)             │            64 │\n",
       "│ (BatchNormalization)            │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_6 (Dropout)             │ (None, 16)             │             0 │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_8 (Dense)                 │ (None, 8)              │           136 │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ batch_normalization_7           │ (None, 8)              │            32 │\n",
       "│ (BatchNormalization)            │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_7 (Dropout)             │ (None, 8)              │             0 │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_9 (Dense)                 │ (None, 16)             │           144 │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ batch_normalization_8           │ (None, 16)             │            64 │\n",
       "│ (BatchNormalization)            │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_8 (Dropout)             │ (None, 16)             │             0 │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_10 (Dense)                │ (None, 32)             │           544 │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ batch_normalization_9           │ (None, 32)             │           128 │\n",
       "│ (BatchNormalization)            │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_9 (Dropout)             │ (None, 32)             │             0 │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_11 (Dense)                │ (None, 30)             │           990 │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
       "
\n" ], "text/plain": [ "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n", "┃\u001b[1m \u001b[0m\u001b[1mLayer (type) \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m Param #\u001b[0m\u001b[1m \u001b[0m┃\n", "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n", "│ input_layer_1 (\u001b[38;5;33mInputLayer\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m30\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ dense_6 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m) │ \u001b[38;5;34m992\u001b[0m │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ batch_normalization_5 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m) │ \u001b[38;5;34m128\u001b[0m │\n", "│ (\u001b[38;5;33mBatchNormalization\u001b[0m) │ │ │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ dropout_5 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ dense_7 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m) │ \u001b[38;5;34m528\u001b[0m │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ batch_normalization_6 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m) │ \u001b[38;5;34m64\u001b[0m │\n", "│ (\u001b[38;5;33mBatchNormalization\u001b[0m) │ │ │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ dropout_6 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ dense_8 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m8\u001b[0m) │ \u001b[38;5;34m136\u001b[0m │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ batch_normalization_7 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m8\u001b[0m) │ \u001b[38;5;34m32\u001b[0m │\n", "│ (\u001b[38;5;33mBatchNormalization\u001b[0m) │ │ │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ dropout_7 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m8\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ dense_9 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m) │ \u001b[38;5;34m144\u001b[0m │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ batch_normalization_8 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m) │ \u001b[38;5;34m64\u001b[0m │\n", "│ (\u001b[38;5;33mBatchNormalization\u001b[0m) │ │ │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ dropout_8 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ dense_10 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m) │ \u001b[38;5;34m544\u001b[0m │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ batch_normalization_9 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m) │ \u001b[38;5;34m128\u001b[0m │\n", "│ (\u001b[38;5;33mBatchNormalization\u001b[0m) │ │ │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ dropout_9 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ dense_11 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m30\u001b[0m) │ \u001b[38;5;34m990\u001b[0m │\n", "└─────────────────────────────────┴────────────────────────┴───────────────┘\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
 Total params: 3,750 (14.65 KB)\n",
       "
\n" ], "text/plain": [ "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m3,750\u001b[0m (14.65 KB)\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
 Trainable params: 3,542 (13.84 KB)\n",
       "
\n" ], "text/plain": [ "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m3,542\u001b[0m (13.84 KB)\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
 Non-trainable params: 208 (832.00 B)\n",
       "
\n" ], "text/plain": [ "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m208\u001b[0m (832.00 B)\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "input_dim = X_combined.shape[1]\n", "encoding_dims = [32, 16, 8]\n", "\n", "input_layer = Input(shape=(input_dim,))\n", "x = input_layer\n", "for dim in encoding_dims:\n", " x = Dense(dim, activation='relu')(x)\n", " x = BatchNormalization()(x)\n", " x = Dropout(0.3)(x)\n", "\n", "latent = x\n", "\n", "for dim in reversed(encoding_dims[:-1]):\n", " x = Dense(dim, activation='relu')(x)\n", " x = BatchNormalization()(x)\n", " x = Dropout(0.3)(x)\n", "\n", "output_layer = Dense(input_dim, activation='sigmoid')(x)\n", "\n", "autoencoder = Model(input_layer, output_layer)\n", "encoder = Model(input_layer, latent)\n", "\n", "autoencoder.compile(\n", " optimizer=optimizers.Adam(0.001),\n", " loss='mse',\n", " metrics=['mae']\n", ")\n", "\n", "print('Autoencoder architecture:')\n", "autoencoder.summary()" ] }, { "cell_type": "markdown", "id": "c3bb013f", "metadata": {}, "source": [ "## Training\n", "Training Configuration\n", "- Epochs: 300 (with early stopping)\n", "- Batch size: 64\n", "- Validation split: 20%\n", "- Early stopping: patience=20 (stop if val_loss doesn't improve)\n", "- LR reduction: factor=0.5, patience=10, min_lr=1e-5\n", "- Output: X_latent (8D embeddings) for KMeans clustering" ] }, { "cell_type": "code", "execution_count": 31, "id": "3f610379", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training done!\n", "Final loss: 0.098345\n", " Val loss: 0.074054\n", "Latent space: (428, 8) (8D embeddings)\n" ] } ], "source": [ "history = autoencoder.fit(\n", " X_combined, X_combined,\n", " epochs=300,\n", " batch_size=64,\n", " validation_split=0.25,\n", " shuffle=False,\n", " callbacks=[\n", " callbacks.EarlyStopping(monitor='val_loss', patience=20, restore_best_weights=True),\n", " callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=10, min_lr=1e-5)\n", " ],\n", " verbose=0\n", ")\n", "\n", "print(f'Training done!')\n", "print(f'Final loss: {history.history[\"loss\"][-1]:.6f}')\n", "print(f' Val loss: {history.history[\"val_loss\"][-1]:.6f}')\n", "\n", "X_latent = encoder.predict(X_combined, verbose=0)\n", "print(f'Latent space: {X_latent.shape} (8D embeddings)')" ] }, { "cell_type": "markdown", "id": "6d811ee5", "metadata": {}, "source": [ "# Metrics Function" ] }, { "cell_type": "markdown", "id": "fcb64663", "metadata": {}, "source": [ "## Interpretability Score" ] }, { "cell_type": "code", "execution_count": 32, "id": "a6f5f27d", "metadata": {}, "outputs": [], "source": [ "def calculate_interpretability_score(df, cluster_col, binary_cols, top_n=5, threshold=0.70):\n", " \"\"\"\n", " Calculate cluster interpretability based on top feature clarity.\n", " \n", " Strategy: Focus on strongest features (not averages) to reduce noise.\n", " Strength Definition: Distance from neutral point (0.5)\n", " - Features closer to 0 or 1 = strong patterns\n", " - Features closer to 0.5 = ambiguous patterns\n", " \n", " Args:\n", " df (pd.DataFrame): Cluster-labeled dataset\n", " cluster_col (str): Column with cluster assignments\n", " binary_cols (list): Binary feature names\n", " top_n (int): Number of top features to evaluate (default: 5)\n", " threshold (float): Unused parameter (API compatibility)\n", " \n", " Returns:\n", " dict: {'mean_interpretability': float [0, 1]}\n", " 1.0 = clear feature patterns, 0.0 = no patterns\n", " \"\"\"\n", " scores = []\n", " unique_clusters = df[cluster_col].unique()\n", " \n", " for cid in unique_clusters:\n", " cdata = df[df[cluster_col] == cid]\n", " n = len(cdata)\n", " if n == 0: \n", " scores.append(0)\n", " continue\n", " \n", " feature_strength = []\n", " for col in binary_cols:\n", " if col in cdata.columns:\n", " avg = cdata[col].mean()\n", " strength = abs(avg - 0.5) * 2 # Normalize distance from neutral (0.5) to [0, 1]\n", " feature_strength.append(strength)\n", " \n", " if feature_strength:\n", " feature_strength.sort(reverse=True)\n", " top_features = feature_strength[:top_n]\n", " scores.append(np.mean(top_features)) # Average of top-N features\n", " else:\n", " scores.append(0)\n", " \n", " return {'mean_interpretability': np.mean(scores) if scores else 0}" ] }, { "cell_type": "markdown", "id": "e5489774", "metadata": {}, "source": [ "## Cluster Purity" ] }, { "cell_type": "code", "execution_count": 33, "id": "ac03f346", "metadata": {}, "outputs": [], "source": [ "def calculate_cluster_purity(df, cluster_col, binary_cols):\n", " \"\"\"\n", " Measure internal cluster homogeneity via majority class dominance.\n", " \n", " Purity Calculation: For each feature, compute max(class0_pct, class1_pct)\n", " Average across all features = cluster purity\n", " Range: [0.5, 1.0] where 1.0 = perfect homogeneity\n", " \n", " Args:\n", " df (pd.DataFrame): Cluster-labeled dataset\n", " cluster_col (str): Column with cluster assignments\n", " binary_cols (list): Binary feature names\n", " \n", " Returns:\n", " dict: {'mean_purity': float [0.5, 1.0]}\n", " \"\"\"\n", " purity_by_cluster = []\n", " unique_clusters = df[cluster_col].unique()\n", " \n", " for cid in unique_clusters:\n", " cdata = df[df[cluster_col] == cid]\n", " n = len(cdata)\n", " if n == 0: continue\n", " \n", " dominances = []\n", " for col in binary_cols:\n", " if col in cdata.columns:\n", " avg = cdata[col].mean()\n", " dominances.append(max(avg, 1 - avg)) # Majority class percentage\n", " \n", " if dominances:\n", " purity_by_cluster.append(np.mean(dominances))\n", " \n", " return {'mean_purity': np.mean(purity_by_cluster) if purity_by_cluster else 0}" ] }, { "cell_type": "markdown", "id": "c82f8d93", "metadata": {}, "source": [ "## Cluster Stability" ] }, { "cell_type": "code", "execution_count": 34, "id": "f4b03254", "metadata": {}, "outputs": [], "source": [ "def calculate_cluster_stability(X, labels, model_func, n_iter=5, seed=42):\n", " \"\"\"\n", " Bootstrap stability testing via Adjusted Rand Index (ARI).\n", " \n", " Process:\n", " 1. Train model on bootstrap sample (with replacement)\n", " 2. Compare original vs bootstrap clustering using ARI\n", " 3. Average ARI across iterations\n", " \n", " ARI Range: [-1, 1]\n", " > 0.5: excellent stability\n", " 0.2-0.5: fair stability\n", " < 0.2: poor stability\n", " \n", " Args:\n", " X (np.ndarray): Feature matrix\n", " labels (np.ndarray): Original cluster assignments\n", " model_func (callable): Returns instantiated clustering model\n", " n_iter (int): Bootstrap iterations (default: 5)\n", " \n", " Returns:\n", " dict: {'mean_ari': float [-1, 1]}\n", " \"\"\"\n", " if len(np.unique(labels)) < 2:\n", " return {'mean_ari': 0}\n", "\n", " n = len(X)\n", " ari_scores = []\n", "\n", " for i in range(n_iter):\n", " rng = np.random.default_rng(seed=42 + i)\n", " idx = rng.choice(n, n, replace=True)\n", " try:\n", " boot_model = model_func()\n", " boot_labels = boot_model.fit_predict(X[idx])\n", " ari = adjusted_rand_score(labels[idx], boot_labels)\n", " ari_scores.append(ari)\n", " except Exception:\n", " continue\n", "\n", " m = np.mean(ari_scores) if ari_scores else 0\n", " return {'mean_ari': m}" ] }, { "cell_type": "markdown", "id": "346b09d4", "metadata": {}, "source": [ "## Comprehensive Evaluation" ] }, { "cell_type": "code", "execution_count": 35, "id": "514690f8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Metrics Function Ready.\n" ] } ], "source": [ "def evaluate_clustering_comprehensive(X, labels, df_original, model_func, binary_cols):\n", " \"\"\"\n", " Multi-metric clustering evaluation combining geometric and business metrics.\n", " \n", " Evaluation Framework:\n", " \n", " 1. GEOMETRIC METRICS (Scikit-learn):\n", " Silhouette [-1, 1]: cluster separation quality\n", " Davies-Bouldin [0, ∞): intra-cluster density (lower better)\n", " Calinski-Harabasz [0, ∞): cluster definition (higher better)\n", " \n", " 2. BUSINESS METRICS:\n", " Purity: internal homogeneity\n", " Interpretability: feature pattern clarity\n", " Stability: clustering consistency\n", " \n", " 3. COMPOSITE SCORING (strategic weights):\n", " Structure (40%): 40% Silhouette + 30% Davies-Bouldin + 30% Calinski-Harabasz\n", " Explainability (30%): 50% Interpretability + 50% Purity\n", " Reliability (30%): Bootstrap ARI stability\n", " \n", " Args:\n", " X (np.ndarray): Latent feature space (typically autoencoder output)\n", " labels (np.ndarray): Cluster assignments [0, K-1]\n", " df_original (pd.DataFrame): Original shoe metadata\n", " model_func (callable): KMeans factory function\n", " binary_cols (list): Binary feature column names\n", " \n", " Returns:\n", " dict: {\n", " 'metrics': {silhouette, davies_bouldin, calinski_harabasz, purity, stability, interpretability},\n", " 'composite_score': float [0, 1]\n", " }\n", " \"\"\"\n", " df_eval = df_original.copy()\n", " df_eval['cluster'] = labels\n", " \n", " sil = silhouette_score(X, labels)\n", " db = davies_bouldin_score(X, labels)\n", " ch = calinski_harabasz_score(X, labels)\n", " \n", " purity_res = calculate_cluster_purity(df_eval, 'cluster', binary_cols)\n", " interp_res = calculate_interpretability_score(df_eval, 'cluster', binary_cols, top_n=5)\n", " stability_res = calculate_cluster_stability(X, labels, model_func, n_iter=3)\n", " \n", " val_purity = purity_res['mean_purity']\n", " val_interp = interp_res['mean_interpretability']\n", " val_stability = stability_res['mean_ari']\n", "\n", " sil_norm = (sil + 1) / 2 # Map Silhouette [-1, 1] → [0, 1]\n", " db_norm = np.exp(-0.5 * db) # Exponential decay: DB lower is better\n", " \n", " if ch > 0:\n", " ch_log = np.log1p(ch)\n", " ch_norm = min(ch_log / 8, 1.0) # Log scaling: assume max log(CH) ≈ 9.2\n", " else:\n", " ch_norm = 0\n", "\n", " score_structure = (0.4 * sil_norm) + (0.3 * db_norm) + (0.3 * ch_norm) # Weight: Silhouette 40%, DB 30%, CH 30%\n", " score_explain = (0.5 * val_interp) + (0.5 * val_purity) # Weight: Interpretability 50%, Purity 50%\n", " score_reliability = max(val_stability, 0) # Clip negative ARI to 0\n", "\n", " composite = (0.40 * score_structure) + (0.30 * score_explain) + (0.30 * score_reliability)\n", "\n", " return {\n", " 'metrics': {\n", " 'silhouette': sil, \n", " 'davies_bouldin': db, \n", " 'calinski_harabasz': ch,\n", " 'purity': val_purity, \n", " 'stability': val_stability,\n", " 'interpretability': val_interp\n", " },\n", " 'composite_score': composite\n", " }\n", "\n", "print('Metrics Function Ready.')" ] }, { "cell_type": "markdown", "id": "e8373ffc", "metadata": {}, "source": [ "# Model Selection\n", "Model Selection Pipeline: K-means Clustering (K=3 to K=9)\n", "\n", "For each K value:\n", " - Train KMeans model\n", " - Evaluate using comprehensive metrics\n", " - Compute composite score\n", "\n", "Select K with highest composite score (40% geometry, 30% explainability, 30% reliability)" ] }, { "cell_type": "code", "execution_count": 36, "id": "d78a6cd5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "| K | Score | Sil. | DB | CH | Purity | Stab. | Interp |\n", "|-----+----------+----------+----------+------------+----------+----------+----------|\n", "| 3 | 0.725575 | 0.442093 | 1.004044 | 223.989851 | 0.849337 | 0.628044 | 0.936774 |\n", "| 4 | 0.860043 | 0.530613 | 0.835884 | 336.071705 | 0.863760 | 0.982536 | 0.979710 |\n", "| 5 | 0.877370 | 0.555704 | 0.694095 | 438.120406 | 0.866944 | 0.997151 | 0.984283 |\n", "| 6 | 0.849927 | 0.485995 | 0.751707 | 440.021792 | 0.868007 | 0.930737 | 0.985898 |\n", "| 7 | 0.818026 | 0.469977 | 0.834006 | 411.950115 | 0.858330 | 0.847469 | 0.986705 |\n", "| 8 | 0.783303 | 0.468941 | 0.889430 | 396.703919 | 0.870448 | 0.734619 | 0.987520 |\n", "| 9 | 0.789632 | 0.466899 | 0.874739 | 399.636980 | 0.874462 | 0.749608 | 0.992294 |\n" ] } ], "source": [ "results = []\n", "\n", "print(f\"| {'K':^3} | {'Score':^8} | {'Sil.':^8} | {'DB':^8} | {'CH':^10} | {'Purity':^8} | {'Stab.':^8} | {'Interp':^8} |\")\n", "print(f\"|{'-'*5}+{'-'*10}+{'-'*10}+{'-'*10}+{'-'*12}+{'-'*10}+{'-'*10}+{'-'*10}|\")\n", "\n", "for i in range(3, 10):\n", " model_factory = lambda: KMeans(n_clusters=i, random_state=42, n_init=20)\n", " \n", " model = model_factory()\n", " labels = model.fit_predict(X_latent)\n", "\n", " metrics_res = evaluate_clustering_comprehensive(\n", " X_latent, labels, df.copy(),\n", " model_factory,\n", " binary_cols\n", " )\n", "\n", " raw_metrics = metrics_res['metrics'] \n", " comp_score = metrics_res['composite_score']\n", "\n", " record = {\n", " 'k': i,\n", " 'model': model,\n", " 'labels': labels,\n", " 'composite_score': comp_score,\n", " **raw_metrics\n", " }\n", " results.append(record)\n", "\n", " print(f\"| {i:^3} | {comp_score:<8.6f} | {raw_metrics['silhouette']:<6.6f} | \"\n", " f\"{raw_metrics['davies_bouldin']:<6.6f} | {raw_metrics['calinski_harabasz']:<8.6f} | \"\n", " f\"{raw_metrics['purity']:<6.6f} | {raw_metrics['stability']:<6.6f} | {raw_metrics['interpretability']:<6.6f} |\")" ] }, { "cell_type": "code", "execution_count": 37, "id": "4bad2a8a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--------------------------------------------------------------------------------\n", "SELECTED BEST K: 5\n", " Silhouette : 0.555704\n", " Composite Score : 0.877370\n" ] } ], "source": [ "df_results = pd.DataFrame(results)\n", "\n", "best_idx = df_results['composite_score'].idxmax()\n", "best_config = df_results.loc[best_idx]\n", "\n", "best_model = best_config['model']\n", "best_labels = best_config['labels']\n", "best_k = best_config['k']\n", "X_for_clustering = X_latent\n", "\n", "print(\"-\" * 80)\n", "print(f'SELECTED BEST K: {best_k}')\n", "print(f' Silhouette : {best_config[\"silhouette\"]:.6f}') \n", "print(f' Composite Score : {best_config[\"composite_score\"]:.6f}')" ] }, { "cell_type": "markdown", "id": "c318097d", "metadata": {}, "source": [ "# Generate Cluster Label" ] }, { "cell_type": "markdown", "id": "8384d05b", "metadata": {}, "source": [ "## Binning\n", "Divides each continuous feature into 3 quantile bins (tertiles).\n", "\n", "Labels: 0 (low), 0.5 (medium), 1 (high)\n", "\n", "Enables interpretable cluster profiling and feature discretization." ] }, { "cell_type": "code", "execution_count": 38, "id": "0f5aaff8", "metadata": {}, "outputs": [], "source": [ "for col in df.select_dtypes('float64').columns.tolist():\n", " new_col_name = col + '_bin'\n", " df[new_col_name] = pd.qcut(df[col], q=3, labels=[0, 0.5, 1]).astype(int)\n", "\n", "non_numeric_cols = df.select_dtypes(exclude=[np.number]).columns.tolist()\n", "\n", "new_column_order = []\n", "\n", "for col in non_numeric_cols:\n", " if col in df.columns:\n", " new_column_order.append(col)\n", "\n", "for col in binary_cols:\n", " if col in df.columns:\n", " new_column_order.append(col)\n", "\n", "for col in continuous_cols:\n", " if col in df.columns:\n", " new_column_order.append(col)\n", " bin_col_name = col + '_bin'\n", " if bin_col_name in df.columns:\n", " new_column_order.append(bin_col_name)\n", "\n", "if 'cluster' in df.columns and 'cluster' not in new_column_order:\n", " new_column_order.append('cluster')\n", "\n", "df = df[new_column_order]" ] }, { "cell_type": "code", "execution_count": 39, "id": "3309c7d5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 428 entries, 0 to 427\n", "Data columns (total 36 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 brand 428 non-null str \n", " 1 name 428 non-null str \n", " 2 lightweight 428 non-null int64 \n", " 3 rocker 428 non-null int64 \n", " 4 removable_insole 428 non-null int64 \n", " 5 pace_daily_running 428 non-null int64 \n", " 6 pace_tempo 428 non-null int64 \n", " 7 pace_competition 428 non-null int64 \n", " 8 arch_neutral 428 non-null int64 \n", " 9 arch_stability 428 non-null int64 \n", " 10 strike_heel 428 non-null int64 \n", " 11 strike_mid 428 non-null int64 \n", " 12 strike_forefoot 428 non-null int64 \n", " 13 plate_rock_plate 428 non-null int64 \n", " 14 plate_carbon_plate 428 non-null int64 \n", " 15 season_summer 428 non-null int64 \n", " 16 season_winter 428 non-null int64 \n", " 17 season_all 428 non-null int64 \n", " 18 weight_lab_oz 428 non-null float64\n", " 19 weight_lab_oz_bin 428 non-null int64 \n", " 20 drop_lab_mm 428 non-null float64\n", " 21 drop_lab_mm_bin 428 non-null int64 \n", " 22 midsole_softness 428 non-null int64 \n", " 23 toebox_durability 428 non-null int64 \n", " 24 heel_durability 428 non-null int64 \n", " 25 outsole_durability 428 non-null int64 \n", " 26 breathability_scaled 428 non-null int64 \n", " 27 width_fit 428 non-null int64 \n", " 28 toebox_width 428 non-null int64 \n", " 29 stiffness_scaled 428 non-null int64 \n", " 30 torsional_rigidity 428 non-null int64 \n", " 31 heel_stiff 428 non-null int64 \n", " 32 heel_lab_mm 428 non-null float64\n", " 33 heel_lab_mm_bin 428 non-null int64 \n", " 34 forefoot_lab_mm 428 non-null float64\n", " 35 forefoot_lab_mm_bin 428 non-null int64 \n", "dtypes: float64(4), int64(30), str(2)\n", "memory usage: 120.5 KB\n" ] } ], "source": [ "df.info()" ] }, { "cell_type": "markdown", "id": "8b8a8479", "metadata": {}, "source": [ "## Cluster Summary\n", "Creates interpretable profile for each cluster showing:\n", "- Size (count + percentage)\n", "- Continuous features (mean values)\n", "- Binary features (dominant variant + prevalence)" ] }, { "cell_type": "code", "execution_count": 40, "id": "03ab527b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cluster Summary:\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
countpercentageweight_lab_ozdrop_lab_mmmidsole_softnesstoebox_durabilityheel_durabilityoutsole_durabilitybreathability_scaledwidth_fit...lightweightrockerremovable_insolepace_daily_runningpacearchstrikeplate_rock_plateplate_carbon_plateseason
0296.8%9.7010.460.030.000.000.000.002.66...no (17%)no (10%)no (28%)yes (100%)tempo (10%)neutral (76%)heel (83%)no (0%)no (0%)summer (0%)
17417.3%7.477.663.502.002.652.434.112.11...yes (92%)yes (70%)yes (80%)no (9%)competition (65%)neutral (99%)mid (86%)no (1%)yes (64%)all (95%)
216538.6%9.687.353.873.053.403.643.072.89...no (16%)no (30%)yes (99%)yes (96%)tempo (18%)neutral (85%)mid (100%)no (0%)no (5%)all (98%)
37818.2%9.698.202.690.440.290.152.011.74...no (23%)no (29%)yes (96%)yes (96%)tempo (18%)neutral (86%)mid (81%)no (0%)no (1%)all (60%)
48219.2%9.9811.663.782.843.263.353.102.71...no (11%)no (17%)yes (100%)yes (94%)tempo (17%)neutral (77%)heel (100%)no (0%)no (4%)all (96%)
\n", "

5 rows × 26 columns

\n", "
" ], "text/plain": [ " count percentage weight_lab_oz drop_lab_mm midsole_softness \\\n", "0 29 6.8% 9.70 10.46 0.03 \n", "1 74 17.3% 7.47 7.66 3.50 \n", "2 165 38.6% 9.68 7.35 3.87 \n", "3 78 18.2% 9.69 8.20 2.69 \n", "4 82 19.2% 9.98 11.66 3.78 \n", "\n", " toebox_durability heel_durability outsole_durability \\\n", "0 0.00 0.00 0.00 \n", "1 2.00 2.65 2.43 \n", "2 3.05 3.40 3.64 \n", "3 0.44 0.29 0.15 \n", "4 2.84 3.26 3.35 \n", "\n", " breathability_scaled width_fit ... lightweight rocker \\\n", "0 0.00 2.66 ... no (17%) no (10%) \n", "1 4.11 2.11 ... yes (92%) yes (70%) \n", "2 3.07 2.89 ... no (16%) no (30%) \n", "3 2.01 1.74 ... no (23%) no (29%) \n", "4 3.10 2.71 ... no (11%) no (17%) \n", "\n", " removable_insole pace_daily_running pace arch \\\n", "0 no (28%) yes (100%) tempo (10%) neutral (76%) \n", "1 yes (80%) no (9%) competition (65%) neutral (99%) \n", "2 yes (99%) yes (96%) tempo (18%) neutral (85%) \n", "3 yes (96%) yes (96%) tempo (18%) neutral (86%) \n", "4 yes (100%) yes (94%) tempo (17%) neutral (77%) \n", "\n", " strike plate_rock_plate plate_carbon_plate season \n", "0 heel (83%) no (0%) no (0%) summer (0%) \n", "1 mid (86%) no (1%) yes (64%) all (95%) \n", "2 mid (100%) no (0%) no (5%) all (98%) \n", "3 mid (81%) no (0%) no (1%) all (60%) \n", "4 heel (100%) no (0%) no (4%) all (96%) \n", "\n", "[5 rows x 26 columns]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "df['cluster'] = best_labels \n", "\n", "bin_groups = {}\n", "for col in binary_cols:\n", " parts = col.split('_')\n", " \n", " if len(parts) > 1:\n", " prefix = '_'.join(parts[:-1])\n", " else:\n", " prefix = col\n", " \n", " bin_groups.setdefault(prefix, []).append(col)\n", "\n", "rows = []\n", "for cid in sorted(df['cluster'].unique()):\n", " subset = df[df['cluster'] == cid]\n", " n = len(subset)\n", " \n", " row = {'count': n, 'percentage': f\"{n/len(df)*100:.1f}%\"}\n", "\n", " for col in continuous_cols:\n", " row[col.lower()] = round(subset[col].mean(), 2)\n", "\n", " for prefix, cols in bin_groups.items():\n", " means = subset[cols].mean()\n", " best_col = means.idxmax()\n", " best_val = means.max()\n", " \n", " if len(cols) > 1:\n", " header = prefix.lower()\n", " val_str = best_col.replace(f\"{prefix}_\", \"\").lower()\n", " row[header] = f\"{val_str} ({best_val*100:.0f}%)\"\n", " \n", " else:\n", " header = cols[0].lower()\n", " val_str = \"yes\" if best_val > 0.5 else \"no\"\n", " row[header] = f\"{val_str} ({best_val*100:.0f}%)\"\n", "\n", " rows.append(row)\n", "\n", "df_summary = pd.DataFrame(rows, index=sorted(df['cluster'].unique()))\n", "df_summary.index.name = None \n", "\n", "print(\"Cluster Summary:\")\n", "display(df_summary)" ] }, { "cell_type": "markdown", "id": "ce21add3", "metadata": {}, "source": [ "# Deep Learn Recommender" ] }, { "cell_type": "markdown", "id": "93d616fb", "metadata": {}, "source": [ "## Priority Handler" ] }, { "cell_type": "code", "execution_count": 41, "id": "83f48aba", "metadata": {}, "outputs": [], "source": [ "def get_priority_val(user_input, priority_list, mapping_dicts):\n", " \"\"\"\n", " Extract feature value from user input with priority hierarchy.\n", " \n", " Strategy: Checks inputs in priority order, returns mapped value from\n", " first non-empty input, ignores lower-priority inputs if higher-priority exists.\n", " \n", " Args:\n", " user_input (dict): User preferences {'running_purpose': 'Daily', ...}\n", " priority_list (list): Input sources in priority order\n", " mapping_dicts (dict): Maps {source: {option: feature_value}}\n", " \n", " Returns:\n", " float: Feature value [0, 1] or 0.5 (neutral) if not found\n", " \"\"\"\n", " for source_key in priority_list:\n", " if source_key in user_input and user_input[source_key]:\n", " user_choice = user_input[source_key]\n", " if source_key in mapping_dicts:\n", " mapping = mapping_dicts[source_key]\n", " if user_choice in mapping:\n", " return mapping[user_choice]\n", " return 0.5" ] }, { "cell_type": "markdown", "id": "d49d3b53", "metadata": {}, "source": [ "## Input Handler" ] }, { "cell_type": "code", "execution_count": 42, "id": "a0a4d712", "metadata": {}, "outputs": [], "source": [ "def preprocess_user_input_with_mask(user_input, binary_cols, continuous_cols):\n", " \"\"\"\n", " Transform user preferences into feature vector with intelligent masking.\n", " \n", " Feature Construction:\n", " 1. SIMPLE FEATURES (single source dependency e.g., lightweight from pace)\n", " 2. PRIORITY OVERWRITE (multi-source with hierarchy e.g., strike_pattern > pace)\n", " 3. MASKING (only include features derived from provided inputs)\n", " 4. FALLBACK (default to 0.5 for unknowns)\n", " \n", " Returns:\n", " tuple: (full_vector_raw, valid_indices)\n", " - full_vector_raw: Feature vector [0-1] for all features\n", " - valid_indices: Positions of user-provided features\n", " (used for masked similarity calculation)\n", " \"\"\"\n", " feats = {col: 0.0 for col in binary_cols + continuous_cols}\n", " \n", " feats['lightweight'] = get_priority_val(user_input, ['pace'], \n", " {'pace': {'Easy': 0.5, 'Steady': 0.5, 'Fast': 1.0}})\n", " feats['rocker'] = get_priority_val(user_input, ['running_purpose'], \n", " {'running_purpose': {'Race': 1.0, 'Tempo': 0.5, 'Daily': 0.0}})\n", " feats['removable_insole'] = get_priority_val(user_input, ['orthotic_usage'], \n", " {'orthotic_usage': {'Yes': 1.0, 'No': 0.5}})\n", " \n", " purp = user_input.get('running_purpose', 'Daily')\n", " feats['pace_daily_running'] = 1.0 if purp == 'Daily' else (0.5 if purp == 'Tempo' else 0.0)\n", " feats['pace_tempo'] = 1.0 if purp == 'Tempo' else 0.5\n", " feats['pace_competition'] = 1.0 if purp == 'Race' else (0.5 if purp == 'Tempo' else 0.0)\n", "\n", " feats['arch_neutral'] = get_priority_val(user_input, ['arch_type'], \n", " {'arch_type': {'Flat': 0.0, 'Normal': 0.8, 'High': 1.0}})\n", " feats['arch_stability'] = get_priority_val(user_input, ['arch_type'], \n", " {'arch_type': {'Flat': 1.0, 'Normal': 0.2, 'High': 0.0}})\n", " \n", " feats['drop_lab_mm'] = get_priority_val(user_input, ['pace'], \n", " {'pace': {'Easy': 1.0, 'Steady': 0.5, 'Fast': 0.0}})\n", "\n", " prio_strike = ['strike_pattern', 'pace']\n", " feats['strike_heel'] = get_priority_val(user_input, prio_strike, {\n", " 'strike_pattern': {'Heel': 1.0, 'Mid': 0.5, 'Forefoot': 0.0}, \n", " 'pace': {'Easy': 1.0, 'Steady': 0.5, 'Fast': 0.0}})\n", " feats['strike_mid'] = get_priority_val(user_input, prio_strike, {\n", " 'strike_pattern': {'Heel': 0.5, 'Mid': 1.0, 'Forefoot': 0.5}, \n", " 'pace': {'Easy': 0.5, 'Steady': 1.0, 'Fast': 0.5}})\n", " feats['strike_forefoot'] = get_priority_val(user_input, prio_strike, {\n", " 'strike_pattern': {'Heel': 0.0, 'Mid': 0.0, 'Forefoot': 1.0}, \n", " 'pace': {'Easy': 0.0, 'Steady': 0.5, 'Fast': 1.0}})\n", "\n", " prio_soft = ['cushion_preferences', 'pace']\n", " feats['midsole_softness'] = get_priority_val(user_input, prio_soft, {\n", " 'cushion_preferences': {'Soft': 1.0, 'Balanced': 0.6, 'Firm': 0.2}, \n", " 'pace': {'Easy': 1.0, 'Steady': 0.6, 'Fast': 0.2}})\n", "\n", " prio_width = ['stability_need', 'foot_width']\n", " feats['width_fit'] = get_priority_val(user_input, prio_width, {\n", " 'stability_need': {'Neutral': 0.5, 'Guided': 0.2}, \n", " 'foot_width': {'Narrow': 0.2, 'Regular': 0.6, 'Wide': 1}})\n", " \n", " feats['toebox_width'] = get_priority_val(user_input, ['stability_need'], \n", " {'stability_need': {'Neutral': 0.5, 'Guided': 0.2}})\n", " \n", " prio_stiff = ['arch_type', 'pace', 'running_purpose']\n", " feats['stiffness_scaled'] = get_priority_val(user_input, prio_stiff, {\n", " 'arch_type': {'Flat': 0.0, 'Normal': 0.5, 'High': 0.5}, \n", " 'pace': {'Easy': 0.2, 'Steady': 0.6, 'Fast': 1.0}, \n", " 'running_purpose': {'Daily': 0.2, 'Tempo': 0.6, 'Race': 1}})\n", "\n", " prio_tor = ['arch_type', 'pace']\n", " feats['torsional_rigidity'] = get_priority_val(user_input, prio_tor, {\n", " 'arch_type': {'Flat': 1.0, 'Normal': 0.5, 'High': 0.5}, \n", " 'pace': {'Easy': 0.2, 'Steady': 0.6, 'Fast': 1.0}})\n", "\n", " feats['heel_stiff'] = get_priority_val(user_input, ['arch_type'], \n", " {'arch_type': {'Flat': 1.0, 'Normal': 0.6, 'High': 0.2}})\n", "\n", " prio_plate = ['pace', 'running_purpose']\n", " feats['plate_rock'] = get_priority_val(user_input, prio_plate, {\n", " 'pace': {'Easy': 0.5, 'Steady': 0.5, 'Fast': 0.5}, \n", " 'running_purpose': {'Daily': 0.5, 'Tempo': 0.5, 'Race': 0.5}})\n", " feats['plate_carbon'] = get_priority_val(user_input, prio_plate, {\n", " 'pace': {'Easy': 0.5, 'Steady': 0.5, 'Fast': 1.0}, \n", " 'running_purpose': {'Daily': 0.5, 'Tempo': 0.5, 'Race': 1.0}})\n", "\n", " prio_stack = ['strike_pattern', 'pace', 'running_purpose']\n", " feats['heel_lab_mm'] = get_priority_val(user_input, prio_stack, {\n", " 'strike_pattern': {'Heel': 1.0, 'Mid': 0.5, 'Forefoot': 0.0}, \n", " 'pace': {'Easy': 1.0, 'Steady': 0.5, 'Fast': 0.0}, \n", " 'running_purpose': {'Daily': 1.0, 'Tempo': 0.5, 'Race': 0.5}})\n", " feats['forefoot_lab_mm'] = get_priority_val(user_input, prio_stack, {\n", " 'strike_pattern': {'Heel': 0.0, 'Mid': 0.5, 'Forefoot': 1.0}, \n", " 'pace': {'Easy': 0.0, 'Steady': 0.5, 'Fast': 1.0}, \n", " 'running_purpose': {'Daily': 1.0, 'Tempo': 0.5, 'Race': 0.5}})\n", "\n", " feats['weight_lab_oz'] = 1.0 - feats['lightweight']\n", " feats['toebox_durability'] = 1.0\n", " feats['heel_durability'] = 1.0\n", " feats['outsole_durability'] = 1.0\n", " feats['breathability'] = 1.0\n", "\n", " feats['season_summer'] = get_priority_val(user_input, ['season'], \n", " {'season': {'Summer': 1.0, 'Spring & Fall': 0.5, 'Winter': 0.0}})\n", " feats['season_winter'] = get_priority_val(user_input, ['season'], \n", " {'season': {'Summer': 0.0, 'Spring & Fall': 0.0, 'Winter': 1.0}})\n", " feats['season_all'] = get_priority_val(user_input, ['season'], \n", " {'season': {'Summer': 0.5, 'Spring & Fall': 1.0, 'Winter': 0.0}})\n", " \n", " provided_inputs = {k for k, v in user_input.items() if v} # Track which inputs user provided\n", " \n", " feature_sources = {\n", " 'lightweight': ['pace'], 'rocker': ['running_purpose'], 'removable_insole': ['orthotic_usage'],\n", " 'pace_daily_running': ['running_purpose'], 'pace_tempo': ['running_purpose'], 'pace_competition': ['running_purpose'],\n", " 'arch_neutral': ['arch_type'], 'arch_stability': ['arch_type'],\n", " 'drop_lab_mm': ['pace'],\n", " 'strike_heel': ['strike_pattern', 'pace'], 'strike_mid': ['strike_pattern', 'pace'], 'strike_forefoot': ['strike_pattern', 'pace'],\n", " 'midsole_softness': ['cushion_preferences', 'pace'],\n", " 'width_fit': ['stability_need', 'foot_width'],\n", " 'toebox_width': ['stability_need'],\n", " 'stiffness_scaled': ['arch_type', 'pace', 'running_purpose'],\n", " 'torsional_rigidity': ['arch_type', 'pace'],\n", " 'heel_stiff': ['arch_type'],\n", " 'plate_rock': ['pace', 'running_purpose'], 'plate_carbon': ['pace', 'running_purpose'],\n", " 'heel_lab_mm': ['strike_pattern', 'pace', 'running_purpose'], \n", " 'forefoot_lab_mm': ['strike_pattern', 'pace', 'running_purpose'],\n", " 'weight_lab_oz': ['pace'],\n", " 'season_summer': ['season'], 'season_winter': ['season'], 'season_all': ['season'],\n", " 'toebox_durability': [], 'heel_durability': [], 'outsole_durability': [], 'breathability': []\n", " }\n", " \n", " all_cols = binary_cols + continuous_cols\n", " full_vector_raw = []\n", " for col in binary_cols:\n", " full_vector_raw.append(feats.get(col, 0.0))\n", " for col in continuous_cols:\n", " full_vector_raw.append(feats.get(col, 0.5))\n", "\n", " valid_indices = []\n", " for i, col in enumerate(all_cols):\n", " sources = feature_sources.get(col, [])\n", " if any(src in provided_inputs for src in sources):\n", " valid_indices.append(i)\n", " \n", " if not valid_indices:\n", " valid_indices = list(range(len(all_cols)))\n", " \n", " return full_vector_raw, valid_indices" ] }, { "cell_type": "markdown", "id": "faa62fd4", "metadata": {}, "source": [ "## Recommendation" ] }, { "cell_type": "code", "execution_count": 43, "id": "9fbd93a4", "metadata": {}, "outputs": [], "source": [ "def recommend_shoes_deep_masked(user_input, df_data, encoder_model, kmeans_model, binary_cols, continuous_cols, X_combined_data):\n", " \"\"\"\n", " Deep Learning Recommendation Pipeline with Masked Similarity.\n", " \n", " Pipeline:\n", " 1. USER PREPROCESSING: Convert user preferences to feature vector with masking\n", " 2. CLUSTER ROUTING: Encode user→latent space, select top K/3 closest clusters\n", " 3. CANDIDATE RANKING: Score shoes via masked cosine similarity\n", " 4. RESULT: Return top 10 ranked recommendations\n", " \n", " Masking Benefits:\n", " - Reduces noise from unanswered questions\n", " - Focuses similarity on user-provided dimensions only\n", " - Example: if user only provided 'pace', similarity computed on pace-related features\n", " \n", " Args:\n", " user_input (dict): User questionnaire responses\n", " df_data (pd.DataFrame): Shoe catalog\n", " encoder_model: Trained keras encoder\n", " kmeans_model: Trained KMeans model (K clusters)\n", " binary_cols (list): Binary feature names\n", " continuous_cols (list): Continuous feature names\n", " X_combined_data (np.ndarray): Preprocessed feature matrix (n_shoes, n_features)\n", " \n", " Returns:\n", " pd.DataFrame: Top 10 shoes with index (row number) and match_score, sorted descending\n", " \"\"\"\n", " full_vector, valid_idx = preprocess_user_input_with_mask(user_input, binary_cols, continuous_cols)\n", " full_vector = np.array([full_vector])\n", "\n", " user_latent = encoder_model.predict(full_vector, verbose=0)\n", " distances = kmeans_model.transform(user_latent)[0]\n", " n_select = math.ceil(kmeans_model.n_clusters / 3) # Select top 1/3 clusters for diversity\n", " closest_clusters = np.argsort(distances)[:n_select]\n", " \n", " print(f\"User mapped to Clusters: {closest_clusters}\")\n", " \n", " candidates = df_data[df_data['cluster'].isin(closest_clusters)].copy()\n", " if candidates.empty: \n", " return pd.DataFrame()\n", " \n", " candidate_vectors = X_combined_data[candidates.index]\n", " \n", " user_vec_masked = full_vector[:, valid_idx] # Slice user vector to only relevant features\n", " cand_vecs_masked = candidate_vectors[:, valid_idx] # Slice candidate vectors accordingly\n", " \n", " if np.all(user_vec_masked == 0):\n", " scores = np.zeros(len(candidates))\n", " else:\n", " scores = cosine_similarity(user_vec_masked, cand_vecs_masked)[0] # Masked similarity calculation\n", " \n", " candidates['match_score'] = scores\n", " \n", " # Return: sorted by match_score descending, take top 10, keep only match_score (index included as row identifier)\n", " return candidates.sort_values('match_score', ascending=False).head(10)[['match_score']]" ] }, { "cell_type": "markdown", "id": "c21dceba", "metadata": {}, "source": [ "# Testing\n", "Input options for recommendation engine test cases.\n", "\n", "Allows generation of random user preference combinations." ] }, { "cell_type": "markdown", "id": "3ce081fc", "metadata": {}, "source": [ "## Define Options" ] }, { "cell_type": "code", "execution_count": 44, "id": "f4bb1851", "metadata": {}, "outputs": [], "source": [ "input_options = {\n", " 'running_purpose': ['Daily', 'Tempo', 'Race'],\n", " 'pace': ['Easy', 'Steady', 'Fast'],\n", " 'orthotic_usage': ['Yes', 'No'],\n", " 'arch_type': ['Flat', 'Normal', 'High'],\n", " 'strike_pattern': ['Heel', 'Mid', 'Forefoot'],\n", " 'cushion_preferences': ['Soft', 'Balanced', 'Firm'],\n", " 'foot_width': ['Narrow', 'Regular', 'Wide'],\n", " 'stability_need': ['Neutral', 'Guided'],\n", " 'season': ['Summer', 'Winter', 'Spring & Fall']\n", "}\n", "\n", "def generate_random_user_input(num_features):\n", " \"\"\"\n", " Generate randomized user preference input for testing and validation.\n", " \n", " Purpose: Creates realistic test cases with variable input completeness.\n", " \n", " Args:\n", " num_features (int): Number of random features to include\n", " \n", " Returns:\n", " dict: User preferences with num_features random keys/values\n", " e.g., {'pace': 'Fast', 'arch_type': 'Normal', 'season': 'Summer'}\n", " \"\"\"\n", " all_keys = list(input_options.keys())\n", " selected_keys = random.sample(all_keys, k=min(num_features, len(all_keys)))\n", " \n", " user_input = {}\n", " for key in selected_keys:\n", " user_input[key] = random.choice(input_options[key])\n", " \n", " return user_input" ] }, { "cell_type": "markdown", "id": "fc2c6a0e", "metadata": {}, "source": [ "## Execution\n", "Test Suite Execution\n", "Runs recommendation engine on multiple test cases with varying input completeness.\n", "\n", "Tests: 3 features (partial), 6 features (moderate), 9 features (complete)" ] }, { "cell_type": "code", "execution_count": 47, "id": "3f033cef", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== RECOMMENDATION ENGINE TEST SUITE ===\n", "\n", "------------------------------------------------------------\n", "TEST CASE #1: User providing 3 preferences\n", "User Input:\n", "{'cushion_preferences': 'Soft', 'strike_pattern': 'Forefoot', 'foot_width': 'Regular'}\n", "User mapped to Clusters: [2 4]\n", "\n", "Top 10 Recommendations:\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
brandnamematch_scorecluster
0altravia olympus 20.9145982
1altratorin 70.9119512
2altraparadigm 70.9085882
3hokagaviota 50.9079432
4altrarivera 30.9077252
5new balancefresh foam x kaiha road0.9034702
6new balancefresh foam x 880 v150.8992752
7new balancefresh foam x more v60.8987082
8altraexperience flow0.8977852
9new balancefresh foam x vongo v60.8977272
\n", "
" ], "text/plain": [ " brand name match_score cluster\n", "0 altra via olympus 2 0.914598 2\n", "1 altra torin 7 0.911951 2\n", "2 altra paradigm 7 0.908588 2\n", "3 hoka gaviota 5 0.907943 2\n", "4 altra rivera 3 0.907725 2\n", "5 new balance fresh foam x kaiha road 0.903470 2\n", "6 new balance fresh foam x 880 v15 0.899275 2\n", "7 new balance fresh foam x more v6 0.898708 2\n", "8 altra experience flow 0.897785 2\n", "9 new balance fresh foam x vongo v6 0.897727 2" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "------------------------------------------------------------\n", "TEST CASE #2: User providing 6 preferences\n", "User Input:\n", "{'season': 'Spring & Fall', 'pace': 'Fast', 'arch_type': 'Normal', 'running_purpose': 'Race', 'strike_pattern': 'Heel', 'orthotic_usage': 'Yes'}\n", "User mapped to Clusters: [1 4]\n", "\n", "Top 10 Recommendations:\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
brandnamematch_scorecluster
0brookshyperion elite 40.8883321
1new balancefuelcell supercomp elite v30.8801291
2oncloudboom echo 30.8800561
3hokacielo x1 2.00.8777011
4brookshyperion elite 50.8654861
5adidasadizero adios pro 30.8611271
6sauconyendorphin pro 20.8601821
7new balancefuelcell supercomp elite v50.8600541
8sauconyendorphin pro 40.8593891
9asicsmagic speed 40.8581921
\n", "
" ], "text/plain": [ " brand name match_score cluster\n", "0 brooks hyperion elite 4 0.888332 1\n", "1 new balance fuelcell supercomp elite v3 0.880129 1\n", "2 on cloudboom echo 3 0.880056 1\n", "3 hoka cielo x1 2.0 0.877701 1\n", "4 brooks hyperion elite 5 0.865486 1\n", "5 adidas adizero adios pro 3 0.861127 1\n", "6 saucony endorphin pro 2 0.860182 1\n", "7 new balance fuelcell supercomp elite v5 0.860054 1\n", "8 saucony endorphin pro 4 0.859389 1\n", "9 asics magic speed 4 0.858192 1" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "------------------------------------------------------------\n", "TEST CASE #3: User providing 9 preferences\n", "User Input:\n", "{'running_purpose': 'Tempo', 'arch_type': 'High', 'foot_width': 'Regular', 'orthotic_usage': 'Yes', 'season': 'Winter', 'pace': 'Steady', 'strike_pattern': 'Heel', 'stability_need': 'Guided', 'cushion_preferences': 'Firm'}\n", "User mapped to Clusters: [3 1]\n", "\n", "Top 10 Recommendations:\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
brandnamematch_scorecluster
0under armourslipspeed mega0.8000233
1brookslaunch 90.7887303
2pumadeviate nitro elite 30.7721771
3asicsmegablast0.7707191
4adidasadizero prime x 2 strung0.7661431
5brookshyperion elite 40.7635071
6adidasadizero prime x3 strung0.7625271
7nikezoom fly 60.7614241
8adidasrunfalcon0.7591343
9asicsnoosa tri 140.7582333
\n", "
" ], "text/plain": [ " brand name match_score cluster\n", "0 under armour slipspeed mega 0.800023 3\n", "1 brooks launch 9 0.788730 3\n", "2 puma deviate nitro elite 3 0.772177 1\n", "3 asics megablast 0.770719 1\n", "4 adidas adizero prime x 2 strung 0.766143 1\n", "5 brooks hyperion elite 4 0.763507 1\n", "6 adidas adizero prime x3 strung 0.762527 1\n", "7 nike zoom fly 6 0.761424 1\n", "8 adidas runfalcon 0.759134 3\n", "9 asics noosa tri 14 0.758233 3" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "target_counts = [3, 6, 9]\n", "\n", "print(\"=== RECOMMENDATION ENGINE TEST SUITE ===\")\n", "\n", "for i, count in enumerate(target_counts):\n", " print(f\"\\n{'-'*60}\")\n", " print(f\"TEST CASE #{i+1}: User providing {count} preferences\")\n", " \n", " random_input = generate_random_user_input(count)\n", " print(f\"User Input:\\n{random_input}\")\n", " \n", " try:\n", " recommendations = recommend_shoes_deep_masked(\n", " random_input, \n", " df, \n", " encoder, \n", " best_model, \n", " binary_cols, \n", " continuous_cols, \n", " X_combined\n", " )\n", " \n", " if not recommendations.empty:\n", " print(\"\\nTop 10 Recommendations:\")\n", " # Get brand, name, cluster from original df using index, add match_score from recommendations\n", " result_df = pd.DataFrame({\n", " 'brand': df.loc[recommendations.index, 'brand'].values,\n", " 'name': df.loc[recommendations.index, 'name'].values,\n", " 'match_score': recommendations['match_score'].values,\n", " 'cluster': df.loc[recommendations.index, 'cluster'].values\n", " })\n", " display(result_df)\n", " else:\n", " print(\"\\nNo recommendations found (cluster empty).\")\n", " \n", " except NameError:\n", " print(\"\\nERROR: Ensure model and preprocessing functions are loaded.\")\n", " except Exception as e:\n", " print(f\"\\nERROR: {e}\")" ] }, { "cell_type": "markdown", "id": "8a509959", "metadata": {}, "source": [ "# Save Artifacts\n", "Saves 4 artifacts for complete model reconstruction:\n", "1. shoe_encoder.keras: Trained autoencoder (feature encoding)\n", "2. kmeans_model.pkl: Trained K-means clusters\n", "3. shoe_metadata.pkl: Complete shoe dataset with cluster assignments\n", "4. shoe_features.pkl: Preprocessed feature matrix (X_combined)\n", "\n", "Artifacts stored in timestamped versioned directories for traceability." ] }, { "cell_type": "code", "execution_count": 46, "id": "9cb4fe88", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Saving models to: ../../model_artifacts/road/v_20260214_114821\n", "Models saved successfully!\n" ] } ], "source": [ "timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n", "save_dir = f\"../../model_artifacts/road/v_{timestamp}\"\n", "\n", "os.makedirs(save_dir, exist_ok=True)\n", "print(f\"Saving models to: {save_dir}\")\n", "\n", "encoder.save(f'{save_dir}/shoe_encoder.keras')\n", "\n", "with open(f'{save_dir}/kmeans_model.pkl', 'wb') as f:\n", " pickle.dump(best_model, f)\n", "\n", "df.to_pickle(f'{save_dir}/shoe_metadata.pkl')\n", "\n", "with open(f'{save_dir}/shoe_features.pkl', 'wb') as f:\n", " pickle.dump(X_combined, f)\n", "\n", "print(\"Models saved successfully!\")" ] } ], "metadata": { "kernelspec": { "display_name": "env (3.13.1)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.1" } }, "nbformat": 4, "nbformat_minor": 5 }