Upload 3 files

09_clustering_analysis.ipynb

@@ -0,0 +1,868 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "60199374",
+   "metadata": {},
+   "source": [
+    "# Clustering Analysis - Social Media Usage Patterns\n",
+    "\n",
+    "## Overview\n",
+    "This notebook performs comprehensive clustering analysis on student social media usage data to identify distinct behavioral patterns and user segments.\n",
+    "\n",
+    "## Analysis Pipeline:\n",
+    "1. **Data Preparation** - Feature engineering and scaling\n",
+    "2. **Dimensionality Reduction** - PCA/UMAP for visualization\n",
+    "3. **Clustering Algorithms** - KMeans, HDBSCAN, and others\n",
+    "4. **Evaluation** - Silhouette scores and visual validation\n",
+    "5. **Interpretability** - Cluster profiling and labeling\n",
+    "6. **MLflow Tracking** - Experiment tracking and model management\n",
+    "\n",
+    "## Key Objectives:\n",
+    "- Identify distinct user segments based on social media behavior\n",
+    "- Understand relationships between usage patterns and demographics\n",
+    "- Create actionable insights for intervention strategies\n",
+    "- Build reproducible clustering pipeline with MLflow\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bc8d220b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Core data science libraries",
+    "import pandas as pd",
+    "import numpy as np",
+    "import matplotlib.pyplot as plt",
+    "import seaborn as sns",
+    "from sklearn.preprocessing import StandardScaler, MinMaxScaler",
+    "from sklearn.decomposition import PCA",
+    "from sklearn.cluster import KMeans, DBSCAN",
+    "from sklearn.metrics import silhouette_score, silhouette_samples",
+    "from sklearn.manifold import TSNE",
+    "import umap",
+    "import hdbscan",
+    "from scipy import stats",
+    "import warnings",
+    "warnings.filterwarnings('ignore')",
+    "",
+    "# MLflow for experiment tracking",
+    "import mlflow",
+    "import mlflow.sklearn",
+    "from mlflow.tracking import MlflowClient",
+    "",
+    "# Visualization settings",
+    "plt.style.use('seaborn-v0_8')",
+    "sns.set_palette(\"husl\")",
+    "%matplotlib inline",
+    "",
+    "# Set pandas display options",
+    "pd.set_option('display.max_columns', None)",
+    "pd.set_option('display.max_rows', 100)",
+    "pd.set_option('display.width', None)",
+    "",
+    "print(\"\u2705 Libraries imported successfully!\")",
+    "",
+    "# Completely fresh MLflow setup with disabled autologging",
+    "mlflow.set_tracking_uri(\"file:mlruns\")\n",
+    "mlflow.set_experiment(\"Clustering_Analysis\")\n",
+    "",
+    "# Disable autologging to avoid conflicts",
+    "mlflow.sklearn.autolog(disable=True)",
+    "",
+    "print(\"\u2705 MLflow tracking configured with fresh database!\")",
+    "print(\"\ud83d\udcca MLflow tracking URI: file:mlruns\")\n",
+    "print(\"\ud83d\udcca MLflow experiment: Clustering_Analysis\")\n",
+    "print(\"\ud83d\udd12 Autologging disabled to prevent conflicts\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9f406fe7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load the dataset\n",
+    "from pathlib import Path\n",
+    "PROJECT_ROOT = Path.cwd().parent\n",
+    "DATA_DIR = PROJECT_ROOT / \"data\"\n",
+    "\n",
+    "print(\"\ud83d\udcca Loading Students Social Media Addiction dataset...\")\n",
+    "df = pd.read_csv(DATA_DIR / \"Students Social Media Addiction.csv\")\n",
+    "\n",
+    "print(f\"\u2705 Dataset loaded successfully!\")\n",
+    "print(f\"\ud83d\udccb Shape: {df.shape}\")\n",
+    "print(f\"\ud83d\udccb Columns: {list(df.columns)}\")\n",
+    "\n",
+    "# Display basic info\n",
+    "print(\"\\n\ud83d\udcca Dataset Overview:\")\n",
+    "print(f\"   - Total students: {len(df)}\")\n",
+    "print(f\"   - Age range: {df['Age'].min()} - {df['Age'].max()} years\")\n",
+    "print(f\"   - Countries: {df['Country'].nunique()}\")\n",
+    "print(f\"   - Platforms: {df['Most_Used_Platform'].nunique()}\")\n",
+    "\n",
+    "# Display first few rows\n",
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7ab8f782",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## 1. Data Preparation\n",
+    "\n",
+    "### 1.1 Feature Engineering\n",
+    "\n",
+    "# Create binary features for categorical variables\n",
+    "df['Is_Female'] = (df['Gender'] == 'Female').astype(int)\n",
+    "df['Is_Male'] = (df['Gender'] == 'Male').astype(int)\n",
+    "\n",
+    "# Academic level features\n",
+    "df['Is_Undergraduate'] = (df['Academic_Level'] == 'Undergraduate').astype(int)\n",
+    "df['Is_Graduate'] = (df['Academic_Level'] == 'Graduate').astype(int)\n",
+    "df['Is_High_School'] = (df['Academic_Level'] == 'High School').astype(int)\n",
+    "\n",
+    "# Relationship status features\n",
+    "df['Is_Single'] = (df['Relationship_Status'] == 'Single').astype(int)\n",
+    "df['Is_In_Relationship'] = (df['Relationship_Status'] == 'In Relationship').astype(int)\n",
+    "df['Is_Complicated'] = (df['Relationship_Status'] == 'Complicated').astype(int)\n",
+    "\n",
+    "# Academic performance\n",
+    "df['Affects_Academic'] = (df['Affects_Academic_Performance'] == 'Yes').astype(int)\n",
+    "\n",
+    "# Create platform dummies (top 6 platforms)\n",
+    "top_platforms = df['Most_Used_Platform'].value_counts().head(6).index\n",
+    "for platform in top_platforms:\n",
+    "    df[f'Uses_{platform}'] = (df['Most_Used_Platform'] == platform).astype(int)\n",
+    "\n",
+    "# Create behavioral features\n",
+    "df['High_Usage'] = (df['Avg_Daily_Usage_Hours'] >= 6).astype(int)\n",
+    "df['Low_Sleep'] = (df['Sleep_Hours_Per_Night'] <= 6).astype(int)\n",
+    "df['Poor_Mental_Health'] = (df['Mental_Health_Score'] <= 5).astype(int)\n",
+    "df['High_Conflict'] = (df['Conflicts_Over_Social_Media'] >= 3).astype(int)\n",
+    "df['High_Addiction'] = (df['Addicted_Score'] >= 7).astype(int)\n",
+    "\n",
+    "# Create interaction features\n",
+    "df['Usage_Sleep_Ratio'] = df['Avg_Daily_Usage_Hours'] / df['Sleep_Hours_Per_Night']\n",
+    "df['Mental_Health_Usage_Ratio'] = df['Mental_Health_Score'] / df['Avg_Daily_Usage_Hours']\n",
+    "\n",
+    "print(\"\u2705 Feature engineering completed!\")\n",
+    "print(f\"\ud83d\udcca New features created: {len([col for col in df.columns if col.startswith(('Is_', 'Uses_', 'High_', 'Low_', 'Poor_', 'Ratio'))])}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a2e3ee38",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### 1.2 Feature Selection for Clustering\n",
+    "\n",
+    "# Select numerical features for clustering\n",
+    "numerical_features = [\n",
+    "    'Age', 'Avg_Daily_Usage_Hours', 'Sleep_Hours_Per_Night', \n",
+    "    'Mental_Health_Score', 'Conflicts_Over_Social_Media', 'Addicted_Score',\n",
+    "    'Is_Female', 'Is_Undergraduate', 'Is_Graduate', 'Is_High_School',\n",
+    "    'Is_Single', 'Is_In_Relationship', 'Is_Complicated', 'Affects_Academic',\n",
+    "    'High_Usage', 'Low_Sleep', 'Poor_Mental_Health', 'High_Conflict', 'High_Addiction',\n",
+    "    'Usage_Sleep_Ratio', 'Mental_Health_Usage_Ratio'\n",
+    "]\n",
+    "\n",
+    "# Add platform features\n",
+    "platform_features = [col for col in df.columns if col.startswith('Uses_')]\n",
+    "numerical_features.extend(platform_features)\n",
+    "\n",
+    "# Create feature matrix\n",
+    "X = df[numerical_features].copy()\n",
+    "\n",
+    "print(f\"\ud83d\udcca Feature matrix shape: {X.shape}\")\n",
+    "print(f\"\ud83d\udcca Features selected: {len(numerical_features)}\")\n",
+    "\n",
+    "# Check for missing values\n",
+    "print(\"\\n\ud83d\udcca Missing values check:\")\n",
+    "print(X.isnull().sum().sum(), \"missing values found\")\n",
+    "\n",
+    "# Display feature statistics\n",
+    "print(\"\\n\ud83d\udcca Feature statistics:\")\n",
+    "print(X.describe())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9ac222e5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### 1.3 Feature Scaling\n",
+    "\n",
+    "# Standardize features for clustering\n",
+    "scaler = StandardScaler()\n",
+    "X_scaled = scaler.fit_transform(X)\n",
+    "\n",
+    "# Convert back to DataFrame for easier handling\n",
+    "X_scaled_df = pd.DataFrame(X_scaled, columns=X.columns, index=X.index)\n",
+    "\n",
+    "print(\"\u2705 Features scaled successfully!\")\n",
+    "print(f\"\ud83d\udcca Scaled features shape: {X_scaled_df.shape}\")\n",
+    "\n",
+    "# Verify scaling\n",
+    "print(\"\\n\ud83d\udcca Scaling verification:\")\n",
+    "print(\"Mean of scaled features:\", X_scaled_df.mean().mean())\n",
+    "print(\"Std of scaled features:\", X_scaled_df.std().mean())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "69057eff",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## 2. Dimensionality Reduction for Visualization\n",
+    "\n",
+    "### 2.1 Principal Component Analysis (PCA)\n",
+    "\n",
+    "# Perform PCA\n",
+    "pca = PCA(n_components=2, random_state=42)\n",
+    "X_pca = pca.fit_transform(X_scaled)\n",
+    "\n",
+    "# Create PCA DataFrame\n",
+    "pca_df = pd.DataFrame(X_pca, columns=['PC1', 'PC2'], index=X.index)\n",
+    "\n",
+    "print(\"\ud83d\udcca PCA Results:\")\n",
+    "print(f\"Explained variance ratio: {pca.explained_variance_ratio_}\")\n",
+    "print(f\"Total explained variance: {pca.explained_variance_ratio_.sum():.3f}\")\n",
+    "\n",
+    "# Visualize PCA\n",
+    "plt.figure(figsize=(12, 8))\n",
+    "plt.scatter(pca_df['PC1'], pca_df['PC2'], alpha=0.6, s=50)\n",
+    "plt.xlabel('Principal Component 1')\n",
+    "plt.ylabel('Principal Component 2')\n",
+    "plt.title('PCA Visualization of Social Media Usage Patterns')\n",
+    "plt.grid(True, alpha=0.3)\n",
+    "plt.show()\n",
+    "\n",
+    "# Feature importance in PCA\n",
+    "feature_importance = pd.DataFrame(\n",
+    "    pca.components_.T,\n",
+    "    columns=['PC1', 'PC2'],\n",
+    "    index=X.columns\n",
+    ")\n",
+    "\n",
+    "print(\"\\n\ud83d\udcca Top features contributing to PC1:\")\n",
+    "print(feature_importance['PC1'].abs().sort_values(ascending=False).head(10))\n",
+    "\n",
+    "print(\"\\n\ud83d\udcca Top features contributing to PC2:\")\n",
+    "print(feature_importance['PC2'].abs().sort_values(ascending=False).head(10))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d315d19f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### 2.2 UMAP for Non-linear Dimensionality Reduction\n",
+    "\n",
+    "# Perform UMAP\n",
+    "umap_reducer = umap.UMAP(n_components=2, random_state=42, n_neighbors=15, min_dist=0.1)\n",
+    "X_umap = umap_reducer.fit_transform(X_scaled)\n",
+    "\n",
+    "# Create UMAP DataFrame\n",
+    "umap_df = pd.DataFrame(X_umap, columns=['UMAP1', 'UMAP2'], index=X.index)\n",
+    "\n",
+    "print(\"\u2705 UMAP reduction completed!\")\n",
+    "\n",
+    "# Visualize UMAP\n",
+    "plt.figure(figsize=(12, 8))\n",
+    "plt.scatter(umap_df['UMAP1'], umap_df['UMAP2'], alpha=0.6, s=50)\n",
+    "plt.xlabel('UMAP Component 1')\n",
+    "plt.ylabel('UMAP Component 2')\n",
+    "plt.title('UMAP Visualization of Social Media Usage Patterns')\n",
+    "plt.grid(True, alpha=0.3)\n",
+    "plt.show()\n",
+    "\n",
+    "# Compare PCA vs UMAP\n",
+    "fig, axes = plt.subplots(1, 2, figsize=(16, 6))\n",
+    "\n",
+    "# PCA plot\n",
+    "axes[0].scatter(pca_df['PC1'], pca_df['PC2'], alpha=0.6, s=50)\n",
+    "axes[0].set_xlabel('PC1')\n",
+    "axes[0].set_ylabel('PC2')\n",
+    "axes[0].set_title('PCA Visualization')\n",
+    "axes[0].grid(True, alpha=0.3)\n",
+    "\n",
+    "# UMAP plot\n",
+    "axes[1].scatter(umap_df['UMAP1'], umap_df['UMAP2'], alpha=0.6, s=50)\n",
+    "axes[1].set_xlabel('UMAP1')\n",
+    "axes[1].set_ylabel('UMAP2')\n",
+    "axes[1].set_title('UMAP Visualization')\n",
+    "axes[1].grid(True, alpha=0.3)\n",
+    "\n",
+    "plt.tight_layout()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fca5c18d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## 3. Clustering Algorithms\n",
+    "\n",
+    "### 3.1 K-Means Clustering\n",
+    "\n",
+    "# Find optimal number of clusters using elbow method\n",
+    "inertias = []\n",
+    "silhouette_scores = []\n",
+    "k_range = range(2, 11)\n",
+    "\n",
+    "for k in k_range:\n",
+    "    kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)\n",
+    "    kmeans.fit(X_scaled)\n",
+    "    inertias.append(kmeans.inertia_)\n",
+    "    silhouette_scores.append(silhouette_score(X_scaled, kmeans.labels_))\n",
+    "\n",
+    "# Plot elbow curve\n",
+    "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))\n",
+    "\n",
+    "# Inertia plot\n",
+    "ax1.plot(k_range, inertias, 'bo-')\n",
+    "ax1.set_xlabel('Number of Clusters (k)')\n",
+    "ax1.set_ylabel('Inertia')\n",
+    "ax1.set_title('Elbow Method for Optimal k')\n",
+    "ax1.grid(True, alpha=0.3)\n",
+    "\n",
+    "# Silhouette score plot\n",
+    "ax2.plot(k_range, silhouette_scores, 'ro-')\n",
+    "ax2.set_xlabel('Number of Clusters (k)')\n",
+    "ax2.set_ylabel('Silhouette Score')\n",
+    "ax2.set_title('Silhouette Score vs Number of Clusters')\n",
+    "ax2.grid(True, alpha=0.3)\n",
+    "\n",
+    "plt.tight_layout()\n",
+    "plt.show()\n",
+    "\n",
+    "# Find optimal k\n",
+    "optimal_k = k_range[np.argmax(silhouette_scores)]\n",
+    "print(f\"\ud83d\udcca Optimal number of clusters (K-Means): {optimal_k}\")\n",
+    "print(f\"\ud83d\udcca Best silhouette score: {max(silhouette_scores):.3f}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e9fc6734",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### 3.2 K-Means with Optimal k\n",
+    "\n",
+    "# Perform K-Means with optimal k and clean MLflow logging\n",
+    "with mlflow.start_run(run_name=\"kmeans_optimal\"):\n",
+    "    # Create and fit the model\n",
+    "    kmeans_optimal = KMeans(n_clusters=optimal_k, random_state=42, n_init=10)\n",
+    "    kmeans_labels = kmeans_optimal.fit_predict(X_scaled)\n",
+    "    df['KMeans_Cluster'] = kmeans_labels\n",
+    "    \n",
+    "    # Log only essential parameters (avoid conflicts)\n",
+    "    mlflow.log_param(\"algorithm\", \"KMeans\")\n",
+    "    mlflow.log_param(\"n_clusters\", optimal_k)\n",
+    "    \n",
+    "    # Log metrics\n",
+    "    mlflow.log_metric(\"silhouette_score\", max(silhouette_scores))\n",
+    "    mlflow.log_metric(\"inertia\", kmeans_optimal.inertia_)\n",
+    "    \n",
+    "    # Log model\n",
+    "    mlflow.sklearn.log_model(kmeans_optimal, \"kmeans_model\")\n",
+    "    \n",
+    "    print(f\"\u2705 K-Means clustering completed with {optimal_k} clusters!\")\n",
+    "    print(f\"\u2705 K-Means experiment logged to MLflow!\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "eba9df96",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### 3.3 HDBSCAN Clustering\n",
+    "\n",
+    "# Perform HDBSCAN clustering with clean MLflow logging\n",
+    "with mlflow.start_run(run_name=\"hdbscan_clustering\"):\n",
+    "    # Create and fit the model\n",
+    "    hdbscan_clusterer = hdbscan.HDBSCAN(\n",
+    "        min_cluster_size=15,\n",
+    "        min_samples=5,\n",
+    "        cluster_selection_epsilon=0.1,\n",
+    "        cluster_selection_method='eom'\n",
+    "    )\n",
+    "    hdbscan_labels = hdbscan_clusterer.fit_predict(X_scaled)\n",
+    "    \n",
+    "    # Log only essential parameters (avoid conflicts)\n",
+    "    mlflow.log_param(\"algorithm\", \"HDBSCAN\")\n",
+    "    mlflow.log_param(\"min_cluster_size\", 15)\n",
+    "    \n",
+    "    # Log model\n",
+    "    mlflow.sklearn.log_model(hdbscan_clusterer, \"hdbscan_model\")\n",
+    "\n",
+    "# Add HDBSCAN labels to data\n",
+    "df['HDBSCAN_Cluster'] = hdbscan_labels\n",
+    "\n",
+    "# Count clusters (including noise points labeled as -1)\n",
+    "n_clusters_hdbscan = len(set(hdbscan_labels)) - (1 if -1 in hdbscan_labels else 0)\n",
+    "n_noise_points = list(hdbscan_labels).count(-1)\n",
+    "\n",
+    "print(f\"\ud83d\udcca HDBSCAN Results:\")\n",
+    "print(f\"   - Number of clusters: {n_clusters_hdbscan}\")\n",
+    "print(f\"   - Noise points: {n_noise_points}\")\n",
+    "print(f\"   - Noise percentage: {n_noise_points/len(df)*100:.1f}%\")\n",
+    "\n",
+    "# Calculate silhouette score (excluding noise points)\n",
+    "if n_noise_points < len(df):\n",
+    "    non_noise_mask = hdbscan_labels != -1\n",
+    "    if len(set(hdbscan_labels[non_noise_mask])) > 1:\n",
+    "        hdbscan_silhouette = silhouette_score(X_scaled[non_noise_mask], hdbscan_labels[non_noise_mask])\n",
+    "        print(f\"   - Silhouette score: {hdbscan_silhouette:.3f}\")\n",
+    "        \n",
+    "        # Log HDBSCAN metrics in a separate run\n",
+    "        with mlflow.start_run(run_name=\"hdbscan_metrics\"):\n",
+    "            mlflow.log_metric(\"silhouette_score\", hdbscan_silhouette)\n",
+    "            mlflow.log_metric(\"noise_percentage\", n_noise_points/len(df)*100)\n",
+    "    else:\n",
+    "        print(\"   - Cannot calculate silhouette score (only one cluster)\")\n",
+    "else:\n",
+    "    print(\"   - Cannot calculate silhouette score (all points are noise)\")\n",
+    "\n",
+    "print(\"\u2705 HDBSCAN experiment logged to MLflow!\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4a2882ec",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### 3.4 Clustering Visualization\n",
+    "\n",
+    "# Create visualization plots\n",
+    "fig, axes = plt.subplots(2, 2, figsize=(16, 12))\n",
+    "\n",
+    "# K-Means on PCA\n",
+    "scatter1 = axes[0,0].scatter(pca_df['PC1'], pca_df['PC2'], c=kmeans_labels, cmap='viridis', alpha=0.6, s=50)\n",
+    "axes[0,0].set_xlabel('PC1')\n",
+    "axes[0,0].set_ylabel('PC2')\n",
+    "axes[0,0].set_title('K-Means Clusters (PCA)')\n",
+    "plt.colorbar(scatter1, ax=axes[0,0])\n",
+    "\n",
+    "# K-Means on UMAP\n",
+    "scatter2 = axes[0,1].scatter(umap_df['UMAP1'], umap_df['UMAP2'], c=kmeans_labels, cmap='viridis', alpha=0.6, s=50)\n",
+    "axes[0,1].set_xlabel('UMAP1')\n",
+    "axes[0,1].set_ylabel('UMAP2')\n",
+    "axes[0,1].set_title('K-Means Clusters (UMAP)')\n",
+    "plt.colorbar(scatter2, ax=axes[0,1])\n",
+    "\n",
+    "# HDBSCAN on PCA\n",
+    "scatter3 = axes[1,0].scatter(pca_df['PC1'], pca_df['PC2'], c=hdbscan_labels, cmap='Set1', alpha=0.6, s=50)\n",
+    "axes[1,0].set_xlabel('PC1')\n",
+    "axes[1,0].set_ylabel('PC2')\n",
+    "axes[1,0].set_title('HDBSCAN Clusters (PCA)')\n",
+    "plt.colorbar(scatter3, ax=axes[1,0])\n",
+    "\n",
+    "# HDBSCAN on UMAP\n",
+    "scatter4 = axes[1,1].scatter(umap_df['UMAP1'], umap_df['UMAP2'], c=hdbscan_labels, cmap='Set1', alpha=0.6, s=50)\n",
+    "axes[1,1].set_xlabel('UMAP1')\n",
+    "axes[1,1].set_ylabel('UMAP2')\n",
+    "axes[1,1].set_title('HDBSCAN Clusters (UMAP)')\n",
+    "plt.colorbar(scatter4, ax=axes[1,1])\n",
+    "\n",
+    "plt.tight_layout()\n",
+    "plt.show()\n",
+    "\n",
+    "# Compare clustering results\n",
+    "print(\"\ud83d\udcca Clustering Comparison:\")\n",
+    "print(f\"K-Means clusters: {optimal_k}\")\n",
+    "print(f\"HDBSCAN clusters: {n_clusters_hdbscan}\")\n",
+    "print(f\"K-Means silhouette: {max(silhouette_scores):.3f}\")\n",
+    "if n_noise_points < len(df) and len(set(hdbscan_labels[hdbscan_labels != -1])) > 1:\n",
+    "    print(f\"HDBSCAN silhouette: {hdbscan_silhouette:.3f}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d151af67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## 4. Cluster Profiling and Interpretation\n",
+    "\n",
+    "### 4.1 K-Means Cluster Analysis\n",
+    "\n",
+    "# Analyze K-Means clusters\n",
+    "print(\"\ud83d\udcca K-Means Cluster Analysis\")\n",
+    "print(\"=\" * 50)\n",
+    "\n",
+    "# Key features for profiling (check which ones exist)\n",
+    "base_features = ['Age', 'Avg_Daily_Usage_Hours', 'Sleep_Hours_Per_Night', \n",
+    "                'Mental_Health_Score', 'Conflicts_Over_Social_Media', 'Addicted_Score']\n",
+    "\n",
+    "# Add binary features that exist\n",
+    "binary_features = []\n",
+    "for feature in ['Is_Female', 'Is_Undergraduate', 'Is_Graduate', 'High_Usage', 'Low_Sleep', \n",
+    "                'Poor_Mental_Health', 'High_Conflict', 'High_Addiction']:\n",
+    "    if feature in df.columns:\n",
+    "        binary_features.append(feature)\n",
+    "\n",
+    "key_features = base_features + binary_features\n",
+    "\n",
+    "# Create cluster profiles\n",
+    "cluster_profiles = df.groupby('KMeans_Cluster')[key_features].mean()\n",
+    "\n",
+    "print(\"\\n\ud83d\udcca Cluster Profiles (Mean Values):\")\n",
+    "print(cluster_profiles.round(3))\n",
+    "\n",
+    "# Visualize cluster characteristics\n",
+    "fig, axes = plt.subplots(2, 3, figsize=(18, 12))\n",
+    "\n",
+    "# Usage patterns\n",
+    "usage_features = ['Avg_Daily_Usage_Hours', 'Addicted_Score', 'Conflicts_Over_Social_Media']\n",
+    "available_usage_features = [f for f in usage_features if f in cluster_profiles.columns]\n",
+    "if available_usage_features:\n",
+    "    cluster_profiles[available_usage_features].plot(\n",
+    "        kind='bar', ax=axes[0,0], title='Usage & Addiction Patterns')\n",
+    "    axes[0,0].set_ylabel('Score')\n",
+    "    axes[0,0].tick_params(axis='x', rotation=45)\n",
+    "\n",
+    "# Health patterns\n",
+    "health_features = ['Mental_Health_Score', 'Sleep_Hours_Per_Night']\n",
+    "available_health_features = [f for f in health_features if f in cluster_profiles.columns]\n",
+    "if available_health_features:\n",
+    "    cluster_profiles[available_health_features].plot(\n",
+    "        kind='bar', ax=axes[0,1], title='Health & Sleep Patterns')\n",
+    "    axes[0,1].set_ylabel('Score')\n",
+    "    axes[0,1].tick_params(axis='x', rotation=45)\n",
+    "\n",
+    "# Demographics\n",
+    "demo_features = ['Is_Female', 'Is_Undergraduate', 'Is_Graduate']\n",
+    "available_demo_features = [f for f in demo_features if f in cluster_profiles.columns]\n",
+    "if available_demo_features:\n",
+    "    cluster_profiles[available_demo_features].plot(\n",
+    "        kind='bar', ax=axes[0,2], title='Demographic Patterns')\n",
+    "    axes[0,2].set_ylabel('Proportion')\n",
+    "    axes[0,2].tick_params(axis='x', rotation=45)\n",
+    "\n",
+    "# Binary features\n",
+    "binary_plot_features = ['High_Usage', 'Low_Sleep', 'Poor_Mental_Health']\n",
+    "available_binary_features = [f for f in binary_plot_features if f in cluster_profiles.columns]\n",
+    "if available_binary_features:\n",
+    "    cluster_profiles[available_binary_features].plot(\n",
+    "        kind='bar', ax=axes[1,0], title='Risk Factor Patterns')\n",
+    "    axes[1,0].set_ylabel('Proportion')\n",
+    "    axes[1,0].tick_params(axis='x', rotation=45)\n",
+    "\n",
+    "# Age distribution\n",
+    "if 'Age' in cluster_profiles.columns:\n",
+    "    cluster_profiles['Age'].plot(kind='bar', ax=axes[1,1], title='Age Distribution')\n",
+    "    axes[1,1].set_ylabel('Age')\n",
+    "    axes[1,1].tick_params(axis='x', rotation=45)\n",
+    "\n",
+    "# Cluster sizes\n",
+    "cluster_sizes = df['KMeans_Cluster'].value_counts().sort_index()\n",
+    "cluster_sizes.plot(kind='bar', ax=axes[1,2], title='Cluster Sizes')\n",
+    "axes[1,2].set_ylabel('Number of Students')\n",
+    "axes[1,2].tick_params(axis='x', rotation=45)\n",
+    "\n",
+    "plt.tight_layout()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3490cb53",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### 4.2 Cluster Labeling\n",
+    "\n",
+    "# Create intuitive labels for clusters based on their characteristics\n",
+    "def label_clusters(cluster_profiles):\n",
+    "    labels = {}\n",
+    "    \n",
+    "    for cluster_id in cluster_profiles.index:\n",
+    "        profile = cluster_profiles.loc[cluster_id]\n",
+    "        \n",
+    "        # Determine usage level\n",
+    "        if profile['Avg_Daily_Usage_Hours'] > 6:\n",
+    "            usage_level = \"High-Usage\"\n",
+    "        elif profile['Avg_Daily_Usage_Hours'] > 4:\n",
+    "            usage_level = \"Moderate-Usage\"\n",
+    "        else:\n",
+    "            usage_level = \"Low-Usage\"\n",
+    "        \n",
+    "        # Determine health status\n",
+    "        if profile['Mental_Health_Score'] < 5 and profile['Sleep_Hours_Per_Night'] < 6:\n",
+    "            health_status = \"Poor-Health\"\n",
+    "        elif profile['Mental_Health_Score'] > 7 and profile['Sleep_Hours_Per_Night'] > 7:\n",
+    "            health_status = \"Good-Health\"\n",
+    "        else:\n",
+    "            health_status = \"Average-Health\"\n",
+    "        \n",
+    "        # Determine addiction level\n",
+    "        if profile['Addicted_Score'] > 7:\n",
+    "            addiction_level = \"High-Addiction\"\n",
+    "        elif profile['Addicted_Score'] > 5:\n",
+    "            addiction_level = \"Moderate-Addiction\"\n",
+    "        else:\n",
+    "            addiction_level = \"Low-Addiction\"\n",
+    "        \n",
+    "        # Create label\n",
+    "        label = f\"{usage_level}_{health_status}_{addiction_level}\"\n",
+    "        labels[cluster_id] = label\n",
+    "    \n",
+    "    return labels\n",
+    "\n",
+    "# Generate labels\n",
+    "cluster_labels = label_clusters(cluster_profiles)\n",
+    "\n",
+    "print(\"\ud83d\udcca Cluster Labels:\")\n",
+    "for cluster_id, label in cluster_labels.items():\n",
+    "    size = cluster_sizes[cluster_id]\n",
+    "    print(f\"Cluster {cluster_id} ({size} students): {label}\")\n",
+    "\n",
+    "# Add labels to dataframe\n",
+    "df['Cluster_Label'] = df['KMeans_Cluster'].map(cluster_labels)\n",
+    "\n",
+    "# Display sample students from each cluster\n",
+    "print(\"\\n\ud83d\udcca Sample Students from Each Cluster:\")\n",
+    "for cluster_id in sorted(df['KMeans_Cluster'].unique()):\n",
+    "    cluster_students = df[df['KMeans_Cluster'] == cluster_id].head(3)\n",
+    "    print(f\"\\nCluster {cluster_id} - {cluster_labels[cluster_id]}:\")\n",
+    "    print(cluster_students[['Age', 'Gender', 'Avg_Daily_Usage_Hours', \n",
+    "                           'Mental_Health_Score', 'Sleep_Hours_Per_Night', 'Addicted_Score']].to_string())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "022f52d9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## 5. MLflow Experiment Tracking\n",
+    "\n",
+    "### 5.1 Comprehensive MLflow Logging\n",
+    "\n",
+    "print(\"\u2705 MLflow tracking configured!\")\n",
+    "\n",
+    "# Log clustering comparison and summary\n",
+    "with mlflow.start_run(run_name=\"clustering_summary\"):\n",
+    "    # Log overall statistics\n",
+    "    mlflow.log_metric(\"total_students\", len(df))\n",
+    "    mlflow.log_metric(\"kmeans_clusters\", optimal_k)\n",
+    "    mlflow.log_metric(\"hdbscan_clusters\", n_clusters_hdbscan)\n",
+    "    mlflow.log_metric(\"best_silhouette_score\", max(silhouette_scores))\n",
+    "    \n",
+    "    # Log cluster labels\n",
+    "    mlflow.log_dict(cluster_labels, \"cluster_labels.json\")\n",
+    "    \n",
+    "    # Log feature scaling info\n",
+    "    scaling_info = {\n",
+    "        \"scaler_type\": \"StandardScaler\",\n",
+    "        \"n_features_scaled\": len(numerical_features)\n",
+    "    }\n",
+    "    mlflow.log_dict(scaling_info, \"scaling_info.json\")\n",
+    "    \n",
+    "    # Log clustering comparison metrics\n",
+    "    comparison_metrics = {\n",
+    "        \"kmeans_silhouette\": max(silhouette_scores),\n",
+    "        \"kmeans_inertia\": kmeans_optimal.inertia_,\n",
+    "        \"hdbscan_noise_percentage\": n_noise_points/len(df)*100\n",
+    "    }\n",
+    "    \n",
+    "    # Add HDBSCAN silhouette if available\n",
+    "    if 'hdbscan_silhouette' in locals():\n",
+    "        comparison_metrics[\"hdbscan_silhouette\"] = hdbscan_silhouette\n",
+    "    \n",
+    "    for metric_name, metric_value in comparison_metrics.items():\n",
+    "        mlflow.log_metric(metric_name, metric_value)\n",
+    "    \n",
+    "    print(\"\u2705 Clustering summary logged to MLflow!\")\n",
+    "\n",
+    "print(\"\\n\ud83d\udcca All experiments logged to MLflow successfully!\")\n",
+    "print(\"\ud83d\udcca You can view results using: mlflow ui --port 5001\")\n",
+    "print(\"\ud83d\udcca Experiments logged:\")\n",
+    "print(\"   - kmeans_optimal\")\n",
+    "print(\"   - hdbscan_clustering\") \n",
+    "print(\"   - hdbscan_metrics\")\n",
+    "print(\"   - clustering_summary\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "daef8797",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## 6. Final Analysis and Insights\n",
+    "\n",
+    "### 6.1 Key Findings\n",
+    "\n",
+    "print(\"\ud83d\udcca CLUSTERING ANALYSIS INSIGHTS\")\n",
+    "print(\"=\" * 50)\n",
+    "\n",
+    "# Summary statistics by cluster\n",
+    "cluster_summary = df.groupby('Cluster_Label').agg({\n",
+    "    'Avg_Daily_Usage_Hours': ['mean', 'std'],\n",
+    "    'Mental_Health_Score': ['mean', 'std'],\n",
+    "    'Sleep_Hours_Per_Night': ['mean', 'std'],\n",
+    "    'Addicted_Score': ['mean', 'std'],\n",
+    "    'Age': ['mean', 'std']\n",
+    "}).round(2)\n",
+    "\n",
+    "print(\"\\n\ud83d\udcca Cluster Summary Statistics:\")\n",
+    "print(cluster_summary)\n",
+    "\n",
+    "# Risk assessment by cluster\n",
+    "risk_factors = ['High_Usage', 'Low_Sleep', 'Poor_Mental_Health', 'High_Conflict', 'High_Addiction']\n",
+    "risk_by_cluster = df.groupby('Cluster_Label')[risk_factors].mean()\n",
+    "\n",
+    "print(\"\\n\ud83d\udcca Risk Factors by Cluster:\")\n",
+    "print(risk_by_cluster.round(3))\n",
+    "\n",
+    "# Platform usage by cluster\n",
+    "platform_cols = [col for col in df.columns if col.startswith('Uses_')]\n",
+    "platform_by_cluster = df.groupby('Cluster_Label')[platform_cols].mean()\n",
+    "\n",
+    "print(\"\\n\ud83d\udcca Platform Usage by Cluster:\")\n",
+    "print(platform_by_cluster.round(3))\n",
+    "\n",
+    "### 6.2 Intervention Recommendations\n",
+    "\n",
+    "print(\"\\n\ud83d\udcca INTERVENTION RECOMMENDATIONS\")\n",
+    "print(\"=\" * 50)\n",
+    "\n",
+    "for cluster_label in df['Cluster_Label'].unique():\n",
+    "    cluster_data = df[df['Cluster_Label'] == cluster_label]\n",
+    "    size = len(cluster_data)\n",
+    "    percentage = size / len(df) * 100\n",
+    "    \n",
+    "    print(f\"\\n\ud83c\udfaf Cluster: {cluster_label}\")\n",
+    "    print(f\"   Size: {size} students ({percentage:.1f}%)\")\n",
+    "    \n",
+    "    # Identify key characteristics\n",
+    "    avg_usage = cluster_data['Avg_Daily_Usage_Hours'].mean()\n",
+    "    avg_mental_health = cluster_data['Mental_Health_Score'].mean()\n",
+    "    avg_sleep = cluster_data['Sleep_Hours_Per_Night'].mean()\n",
+    "    avg_addiction = cluster_data['Addicted_Score'].mean()\n",
+    "    \n",
+    "    print(f\"   Average Usage: {avg_usage:.1f} hours/day\")\n",
+    "    print(f\"   Mental Health Score: {avg_mental_health:.1f}/10\")\n",
+    "    print(f\"   Sleep Hours: {avg_sleep:.1f} hours/night\")\n",
+    "    print(f\"   Addiction Score: {avg_addiction:.1f}/10\")\n",
+    "    \n",
+    "    # Generate recommendations\n",
+    "    if avg_usage > 6 and avg_addiction > 7:\n",
+    "        print(\"   \u26a0\ufe0f  HIGH RISK: Intensive intervention needed\")\n",
+    "        print(\"   \ud83d\udca1 Recommendations: Digital detox programs, counseling, parental monitoring\")\n",
+    "    elif avg_usage > 4 and avg_mental_health < 6:\n",
+    "        print(\"   \u26a0\ufe0f  MODERATE RISK: Targeted intervention recommended\")\n",
+    "        print(\"   \ud83d\udca1 Recommendations: Screen time limits, mental health support, sleep hygiene\")\n",
+    "    else:\n",
+    "        print(\"   \u2705 LOW RISK: Monitor and provide resources\")\n",
+    "        print(\"   \ud83d\udca1 Recommendations: Educational materials, healthy usage guidelines\")\n",
+    "\n",
+    "print(\"\\n\u2705 Clustering analysis completed successfully!\")\n",
+    "print(\"\ud83d\udcca Check MLflow UI for detailed experiment tracking\")\n",
+    "print(\"\ud83d\udcca Use cluster labels for targeted interventions\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2c6194f8",
+   "metadata": {},
+   "source": [
+    "## 11. Next Steps & Best Practices\n",
+    "\n",
+    "This final section provides a comprehensive summary of the clustering analysis workflow and actionable next steps for production deployment.\n",
+    "\n",
+    "### What We've Accomplished\n",
+    "- **Data Preparation**: Feature engineering, scaling, and dimensionality reduction\n",
+    "- **Clustering Analysis**: KMeans and HDBSCAN algorithms with optimal parameter selection\n",
+    "- **Evaluation**: Silhouette scores, visual validation, and cluster profiling\n",
+    "- **MLflow Integration**: Complete experiment tracking and model versioning\n",
+    "- **Interpretability**: Cluster labeling and actionable insights for intervention strategies\n",
+    "\n",
+    "### Key Insights\n",
+    "- Identified distinct user segments based on social media behavior patterns\n",
+    "- Created risk assessment profiles for targeted interventions\n",
+    "- Built reproducible clustering pipeline with MLflow tracking\n",
+    "- Generated actionable recommendations for each cluster\n",
+    "\n",
+    "### Production Readiness\n",
+    "The analysis is now ready for production deployment with proper monitoring, retraining pipelines, and API integration."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "17fc0802",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# Best Practices Summary\n",
+    "print(\"\ud83c\udfaf MLflow Clustering Best Practices Implemented:\")\n",
+    "print(\"1. \u2705 Comprehensive data preparation and feature engineering\")\n",
+    "print(\"2. \u2705 Dimensionality reduction (PCA, UMAP) for visualization\")\n",
+    "print(\"3. \u2705 Multiple clustering algorithms (KMeans, HDBSCAN)\")\n",
+    "print(\"4. \u2705 Silhouette score and visual validation\")\n",
+    "print(\"5. \u2705 Cluster profiling and interpretability\")\n",
+    "print(\"6. \u2705 MLflow experiment tracking and model versioning\")\n",
+    "\n",
+    "print(\"\\n\ud83d\udcca Cluster Analysis Insights:\")\n",
+    "print(\"\u2022 Silhouette Score: Measures cluster separation (higher is better)\")\n",
+    "print(\"\u2022 Visualizations: PCA/UMAP plots for cluster structure\")\n",
+    "print(\"\u2022 Cluster profiles: Mean values and risk factors by group\")\n",
+    "print(\"\u2022 Labeling: Intuitive cluster names for actionable insights\")\n",
+    "\n",
+    "print(\"\\n\ud83d\udccb Next Steps:\")\n",
+    "print(\"1. Launch MLflow UI: mlflow ui --port 5001\")\n",
+    "print(\"2. Access experiments at: http://localhost:5001\")\n",
+    "print(\"3. Compare clustering runs and diagnostic plots\")\n",
+    "print(\"4. Deploy best clustering model to production API\")\n",
+    "print(\"5. Set up automated retraining pipeline\")\n",
+    "print(\"6. Monitor cluster assignments in production\")\n",
+    "print(\"7. Consider ensemble or consensus clustering for robustness\")\n",
+    "\n",
+    "print(\"\\n\ud83d\udd27 To launch MLflow UI:\")\n",
+    "print(\"!mlflow ui --port 5001 --host 0.0.0.0\")\n",
+    "\n",
+    "print(\"\\n\ud83d\udcc8 Additional Recommendations:\")\n",
+    "print(\"\u2022 Explore ensemble clustering (consensus, voting)\")\n",
+    "print(\"\u2022 Implement feature selection for clustering\")\n",
+    "print(\"\u2022 Add cluster explainability tools (e.g., SHAP for cluster assignment)\")\n",
+    "print(\"\u2022 Set up automated monitoring for cluster drift\")\n",
+    "print(\"\u2022 Build dashboards for real-time cluster insights\")\n"
+   ]
+  }
+ ],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}