{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e7144afc-a75f-4273-934d-0fa7c3c1d511",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pima shape:       (768, 9)\n",
      "Frankfurt shape:  (2000, 9)\n",
      "\n",
      "Pima columns:      ['Pregnancies', 'Glucose', 'BloodPressure', 'SkinThickness', 'Insulin', 'BMI', 'DiabetesPedigreeFunction', 'Age', 'Outcome']\n",
      "Frankfurt columns: ['Pregnancies', 'Glucose', 'BloodPressure', 'SkinThickness', 'Insulin', 'BMI', 'DiabetesPedigreeFunction', 'Age', 'Outcome']\n",
      "\n",
      "Merged shape: (2768, 9)\n",
      "\n",
      "Missing values after fixing zeros:\n",
      "Pregnancies                    0\n",
      "Glucose                       18\n",
      "BloodPressure                125\n",
      "SkinThickness                800\n",
      "Insulin                     1330\n",
      "BMI                           39\n",
      "DiabetesPedigreeFunction       0\n",
      "Age                            0\n",
      "Outcome                        0\n",
      "dtype: int64\n",
      "\n",
      "Class distribution:\n",
      "Outcome\n",
      "0    1816\n",
      "1     952\n",
      "Name: count, dtype: int64\n",
      "Outcome\n",
      "0    65.6\n",
      "1    34.4\n",
      "Name: proportion, dtype: float64\n",
      "\n",
      "Saved to: C:\\Users\\abdul\\Amr senior\\dataset\\diabetes_merged.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\abdul\\AppData\\Local\\Temp\\ipykernel_26020\\6553557.py:31: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.\n",
      "The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.\n",
      "\n",
      "For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.\n",
      "\n",
      "\n",
      "  df[col].fillna(df[col].median(), inplace=True)\n",
      "C:\\Users\\abdul\\AppData\\Local\\Temp\\ipykernel_26020\\6553557.py:31: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.\n",
      "The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.\n",
      "\n",
      "For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.\n",
      "\n",
      "\n",
      "  df[col].fillna(df[col].median(), inplace=True)\n",
      "C:\\Users\\abdul\\AppData\\Local\\Temp\\ipykernel_26020\\6553557.py:31: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.\n",
      "The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.\n",
      "\n",
      "For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.\n",
      "\n",
      "\n",
      "  df[col].fillna(df[col].median(), inplace=True)\n",
      "C:\\Users\\abdul\\AppData\\Local\\Temp\\ipykernel_26020\\6553557.py:31: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.\n",
      "The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.\n",
      "\n",
      "For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.\n",
      "\n",
      "\n",
      "  df[col].fillna(df[col].median(), inplace=True)\n",
      "C:\\Users\\abdul\\AppData\\Local\\Temp\\ipykernel_26020\\6553557.py:31: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.\n",
      "The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.\n",
      "\n",
      "For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.\n",
      "\n",
      "\n",
      "  df[col].fillna(df[col].median(), inplace=True)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# ── Load both datasets ──────────────────────────────────────────\n",
    "pima = pd.read_csv(r\"C:\\Users\\abdul\\Amr senior\\dataset\\archive (8)\\diabetes.csv\")\n",
    "frankfurt = pd.read_csv(r\"C:\\Users\\abdul\\Amr senior\\dataset\\archive (7)\\diabetes.csv\")\n",
    "\n",
    "print(\"Pima shape:      \", pima.shape)\n",
    "print(\"Frankfurt shape: \", frankfurt.shape)\n",
    "\n",
    "# ── Align columns (in case of any naming differences) ───────────\n",
    "print(\"\\nPima columns:     \", pima.columns.tolist())\n",
    "print(\"Frankfurt columns:\", frankfurt.columns.tolist())\n",
    "\n",
    "# Rename to match if needed (usually identical)\n",
    "frankfurt.columns = pima.columns\n",
    "\n",
    "# ── Merge ────────────────────────────────────────────────────────\n",
    "df = pd.concat([pima, frankfurt], ignore_index=True)\n",
    "print(f\"\\nMerged shape: {df.shape}\")  # Expected: (2768, 9)\n",
    "\n",
    "# ── Fix zero values (medically impossible zeros → NaN) ──────────\n",
    "zero_invalid_cols = ['Glucose', 'BloodPressure', 'SkinThickness', 'Insulin', 'BMI']\n",
    "df[zero_invalid_cols] = df[zero_invalid_cols].replace(0, np.nan)\n",
    "\n",
    "print(\"\\nMissing values after fixing zeros:\")\n",
    "print(df.isnull().sum())\n",
    "\n",
    "# ── Impute with median ───────────────────────────────────────────\n",
    "for col in zero_invalid_cols:\n",
    "    df[col].fillna(df[col].median(), inplace=True)\n",
    "\n",
    "# ── Check class balance ──────────────────────────────────────────\n",
    "print(\"\\nClass distribution:\")\n",
    "print(df['Outcome'].value_counts())\n",
    "print(df['Outcome'].value_counts(normalize=True).mul(100).round(1))\n",
    "\n",
    "# ── Save merged dataset ──────────────────────────────────────────\n",
    "save_path = r\"C:\\Users\\abdul\\Amr senior\\dataset\\diabetes_merged.csv\"\n",
    "df.to_csv(save_path, index=False)\n",
    "print(f\"\\nSaved to: {save_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9bccf5b9-7f44-4507-84ae-2a9ca5eec9f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✅ Missing after imputation:\n",
      "0 nulls remaining\n",
      "\n",
      "✅ Features after engineering: 12 features\n",
      "\n",
      "✅ Class distribution after SMOTE:\n",
      "Outcome\n",
      "0    1453\n",
      "1    1453\n",
      "Name: count, dtype: int64\n",
      "\n",
      "✅ All files saved and ready for training!\n",
      "   Train: (2906, 12) | Test: (554, 12)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "df = pd.read_csv(r\"C:\\Users\\abdul\\Amr senior\\dataset\\diabetes_merged.csv\")\n",
    "\n",
    "# ── Step 1: Smart Imputation (KNN-style iterative) ───────────────\n",
    "# Much better than median for high-missing columns like Insulin\n",
    "imputer = IterativeImputer(max_iter=10, random_state=42)\n",
    "cols_to_impute = ['Glucose','BloodPressure','SkinThickness','Insulin','BMI']\n",
    "\n",
    "df[cols_to_impute] = imputer.fit_transform(df[cols_to_impute])\n",
    "\n",
    "# Clip negative values that imputer might produce\n",
    "df[cols_to_impute] = df[cols_to_impute].clip(lower=0)\n",
    "\n",
    "print(\"✅ Missing after imputation:\")\n",
    "print(df.isnull().sum().sum(), \"nulls remaining\")\n",
    "\n",
    "# ── Step 2: Feature Engineering ──────────────────────────────────\n",
    "df['Glucose_BMI']    = df['Glucose'] * df['BMI']\n",
    "df['Age_BMI']        = df['Age'] * df['BMI']\n",
    "df['Glucose_squared']= df['Glucose'] ** 2\n",
    "df['Insulin_BMI']    = df['Insulin'] * df['BMI']\n",
    "\n",
    "print(\"\\n✅ Features after engineering:\", df.shape[1] - 1, \"features\")\n",
    "\n",
    "# ── Step 3: Split BEFORE scaling/SMOTE (prevent data leakage) ────\n",
    "X = df.drop('Outcome', axis=1)\n",
    "y = df['Outcome']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.2, random_state=42, stratify=y\n",
    ")\n",
    "\n",
    "# ── Step 4: Normalize ─────────────────────────────────────────────\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)   # fit only on train\n",
    "X_test_scaled  = scaler.transform(X_test)         # transform test\n",
    "\n",
    "# ── Step 5: SMOTE on training set only ───────────────────────────\n",
    "smote = SMOTE(random_state=42)\n",
    "X_train_bal, y_train_bal = smote.fit_resample(X_train_scaled, y_train)\n",
    "\n",
    "print(\"\\n✅ Class distribution after SMOTE:\")\n",
    "print(pd.Series(y_train_bal).value_counts())\n",
    "\n",
    "# ── Step 6: Save processed splits ────────────────────────────────\n",
    "base = r\"C:\\Users\\abdul\\Amr senior\\dataset\"\n",
    "np.save(f\"{base}\\\\X_train.npy\", X_train_bal)\n",
    "np.save(f\"{base}\\\\X_test.npy\",  X_test_scaled)\n",
    "np.save(f\"{base}\\\\y_train.npy\", y_train_bal)\n",
    "np.save(f\"{base}\\\\y_test.npy\",  y_test.values)\n",
    "import joblib\n",
    "joblib.dump(scaler, f\"{base}\\\\scaler.pkl\")\n",
    "\n",
    "print(\"\\n✅ All files saved and ready for training!\")\n",
    "print(f\"   Train: {X_train_bal.shape} | Test: {X_test_scaled.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8eb333a4-afab-4aae-b6f5-2c9e11cdc69f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa396b1d-2a87-4a4e-af76-d109e20f067c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}