Upload final_project.ipynb

final_project.ipynb

@@ -0,0 +1,687 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": 6,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "C36kdei0JAGU",
+        "outputId": "a3b9ca41-83ba-4246-ebd3-a88937443fd9"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.12/dist-packages/sklearn/feature_selection/_univariate_selection.py:111: UserWarning: Features [16] are constant.\n",
+            "  warnings.warn(\"Features %s are constant.\" % constant_features_idx, UserWarning)\n",
+            "/usr/local/lib/python3.12/dist-packages/sklearn/feature_selection/_univariate_selection.py:112: RuntimeWarning: invalid value encountered in divide\n",
+            "  f = msb / msw\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "New shape after feature selection: (110596, 50)\n",
+            "Epoch 1/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m13s\u001b[0m 5ms/step - accuracy: 0.9748 - loss: 0.0709 - val_accuracy: 0.7912 - val_loss: 0.7181\n",
+            "Epoch 2/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 3ms/step - accuracy: 0.9827 - loss: 0.0469 - val_accuracy: 0.7963 - val_loss: 0.8565\n",
+            "Epoch 3/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m6s\u001b[0m 3ms/step - accuracy: 0.9843 - loss: 0.0415 - val_accuracy: 0.7947 - val_loss: 0.9044\n",
+            "Epoch 4/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 3ms/step - accuracy: 0.9858 - loss: 0.0377 - val_accuracy: 0.7976 - val_loss: 0.8448\n",
+            "Epoch 5/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 3ms/step - accuracy: 0.9863 - loss: 0.0361 - val_accuracy: 0.8339 - val_loss: 0.8099\n",
+            "Epoch 6/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 3ms/step - accuracy: 0.9871 - loss: 0.0340 - val_accuracy: 0.8187 - val_loss: 0.8643\n",
+            "Epoch 7/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m6s\u001b[0m 3ms/step - accuracy: 0.9875 - loss: 0.0331 - val_accuracy: 0.8238 - val_loss: 0.9187\n",
+            "Epoch 8/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 3ms/step - accuracy: 0.9881 - loss: 0.0326 - val_accuracy: 0.8306 - val_loss: 0.8933\n",
+            "Epoch 9/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m6s\u001b[0m 3ms/step - accuracy: 0.9883 - loss: 0.0316 - val_accuracy: 0.8199 - val_loss: 0.8902\n",
+            "Epoch 10/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 3ms/step - accuracy: 0.9885 - loss: 0.0306 - val_accuracy: 0.8251 - val_loss: 0.9340\n",
+            "Epoch 11/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 3ms/step - accuracy: 0.9890 - loss: 0.0297 - val_accuracy: 0.8217 - val_loss: 1.0413\n",
+            "Epoch 12/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 3ms/step - accuracy: 0.9888 - loss: 0.0295 - val_accuracy: 0.7996 - val_loss: 1.2353\n",
+            "Epoch 13/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 3ms/step - accuracy: 0.9893 - loss: 0.0289 - val_accuracy: 0.8299 - val_loss: 1.0090\n",
+            "Epoch 14/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m6s\u001b[0m 3ms/step - accuracy: 0.9893 - loss: 0.0279 - val_accuracy: 0.8273 - val_loss: 0.8989\n",
+            "Epoch 15/15\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 3ms/step - accuracy: 0.9896 - loss: 0.0288 - val_accuracy: 0.8173 - val_loss: 1.1206\n",
+            "\u001b[1m705/705\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step - accuracy: 0.8173 - loss: 1.1206\n",
+            "Final Accuracy: 0.8173350095748901\n",
+            "\u001b[1m705/705\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step\n",
+            "\n",
+            "Classification Report:\n",
+            "\n",
+            "              precision    recall  f1-score   support\n",
+            "\n",
+            "           0       0.71      0.97      0.82      9711\n",
+            "           1       0.97      0.70      0.81     12833\n",
+            "\n",
+            "    accuracy                           0.82     22544\n",
+            "   macro avg       0.84      0.84      0.82     22544\n",
+            "weighted avg       0.86      0.82      0.82     22544\n",
+            "\n",
+            "\n",
+            "Confusion Matrix:\n",
+            "\n",
+            "[[9392  319]\n",
+            " [3799 9034]]\n"
+          ]
+        }
+      ],
+      "source": [
+        "# =========================\n",
+        "# 1. IMPORTS\n",
+        "# =========================\n",
+        "import pandas as pd\n",
+        "import numpy as np\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "from sklearn.preprocessing import StandardScaler\n",
+        "from sklearn.feature_selection import SelectKBest, f_classif\n",
+        "from sklearn.model_selection import train_test_split\n",
+        "from sklearn.metrics import classification_report, confusion_matrix\n",
+        "\n",
+        "# =========================\n",
+        "# 2. LOAD DATA\n",
+        "# =========================\n",
+        "train_path = \"KDDTrain+.txt\"\n",
+        "test_path = \"KDDTest+.txt\"\n",
+        "\n",
+        "columns = [\n",
+        "    \"duration\",\"protocol_type\",\"service\",\"flag\",\"src_bytes\",\"dst_bytes\",\"land\",\n",
+        "    \"wrong_fragment\",\"urgent\",\"hot\",\"num_failed_logins\",\"logged_in\",\n",
+        "    \"num_compromised\",\"root_shell\",\"su_attempted\",\"num_root\",\"num_file_creations\",\n",
+        "    \"num_shells\",\"num_access_files\",\"num_outbound_cmds\",\"is_host_login\",\n",
+        "    \"is_guest_login\",\"count\",\"srv_count\",\"serror_rate\",\"srv_serror_rate\",\n",
+        "    \"rerror_rate\",\"srv_rerror_rate\",\"same_srv_rate\",\"diff_srv_rate\",\n",
+        "    \"srv_diff_host_rate\",\"dst_host_count\",\"dst_host_srv_count\",\n",
+        "    \"dst_host_same_srv_rate\",\"dst_host_diff_srv_rate\",\n",
+        "    \"dst_host_same_src_port_rate\",\"dst_host_srv_diff_host_rate\",\n",
+        "    \"dst_host_serror_rate\",\"dst_host_srv_serror_rate\",\n",
+        "    \"dst_host_rerror_rate\",\"dst_host_srv_rerror_rate\",\n",
+        "    \"label\",\"difficulty\"\n",
+        "]\n",
+        "\n",
+        "train_df = pd.read_csv(train_path, names=columns)\n",
+        "test_df = pd.read_csv(test_path, names=columns)\n",
+        "\n",
+        "# =========================\n",
+        "# 3. LABEL CONVERSION\n",
+        "# =========================\n",
+        "def label_map(x):\n",
+        "    return 0 if x == \"normal\" else 1\n",
+        "\n",
+        "train_df['label'] = train_df['label'].apply(label_map)\n",
+        "test_df['label'] = test_df['label'].apply(label_map)\n",
+        "\n",
+        "# =========================\n",
+        "# 4. ONE-HOT ENCODING\n",
+        "# =========================\n",
+        "categorical_cols = ['protocol_type', 'service', 'flag']\n",
+        "\n",
+        "train_df = pd.get_dummies(train_df, columns=categorical_cols)\n",
+        "test_df = pd.get_dummies(test_df, columns=categorical_cols)\n",
+        "\n",
+        "train_df, test_df = train_df.align(test_df, join='left', axis=1, fill_value=0)\n",
+        "\n",
+        "# =========================\n",
+        "# 5. SPLIT FEATURES\n",
+        "# =========================\n",
+        "X_train = train_df.drop(['label', 'difficulty'], axis=1)\n",
+        "y_train = train_df['label']\n",
+        "\n",
+        "X_test = test_df.drop(['label', 'difficulty'], axis=1)\n",
+        "y_test = test_df['label']\n",
+        "\n",
+        "# =========================\n",
+        "# 6. NORMALIZATION\n",
+        "# =========================\n",
+        "scaler = StandardScaler()\n",
+        "X_train = scaler.fit_transform(X_train)\n",
+        "X_test = scaler.transform(X_test)\n",
+        "\n",
+        "# =========================\n",
+        "# 7. FEATURE SELECTION (IMPORTANT)\n",
+        "# =========================\n",
+        "selector = SelectKBest(score_func=f_classif, k=50)\n",
+        "\n",
+        "X_train = selector.fit_transform(X_train, y_train)\n",
+        "X_test = selector.transform(X_test)\n",
+        "\n",
+        "print(\"New shape after feature selection:\", X_train.shape)\n",
+        "\n",
+        "# =========================\n",
+        "# 8. BUILD MODEL (IMPROVED)\n",
+        "# =========================\n",
+        "model = tf.keras.Sequential([\n",
+        "    tf.keras.layers.Input(shape=(X_train.shape[1],)),\n",
+        "\n",
+        "    tf.keras.layers.Dense(128, activation='relu'),\n",
+        "    tf.keras.layers.BatchNormalization(),\n",
+        "    tf.keras.layers.Dropout(0.3),\n",
+        "\n",
+        "    tf.keras.layers.Dense(64, activation='relu'),\n",
+        "    tf.keras.layers.BatchNormalization(),\n",
+        "    tf.keras.layers.Dropout(0.3),\n",
+        "\n",
+        "    tf.keras.layers.Dense(32, activation='relu'),\n",
+        "\n",
+        "    tf.keras.layers.Dense(1, activation='sigmoid')\n",
+        "])\n",
+        "\n",
+        "model.compile(\n",
+        "    optimizer='adam',\n",
+        "    loss='binary_crossentropy',\n",
+        "    metrics=['accuracy']\n",
+        ")\n",
+        "\n",
+        "# =========================\n",
+        "# 9. TRAIN MODEL\n",
+        "# =========================\n",
+        "history = model.fit(\n",
+        "    X_train, y_train,\n",
+        "    epochs=15,\n",
+        "    batch_size=64,\n",
+        "    validation_data=(X_test, y_test)\n",
+        ")\n",
+        "\n",
+        "# =========================\n",
+        "# 10. EVALUATE\n",
+        "# =========================\n",
+        "loss, acc = model.evaluate(X_test, y_test)\n",
+        "print(\"Final Accuracy:\", acc)\n",
+        "\n",
+        "# =========================\n",
+        "# 11. METRICS (IMPORTANT FOR REPORT)\n",
+        "# =========================\n",
+        "y_pred = (model.predict(X_test) > 0.5).astype(\"int32\")\n",
+        "\n",
+        "print(\"\\nClassification Report:\\n\")\n",
+        "print(classification_report(y_test, y_pred))\n",
+        "\n",
+        "print(\"\\nConfusion Matrix:\\n\")\n",
+        "print(confusion_matrix(y_test, y_pred))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from tensorflow.keras import layers, models\n",
+        "\n",
+        "# Train ONLY on normal data\n",
+        "X_train_normal = X_train[y_train == 0]\n",
+        "\n",
+        "# Autoencoder model\n",
+        "input_dim = X_train.shape[1]\n",
+        "\n",
+        "autoencoder = models.Sequential([\n",
+        "    layers.Input(shape=(input_dim,)),\n",
+        "\n",
+        "    layers.Dense(64, activation='relu'),\n",
+        "    layers.Dense(32, activation='relu'),\n",
+        "    layers.Dense(16, activation='relu'),\n",
+        "\n",
+        "    layers.Dense(32, activation='relu'),\n",
+        "    layers.Dense(64, activation='relu'),\n",
+        "\n",
+        "    layers.Dense(input_dim, activation='sigmoid')\n",
+        "])\n",
+        "\n",
+        "autoencoder.compile(optimizer='adam', loss='mse')\n",
+        "\n",
+        "# Train\n",
+        "autoencoder.fit(\n",
+        "    X_train_normal,\n",
+        "    X_train_normal,\n",
+        "    epochs=15,\n",
+        "    batch_size=64,\n",
+        "    validation_data=(X_test, X_test)\n",
+        ")"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "WUgrtCu68UgM",
+        "outputId": "8113c359-5ebc-4f8e-e870-32f4bd79c5e9"
+      },
+      "execution_count": 7,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Epoch 1/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m7s\u001b[0m 5ms/step - loss: 0.2818 - val_loss: 0.6870\n",
+            "Epoch 2/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 4ms/step - loss: 0.2517 - val_loss: 0.6827\n",
+            "Epoch 3/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3ms/step - loss: 0.2508 - val_loss: 0.6792\n",
+            "Epoch 4/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3ms/step - loss: 0.2506 - val_loss: 0.6784\n",
+            "Epoch 5/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3ms/step - loss: 0.2500 - val_loss: 0.6769\n",
+            "Epoch 6/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 4ms/step - loss: 0.2499 - val_loss: 0.6767\n",
+            "Epoch 7/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3ms/step - loss: 0.2499 - val_loss: 0.6818\n",
+            "Epoch 8/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3ms/step - loss: 0.2498 - val_loss: 0.6743\n",
+            "Epoch 9/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3ms/step - loss: 0.2498 - val_loss: 0.6778\n",
+            "Epoch 10/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3ms/step - loss: 0.2497 - val_loss: 0.6721\n",
+            "Epoch 11/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3ms/step - loss: 0.2497 - val_loss: 0.6738\n",
+            "Epoch 12/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3ms/step - loss: 0.2497 - val_loss: 0.6773\n",
+            "Epoch 13/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3ms/step - loss: 0.2497 - val_loss: 0.6788\n",
+            "Epoch 14/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3ms/step - loss: 0.2496 - val_loss: 0.6767\n",
+            "Epoch 15/15\n",
+            "\u001b[1m924/924\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3ms/step - loss: 0.2497 - val_loss: 0.6770\n"
+          ]
+        },
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "<keras.src.callbacks.history.History at 0x79e9b6ce9610>"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 7
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Reconstruction error\n",
+        "reconstructions = autoencoder.predict(X_test)\n",
+        "\n",
+        "mse = np.mean(np.power(X_test - reconstructions, 2), axis=1)\n",
+        "\n",
+        "# Threshold\n",
+        "# Get reconstruction error for NORMAL training data\n",
+        "train_recon = autoencoder.predict(X_train_normal)\n",
+        "\n",
+        "train_mse = np.mean(np.power(X_train_normal - train_recon, 2), axis=1)\n",
+        "\n",
+        "# Better threshold\n",
+        "threshold = np.percentile(train_mse, 95)\n",
+        "\n",
+        "# Predictions\n",
+        "y_pred_ae = (mse > threshold).astype(int)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "2mTcym9l8Wd7",
+        "outputId": "11596bb4-da65-41ba-9d72-1f61112f2b76"
+      },
+      "execution_count": 8,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "\u001b[1m705/705\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step\n",
+            "\u001b[1m1847/1847\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 2ms/step\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from sklearn.metrics import classification_report\n",
+        "\n",
+        "print(classification_report(y_test, y_pred_ae))"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "5QkUi8bSK7XH",
+        "outputId": "a4cb6f1a-78ed-4a63-da1c-57aee27d0b46"
+      },
+      "execution_count": 9,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "              precision    recall  f1-score   support\n",
+            "\n",
+            "           0       0.64      0.93      0.75      9711\n",
+            "           1       0.91      0.60      0.72     12833\n",
+            "\n",
+            "    accuracy                           0.74     22544\n",
+            "   macro avg       0.78      0.76      0.74     22544\n",
+            "weighted avg       0.79      0.74      0.74     22544\n",
+            "\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from tensorflow.keras import layers, models\n",
+        "\n",
+        "# Reshape data for LSTM\n",
+        "X_train_lstm = X_train.reshape((X_train.shape[0], 1, X_train.shape[1]))\n",
+        "X_test_lstm = X_test.reshape((X_test.shape[0], 1, X_test.shape[1]))\n",
+        "\n",
+        "# Build LSTM model\n",
+        "model_lstm = models.Sequential([\n",
+        "    layers.LSTM(64, input_shape=(1, X_train.shape[1])),\n",
+        "    layers.Dropout(0.3),\n",
+        "\n",
+        "    layers.Dense(32, activation='relu'),\n",
+        "    layers.Dense(1, activation='sigmoid')\n",
+        "])\n",
+        "\n",
+        "model_lstm.compile(\n",
+        "    optimizer='adam',\n",
+        "    loss='binary_crossentropy',\n",
+        "    metrics=['accuracy']\n",
+        ")\n",
+        "\n",
+        "# Train\n",
+        "model_lstm.fit(\n",
+        "    X_train_lstm, y_train,\n",
+        "    epochs=10,\n",
+        "    batch_size=64,\n",
+        "    validation_data=(X_test_lstm, y_test)\n",
+        ")\n",
+        "\n",
+        "# Evaluate\n",
+        "loss, acc = model_lstm.evaluate(X_test_lstm, y_test)\n",
+        "print(\"LSTM Accuracy:\", acc)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "-WQDtVbqLlPK",
+        "outputId": "906b349f-f0d0-40f8-8b9b-0bbdf0d548d6"
+      },
+      "execution_count": 10,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Epoch 1/10\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.12/dist-packages/keras/src/layers/rnn/rnn.py:199: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
+            "  super().__init__(**kwargs)\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 6ms/step - accuracy: 0.9768 - loss: 0.0723 - val_accuracy: 0.7811 - val_loss: 0.9115\n",
+            "Epoch 2/10\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m9s\u001b[0m 5ms/step - accuracy: 0.9841 - loss: 0.0417 - val_accuracy: 0.7835 - val_loss: 0.9094\n",
+            "Epoch 3/10\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 6ms/step - accuracy: 0.9858 - loss: 0.0375 - val_accuracy: 0.7972 - val_loss: 0.9576\n",
+            "Epoch 4/10\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 6ms/step - accuracy: 0.9873 - loss: 0.0339 - val_accuracy: 0.7971 - val_loss: 1.0328\n",
+            "Epoch 5/10\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 6ms/step - accuracy: 0.9882 - loss: 0.0317 - val_accuracy: 0.8055 - val_loss: 1.0339\n",
+            "Epoch 6/10\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 6ms/step - accuracy: 0.9887 - loss: 0.0302 - val_accuracy: 0.8074 - val_loss: 1.1582\n",
+            "Epoch 7/10\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 6ms/step - accuracy: 0.9893 - loss: 0.0286 - val_accuracy: 0.8054 - val_loss: 1.2248\n",
+            "Epoch 8/10\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 6ms/step - accuracy: 0.9897 - loss: 0.0279 - val_accuracy: 0.8148 - val_loss: 1.1761\n",
+            "Epoch 9/10\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 6ms/step - accuracy: 0.9905 - loss: 0.0266 - val_accuracy: 0.8226 - val_loss: 1.1026\n",
+            "Epoch 10/10\n",
+            "\u001b[1m1729/1729\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 6ms/step - accuracy: 0.9904 - loss: 0.0257 - val_accuracy: 0.8242 - val_loss: 1.1574\n",
+            "\u001b[1m705/705\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 3ms/step - accuracy: 0.8242 - loss: 1.1574\n",
+            "LSTM Accuracy: 0.8241660594940186\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# =========================\n",
+        "# 1. INSTALL + IMPORT\n",
+        "# =========================\n",
+        "!pip install pyspark\n",
+        "\n",
+        "from pyspark.sql import SparkSession\n",
+        "from pyspark.sql.functions import when\n",
+        "\n",
+        "# =========================\n",
+        "# 2. START SPARK\n",
+        "# =========================\n",
+        "spark = SparkSession.builder \\\n",
+        "    .appName(\"IDS_Project\") \\\n",
+        "    .getOrCreate()\n",
+        "\n",
+        "print(\"Spark Started ✅\")\n",
+        "\n",
+        "# =========================\n",
+        "# 3. LOAD DATA\n",
+        "# =========================\n",
+        "spark_df = spark.read.csv(\n",
+        "    \"KDDTrain+.txt\",\n",
+        "    header=False,\n",
+        "    inferSchema=True\n",
+        ")\n",
+        "\n",
+        "# =========================\n",
+        "# 4. ADD COLUMN NAMES\n",
+        "# =========================\n",
+        "spark_df = spark_df.toDF(*columns)\n",
+        "\n",
+        "print(\"Columns assigned ✅\")\n",
+        "\n",
+        "# =========================\n",
+        "# 5. BASIC CHECK\n",
+        "# =========================\n",
+        "spark_df.show(5)\n",
+        "\n",
+        "# =========================\n",
+        "# 6. DISTRIBUTED LABEL CONVERSION\n",
+        "# =========================\n",
+        "spark_df = spark_df.withColumn(\n",
+        "    \"label\",\n",
+        "    when(spark_df[\"label\"] == \"normal\", 0).otherwise(1)\n",
+        ")\n",
+        "\n",
+        "print(\"Label converted ✅\")\n",
+        "\n",
+        "spark_df.groupBy(\"label\").count().show()\n",
+        "\n",
+        "# =========================\n",
+        "# 7. DISTRIBUTED FEATURE ENGINEERING\n",
+        "# =========================\n",
+        "spark_df = spark_df.withColumn(\n",
+        "    \"bytes_total\",\n",
+        "    spark_df[\"src_bytes\"] + spark_df[\"dst_bytes\"]\n",
+        ")\n",
+        "\n",
+        "spark_df.select(\"src_bytes\", \"dst_bytes\", \"bytes_total\").show(5)\n",
+        "\n",
+        "# =========================\n",
+        "# 8. DISTRIBUTED FILTERING\n",
+        "# =========================\n",
+        "normal_df = spark_df.filter(spark_df[\"label\"] == 0)\n",
+        "attack_df = spark_df.filter(spark_df[\"label\"] == 1)\n",
+        "\n",
+        "print(\"Normal count:\", normal_df.count())\n",
+        "print(\"Attack count:\", attack_df.count())\n",
+        "\n",
+        "# =========================\n",
+        "# 9. SHOW DISTRIBUTION\n",
+        "# =========================\n",
+        "spark_df.groupBy(\"protocol_type\").count().show()\n",
+        "\n",
+        "print(\"PySpark processing complete ✅\")"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "IqP3MdTLQpTU",
+        "outputId": "70a25dd3-3e6c-42e1-d841-e560231955f0"
+      },
+      "execution_count": 11,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Requirement already satisfied: pyspark in /usr/local/lib/python3.12/dist-packages (4.0.2)\n",
+            "Requirement already satisfied: py4j<0.10.9.10,>=0.10.9.7 in /usr/local/lib/python3.12/dist-packages (from pyspark) (0.10.9.9)\n",
+            "Spark Started ✅\n",
+            "Columns assigned ✅\n",
+            "+--------+-------------+--------+----+---------+---------+----+--------------+------+---+-----------------+---------+---------------+----------+------------+--------+------------------+----------+----------------+-----------------+-------------+--------------+-----+---------+-----------+---------------+-----------+---------------+-------------+-------------+------------------+--------------+------------------+----------------------+----------------------+---------------------------+---------------------------+--------------------+------------------------+--------------------+------------------------+-------+----------+\n",
+            "|duration|protocol_type| service|flag|src_bytes|dst_bytes|land|wrong_fragment|urgent|hot|num_failed_logins|logged_in|num_compromised|root_shell|su_attempted|num_root|num_file_creations|num_shells|num_access_files|num_outbound_cmds|is_host_login|is_guest_login|count|srv_count|serror_rate|srv_serror_rate|rerror_rate|srv_rerror_rate|same_srv_rate|diff_srv_rate|srv_diff_host_rate|dst_host_count|dst_host_srv_count|dst_host_same_srv_rate|dst_host_diff_srv_rate|dst_host_same_src_port_rate|dst_host_srv_diff_host_rate|dst_host_serror_rate|dst_host_srv_serror_rate|dst_host_rerror_rate|dst_host_srv_rerror_rate|  label|difficulty|\n",
+            "+--------+-------------+--------+----+---------+---------+----+--------------+------+---+-----------------+---------+---------------+----------+------------+--------+------------------+----------+----------------+-----------------+-------------+--------------+-----+---------+-----------+---------------+-----------+---------------+-------------+-------------+------------------+--------------+------------------+----------------------+----------------------+---------------------------+---------------------------+--------------------+------------------------+--------------------+------------------------+-------+----------+\n",
+            "|       0|          tcp|ftp_data|  SF|      491|        0|   0|             0|     0|  0|                0|        0|              0|         0|           0|       0|                 0|         0|               0|                0|            0|             0|    2|        2|        0.0|            0.0|        0.0|            0.0|          1.0|          0.0|               0.0|           150|                25|                  0.17|                  0.03|                       0.17|                        0.0|                 0.0|                     0.0|                0.05|                     0.0| normal|        20|\n",
+            "|       0|          udp|   other|  SF|      146|        0|   0|             0|     0|  0|                0|        0|              0|         0|           0|       0|                 0|         0|               0|                0|            0|             0|   13|        1|        0.0|            0.0|        0.0|            0.0|         0.08|         0.15|               0.0|           255|                 1|                   0.0|                   0.6|                       0.88|                        0.0|                 0.0|                     0.0|                 0.0|                     0.0| normal|        15|\n",
+            "|       0|          tcp| private|  S0|        0|        0|   0|             0|     0|  0|                0|        0|              0|         0|           0|       0|                 0|         0|               0|                0|            0|             0|  123|        6|        1.0|            1.0|        0.0|            0.0|         0.05|         0.07|               0.0|           255|                26|                   0.1|                  0.05|                        0.0|                        0.0|                 1.0|                     1.0|                 0.0|                     0.0|neptune|        19|\n",
+            "|       0|          tcp|    http|  SF|      232|     8153|   0|             0|     0|  0|                0|        1|              0|         0|           0|       0|                 0|         0|               0|                0|            0|             0|    5|        5|        0.2|            0.2|        0.0|            0.0|          1.0|          0.0|               0.0|            30|               255|                   1.0|                   0.0|                       0.03|                       0.04|                0.03|                    0.01|                 0.0|                    0.01| normal|        21|\n",
+            "|       0|          tcp|    http|  SF|      199|      420|   0|             0|     0|  0|                0|        1|              0|         0|           0|       0|                 0|         0|               0|                0|            0|             0|   30|       32|        0.0|            0.0|        0.0|            0.0|          1.0|          0.0|              0.09|           255|               255|                   1.0|                   0.0|                        0.0|                        0.0|                 0.0|                     0.0|                 0.0|                     0.0| normal|        21|\n",
+            "+--------+-------------+--------+----+---------+---------+----+--------------+------+---+-----------------+---------+---------------+----------+------------+--------+------------------+----------+----------------+-----------------+-------------+--------------+-----+---------+-----------+---------------+-----------+---------------+-------------+-------------+------------------+--------------+------------------+----------------------+----------------------+---------------------------+---------------------------+--------------------+------------------------+--------------------+------------------------+-------+----------+\n",
+            "only showing top 5 rows\n",
+            "Label converted ✅\n",
+            "+-----+-----+\n",
+            "|label|count|\n",
+            "+-----+-----+\n",
+            "|    1|58630|\n",
+            "|    0|67343|\n",
+            "+-----+-----+\n",
+            "\n",
+            "+---------+---------+-----------+\n",
+            "|src_bytes|dst_bytes|bytes_total|\n",
+            "+---------+---------+-----------+\n",
+            "|      491|        0|        491|\n",
+            "|      146|        0|        146|\n",
+            "|        0|        0|          0|\n",
+            "|      232|     8153|       8385|\n",
+            "|      199|      420|        619|\n",
+            "+---------+---------+-----------+\n",
+            "only showing top 5 rows\n",
+            "Normal count: 67343\n",
+            "Attack count: 58630\n",
+            "+-------------+------+\n",
+            "|protocol_type| count|\n",
+            "+-------------+------+\n",
+            "|          tcp|102689|\n",
+            "|          udp| 14993|\n",
+            "|         icmp|  8291|\n",
+            "+-------------+------+\n",
+            "\n",
+            "PySpark processing complete ✅\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# =========================\n",
+        "# FINAL SECURE API (SIMULATION)\n",
+        "# =========================\n",
+        "\n",
+        "API_KEY = \"12345\"\n",
+        "\n",
+        "def secure_predict(input_data, api_key):\n",
+        "\n",
+        "    # 🔐 Security check\n",
+        "    if api_key != API_KEY:\n",
+        "        return {\"error\": \"Unauthorized access\"}\n",
+        "\n",
+        "    # Convert input\n",
+        "    data = np.array(input_data).reshape(1, -1)\n",
+        "\n",
+        "    # Model prediction\n",
+        "    prediction = model.predict(data)\n",
+        "    result = int(prediction[0][0] > 0.5)\n",
+        "\n",
+        "    return {\n",
+        "        \"prediction\": result,\n",
+        "        \"message\": \"Attack\" if result == 1 else \"Normal\"\n",
+        "    }"
+      ],
+      "metadata": {
+        "id": "Lfe5tGxj6njn"
+      },
+      "execution_count": 12,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "sample = X_test[0]\n",
+        "\n",
+        "output = secure_predict(sample, \"12345\")\n",
+        "\n",
+        "print(output)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "gmfxYPEa7lyg",
+        "outputId": "ed3a28fc-b094-432b-b84b-60704d7f41b1"
+      },
+      "execution_count": 13,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 267ms/step\n",
+            "{'prediction': 1, 'message': 'Attack'}\n"
+          ]
+        }
+      ]
+    }
+  ]
+}