Fix CyberForgeFeaturePipeline class definition

notebooks/02_feature_engineering.ipynb

@@ -455,7 +455,125 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "    def process_dataset(self, df: pd.DataFrame, url_column: str = 'url') -> pd.DataFrame:\n        \"\"\"Process a dataset and extract URL features\"\"\"\n        if url_column not in df.columns:\n            print(f\"  \u26a0 No '{url_column}' column found\")\n            return df\n        \n        try:\n            # Extract URL features\n            url_features = df[url_column].apply(lambda x: self.url_extractor.extract(x))\n            url_df = pd.DataFrame(url_features.tolist())\n            \n            # Drop non-numeric 'tld' column before renaming\n            if 'tld' in url_df.columns:\n                url_df = url_df.drop(columns=['tld'])\n            \n            # Rename columns with url_ prefix\n            url_df.columns = [f\"url_{c}\" for c in url_df.columns]\n            \n            # Combine with original features (drop original url column to avoid issues)\n            result_df = df.drop(columns=[url_column]).reset_index(drop=True)\n            result = pd.concat([result_df, url_df.reset_index(drop=True)], axis=1)\n            \n            return result\n        except Exception as e:\n            print(f\"  \u26a0 URL feature extraction error: {e}\")\n            return df\n    \n    def prepare_for_training(self, df: pd.DataFrame, label_column: str = 'label') -> tuple:\n        \"\"\"Prepare features for model training\"\"\"\n        df = df.copy()\n        \n        # Find label column (case insensitive, multiple names)\n        label_candidates = ['label', 'target', 'class', 'is_malicious', 'attack_type', \n                           'attack', 'category', 'malware', 'phishing', 'threat', 'type', 'y']\n        actual_label_col = None\n        for col in df.columns:\n            if col.lower() in [lc.lower() for lc in label_candidates]:\n                actual_label_col = col\n                break\n        \n        # Separate features and labels\n        if actual_label_col:\n            y = df[actual_label_col]\n            X = df.drop(columns=[actual_label_col])\n        else:\n            y = None\n            X = df\n        \n        # Select numeric columns only\n        numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()\n        bool_cols = X.select_dtypes(include=[bool]).columns.tolist()\n        \n        X_numeric = X[numeric_cols].fillna(0)\n        \n        # Convert boolean to int\n        for col in bool_cols:\n            if col in X.columns:\n                X_numeric[col] = X[col].astype(int)\n        \n        self.feature_names = X_numeric.columns.tolist()\n        \n        # Encode labels if present\n        if y is not None:\n            if y.dtype == 'object':\n                y = self.label_encoder.fit_transform(y)\n            else:\n                y = y.values\n        \n        return X_numeric, y\n"
+    "class CyberForgeFeaturePipeline:\n",
+    "    \"\"\"\n",
+    "    Unified feature extraction pipeline.\n",
+    "    Combines all extractors for complete feature engineering.\n",
+    "    \"\"\"\n",
+    "    \n",
+    "    def __init__(self):\n",
+    "        self.url_extractor = url_extractor\n",
+    "        self.network_extractor = network_extractor\n",
+    "        self.header_extractor = header_extractor\n",
+    "        self.js_extractor = js_extractor\n",
+    "        self.scaler = StandardScaler()\n",
+    "        self.label_encoder = LabelEncoder()\n",
+    "        self.feature_names = []\n",
+    "    \n",
+    "    def extract_website_features(self, scraped_data: Dict) -> Dict[str, Any]:\n",
+    "        \"\"\"Extract all features from website scraped data\"\"\"\n",
+    "        features = {}\n",
+    "        \n",
+    "        # URL features\n",
+    "        url_features = self.url_extractor.extract(scraped_data.get('url', ''))\n",
+    "        features.update({f\"url_{k}\": v for k, v in url_features.items() if k != 'tld'})\n",
+    "        \n",
+    "        # Network features\n",
+    "        network_features = self.network_extractor.extract_from_requests(\n",
+    "            scraped_data.get('network_requests', [])\n",
+    "        )\n",
+    "        features.update({f\"net_{k}\": v for k, v in network_features.items()})\n",
+    "        \n",
+    "        # Security header features\n",
+    "        header_features = self.header_extractor.extract(\n",
+    "            scraped_data.get('response_headers', {}),\n",
+    "            scraped_data.get('security_report', {})\n",
+    "        )\n",
+    "        features.update({f\"sec_{k}\": v for k, v in header_features.items()})\n",
+    "        \n",
+    "        # JavaScript features\n",
+    "        js_features = self.js_extractor.extract_from_console_logs(\n",
+    "            scraped_data.get('console_logs', [])\n",
+    "        )\n",
+    "        features.update({f\"js_{k}\": v for k, v in js_features.items()})\n",
+    "        \n",
+    "        # Calculate risk score\n",
+    "        features['security_score'] = self.header_extractor.calculate_security_score(header_features)\n",
+    "        \n",
+    "        return features\n",
+    "    \n",
+    "    def process_dataset(self, df: pd.DataFrame, url_column: str = 'url') -> pd.DataFrame:\n",
+    "        \"\"\"Process a dataset and extract URL features\"\"\"\n",
+    "        if url_column not in df.columns:\n",
+    "            print(f\"  Warning: No '{url_column}' column found\")\n",
+    "            return df\n",
+    "        \n",
+    "        try:\n",
+    "            # Extract URL features\n",
+    "            url_features = df[url_column].apply(lambda x: self.url_extractor.extract(x))\n",
+    "            url_df = pd.DataFrame(url_features.tolist())\n",
+    "            \n",
+    "            # Drop non-numeric 'tld' column before renaming\n",
+    "            if 'tld' in url_df.columns:\n",
+    "                url_df = url_df.drop(columns=['tld'])\n",
+    "            \n",
+    "            # Rename columns with url_ prefix\n",
+    "            url_df.columns = [f\"url_{c}\" for c in url_df.columns]\n",
+    "            \n",
+    "            # Combine with original features (drop original url column to avoid issues)\n",
+    "            result_df = df.drop(columns=[url_column]).reset_index(drop=True)\n",
+    "            result = pd.concat([result_df, url_df.reset_index(drop=True)], axis=1)\n",
+    "            \n",
+    "            return result\n",
+    "        except Exception as e:\n",
+    "            print(f\"  Warning: URL feature extraction error: {e}\")\n",
+    "            return df\n",
+    "    \n",
+    "    def prepare_for_training(self, df: pd.DataFrame, label_column: str = 'label') -> tuple:\n",
+    "        \"\"\"Prepare features for model training\"\"\"\n",
+    "        df = df.copy()\n",
+    "        \n",
+    "        # Find label column (case insensitive, multiple names)\n",
+    "        label_candidates = ['label', 'target', 'class', 'is_malicious', 'attack_type', \n",
+    "                           'attack', 'category', 'malware', 'phishing', 'threat', 'type', 'y']\n",
+    "        actual_label_col = None\n",
+    "        for col in df.columns:\n",
+    "            if col.lower() in [lc.lower() for lc in label_candidates]:\n",
+    "                actual_label_col = col\n",
+    "                break\n",
+    "        \n",
+    "        # Separate features and labels\n",
+    "        if actual_label_col:\n",
+    "            y = df[actual_label_col]\n",
+    "            X = df.drop(columns=[actual_label_col])\n",
+    "        else:\n",
+    "            y = None\n",
+    "            X = df\n",
+    "        \n",
+    "        # Select numeric columns only\n",
+    "        numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()\n",
+    "        bool_cols = X.select_dtypes(include=[bool]).columns.tolist()\n",
+    "        \n",
+    "        X_numeric = X[numeric_cols].fillna(0)\n",
+    "        \n",
+    "        # Convert boolean to int\n",
+    "        for col in bool_cols:\n",
+    "            if col in X.columns:\n",
+    "                X_numeric[col] = X[col].astype(int)\n",
+    "        \n",
+    "        self.feature_names = X_numeric.columns.tolist()\n",
+    "        \n",
+    "        # Encode labels if present\n",
+    "        if y is not None:\n",
+    "            if y.dtype == 'object':\n",
+    "                y = self.label_encoder.fit_transform(y)\n",
+    "            else:\n",
+    "                y = y.values\n",
+    "        \n",
+    "        return X_numeric, y\n",
+    "\n",
+    "pipeline = CyberForgeFeaturePipeline()\n",
+    "print(\"Feature Pipeline initialized\")\n"
    ]
   },
   {