Fix URL feature extraction column mismatch

notebooks/02_feature_engineering.ipynb

@@ -52,8 +52,8 @@
     "FEATURES_DIR = DATASETS_DIR / \"features\"\n",
     "FEATURES_DIR.mkdir(exist_ok=True)\n",
     "\n",
-    "print(f\"✓ Configuration loaded\")\n",
-    "print(f\"✓ Features output: {FEATURES_DIR}\")"
+    "print(f\"\u2713 Configuration loaded\")\n",
+    "print(f\"\u2713 Features output: {FEATURES_DIR}\")"
    ]
   },
   {
@@ -165,7 +165,7 @@
     "        return pd.DataFrame(features)\n",
     "\n",
     "url_extractor = URLFeatureExtractor()\n",
-    "print(\"✓ URL Feature Extractor initialized\")\n",
+    "print(\"\u2713 URL Feature Extractor initialized\")\n",
     "\n",
     "# Test\n",
     "test_features = url_extractor.extract(\"https://suspicious-login.example.com/verify?id=123\")\n",
@@ -265,7 +265,7 @@
     "        }\n",
     "\n",
     "network_extractor = NetworkFeatureExtractor()\n",
-    "print(\"✓ Network Feature Extractor initialized\")"
+    "print(\"\u2713 Network Feature Extractor initialized\")"
    ]
   },
   {
@@ -348,7 +348,7 @@
     "        return min(100, max(0, score))\n",
     "\n",
     "header_extractor = SecurityHeaderExtractor()\n",
-    "print(\"✓ Security Header Extractor initialized\")"
+    "print(\"\u2713 Security Header Extractor initialized\")"
    ]
   },
   {
@@ -437,7 +437,7 @@
     "        }\n",
     "\n",
     "js_extractor = JavaScriptFeatureExtractor()\n",
-    "print(\"✓ JavaScript Feature Extractor initialized\")"
+    "print(\"\u2713 JavaScript Feature Extractor initialized\")"
    ]
   },
   {
@@ -455,103 +455,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "class CyberForgeFeaturePipeline:\n",
-    "    \"\"\"\n",
-    "    Unified feature extraction pipeline for CyberForge AI.\n",
-    "    Combines all extractors for comprehensive security feature engineering.\n",
-    "    \"\"\"\n",
-    "    \n",
-    "    def __init__(self):\n",
-    "        self.url_extractor = URLFeatureExtractor()\n",
-    "        self.network_extractor = NetworkFeatureExtractor()\n",
-    "        self.header_extractor = SecurityHeaderExtractor()\n",
-    "        self.js_extractor = JavaScriptFeatureExtractor()\n",
-    "        self.scaler = StandardScaler()\n",
-    "        self.label_encoder = LabelEncoder()\n",
-    "        self.feature_names = []\n",
-    "    \n",
-    "    def extract_website_features(self, scraped_data: Dict) -> Dict[str, Any]:\n",
-    "        \"\"\"Extract all features from website scraped data\"\"\"\n",
-    "        features = {}\n",
-    "        \n",
-    "        # URL features\n",
-    "        url_features = self.url_extractor.extract(scraped_data.get('url', ''))\n",
-    "        features.update({f\"url_{k}\": v for k, v in url_features.items() if k != 'tld'})\n",
-    "        \n",
-    "        # Network features\n",
-    "        network_features = self.network_extractor.extract_from_requests(\n",
-    "            scraped_data.get('network_requests', [])\n",
-    "        )\n",
-    "        features.update({f\"net_{k}\": v for k, v in network_features.items()})\n",
-    "        \n",
-    "        # Security header features\n",
-    "        header_features = self.header_extractor.extract(\n",
-    "            scraped_data.get('response_headers', {}),\n",
-    "            scraped_data.get('security_report', {})\n",
-    "        )\n",
-    "        features.update({f\"sec_{k}\": v for k, v in header_features.items()})\n",
-    "        \n",
-    "        # JavaScript features\n",
-    "        js_features = self.js_extractor.extract_from_console_logs(\n",
-    "            scraped_data.get('console_logs', [])\n",
-    "        )\n",
-    "        features.update({f\"js_{k}\": v for k, v in js_features.items()})\n",
-    "        \n",
-    "        # Calculate risk score\n",
-    "        features['security_score'] = self.header_extractor.calculate_security_score(header_features)\n",
-    "        \n",
-    "        return features\n",
-    "    \n",
-    "    def process_dataset(self, df: pd.DataFrame, url_column: str = 'url') -> pd.DataFrame:\n",
-    "        \"\"\"Process a dataset and extract URL features\"\"\"\n",
-    "        if url_column not in df.columns:\n",
-    "            print(f\"  ⚠ No '{url_column}' column found\")\n",
-    "            return df\n",
-    "        \n",
-    "        # Extract URL features\n",
-    "        url_features = df[url_column].apply(lambda x: self.url_extractor.extract(x))\n",
-    "        url_df = pd.DataFrame(url_features.tolist())\n",
-    "        url_df.columns = [f\"url_{c}\" for c in url_df.columns if c != 'tld']\n",
-    "        \n",
-    "        # Combine with original features\n",
-    "        result = pd.concat([df.reset_index(drop=True), url_df.reset_index(drop=True)], axis=1)\n",
-    "        \n",
-    "        return result\n",
-    "    \n",
-    "    def prepare_for_training(self, df: pd.DataFrame, label_column: str = 'label') -> tuple:\n",
-    "        \"\"\"Prepare features for model training\"\"\"\n",
-    "        df = df.copy()\n",
-    "        \n",
-    "        # Separate features and labels\n",
-    "        if label_column in df.columns:\n",
-    "            y = df[label_column]\n",
-    "            X = df.drop(columns=[label_column])\n",
-    "        else:\n",
-    "            y = None\n",
-    "            X = df\n",
-    "        \n",
-    "        # Select numeric columns only\n",
-    "        numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()\n",
-    "        X_numeric = X[numeric_cols].fillna(0)\n",
-    "        \n",
-    "        # Convert boolean to int\n",
-    "        bool_cols = X.select_dtypes(include=[bool]).columns.tolist()\n",
-    "        for col in bool_cols:\n",
-    "            X_numeric[col] = X[col].astype(int)\n",
-    "        \n",
-    "        self.feature_names = X_numeric.columns.tolist()\n",
-    "        \n",
-    "        # Encode labels if present\n",
-    "        if y is not None:\n",
-    "            if y.dtype == 'object':\n",
-    "                y = self.label_encoder.fit_transform(y)\n",
-    "            else:\n",
-    "                y = y.values\n",
-    "        \n",
-    "        return X_numeric, y\n",
-    "\n",
-    "pipeline = CyberForgeFeaturePipeline()\n",
-    "print(\"✓ Feature Pipeline initialized\")"
+    "    def process_dataset(self, df: pd.DataFrame, url_column: str = 'url') -> pd.DataFrame:\n        \"\"\"Process a dataset and extract URL features\"\"\"\n        if url_column not in df.columns:\n            print(f\"  \u26a0 No '{url_column}' column found\")\n            return df\n        \n        try:\n            # Extract URL features\n            url_features = df[url_column].apply(lambda x: self.url_extractor.extract(x))\n            url_df = pd.DataFrame(url_features.tolist())\n            \n            # Drop non-numeric 'tld' column before renaming\n            if 'tld' in url_df.columns:\n                url_df = url_df.drop(columns=['tld'])\n            \n            # Rename columns with url_ prefix\n            url_df.columns = [f\"url_{c}\" for c in url_df.columns]\n            \n            # Combine with original features (drop original url column to avoid issues)\n            result_df = df.drop(columns=[url_column]).reset_index(drop=True)\n            result = pd.concat([result_df, url_df.reset_index(drop=True)], axis=1)\n            \n            return result\n        except Exception as e:\n            print(f\"  \u26a0 URL feature extraction error: {e}\")\n            return df\n    \n    def prepare_for_training(self, df: pd.DataFrame, label_column: str = 'label') -> tuple:\n        \"\"\"Prepare features for model training\"\"\"\n        df = df.copy()\n        \n        # Find label column (case insensitive, multiple names)\n        label_candidates = ['label', 'target', 'class', 'is_malicious', 'attack_type', \n                           'attack', 'category', 'malware', 'phishing', 'threat', 'type', 'y']\n        actual_label_col = None\n        for col in df.columns:\n            if col.lower() in [lc.lower() for lc in label_candidates]:\n                actual_label_col = col\n                break\n        \n        # Separate features and labels\n        if actual_label_col:\n            y = df[actual_label_col]\n            X = df.drop(columns=[actual_label_col])\n        else:\n            y = None\n            X = df\n        \n        # Select numeric columns only\n        numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()\n        bool_cols = X.select_dtypes(include=[bool]).columns.tolist()\n        \n        X_numeric = X[numeric_cols].fillna(0)\n        \n        # Convert boolean to int\n        for col in bool_cols:\n            if col in X.columns:\n                X_numeric[col] = X[col].astype(int)\n        \n        self.feature_names = X_numeric.columns.tolist()\n        \n        # Encode labels if present\n        if y is not None:\n            if y.dtype == 'object':\n                y = self.label_encoder.fit_transform(y)\n            else:\n                y = y.values\n        \n        return X_numeric, y\\n"
    ]
   },
   {
@@ -574,9 +478,9 @@
     "if manifest_path.exists():\n",
     "    with open(manifest_path) as f:\n",
     "        manifest = json.load(f)\n",
-    "    print(f\"✓ Loaded manifest with {len(manifest)} datasets\")\n",
+    "    print(f\"\u2713 Loaded manifest with {len(manifest)} datasets\")\n",
     "else:\n",
-    "    print(\"⚠ No manifest found. Run 01_data_acquisition.ipynb first.\")\n",
+    "    print(\"\u26a0 No manifest found. Run 01_data_acquisition.ipynb first.\")\n",
     "    manifest = []"
    ]
   },
@@ -598,7 +502,7 @@
     "    path = Path(\"..\") / entry['path']\n",
     "    \n",
     "    if not path.exists():\n",
-    "        print(f\"  ⚠ {name}: File not found\")\n",
+    "        print(f\"  \u26a0 {name}: File not found\")\n",
     "        continue\n",
     "    \n",
     "    print(f\"  Processing: {name}\")\n",
@@ -622,7 +526,7 @@
     "            'n_features': len(pipeline.feature_names)\n",
     "        }\n",
     "        \n",
-    "        print(f\"    ✓ {len(X)} samples, {len(pipeline.feature_names)} features\")\n",
+    "        print(f\"    \u2713 {len(X)} samples, {len(pipeline.feature_names)} features\")\n",
     "        \n",
     "        feature_stats.append({\n",
     "            'name': name,\n",
@@ -632,9 +536,9 @@
     "        })\n",
     "        \n",
     "    except Exception as e:\n",
-    "        print(f\"    ⚠ Error: {e}\")\n",
+    "        print(f\"    \u26a0 Error: {e}\")\n",
     "\n",
-    "print(f\"\\n✓ Processed {len(processed_datasets)} datasets\")"
+    "print(f\"\\n\u2713 Processed {len(processed_datasets)} datasets\")"
    ]
   },
   {
@@ -679,7 +583,7 @@
     "        'has_labels': data['y'] is not None\n",
     "    })\n",
     "    \n",
-    "    print(f\"  ✓ Saved: {output_path.name}\")\n",
+    "    print(f\"  \u2713 Saved: {output_path.name}\")\n",
     "\n",
     "# Save feature manifest\n",
     "manifest_path = FEATURES_DIR / \"feature_manifest.json\"\n",
@@ -690,8 +594,8 @@
     "pipeline_path = FEATURES_DIR / \"feature_pipeline.pkl\"\n",
     "joblib.dump(pipeline, pipeline_path)\n",
     "\n",
-    "print(f\"\\n✓ Feature manifest saved to: {manifest_path}\")\n",
-    "print(f\"✓ Feature pipeline saved to: {pipeline_path}\")"
+    "print(f\"\\n\u2713 Feature manifest saved to: {manifest_path}\")\n",
+    "print(f\"\u2713 Feature pipeline saved to: {pipeline_path}\")"
    ]
   },
   {
@@ -717,26 +621,26 @@
     "total_features = max(d['n_features'] for d in processed_datasets.values()) if processed_datasets else 0\n",
     "\n",
     "print(f\"\"\"\n",
-    "🔧 Feature Engineering Summary:\n",
+    "\ud83d\udd27 Feature Engineering Summary:\n",
     "   - Datasets processed: {len(processed_datasets)}\n",
     "   - Total samples: {total_samples:,}\n",
     "   - Max features: {total_features}\n",
     "   - Output directory: {FEATURES_DIR}\n",
     "\n",
-    "📊 Feature Categories:\n",
+    "\ud83d\udcca Feature Categories:\n",
     "   - URL Features: Domain, path, security indicators\n",
     "   - Network Features: Request patterns, status codes\n",
     "   - Security Headers: CSP, HSTS, X-Frame-Options\n",
     "   - JavaScript: Console logs, suspicious APIs\n",
     "\n",
-    "📁 Datasets Ready for Training:\"\"\")\n",
+    "\ud83d\udcc1 Datasets Ready for Training:\"\"\")\n",
     "\n",
     "for entry in feature_manifest:\n",
-    "    print(f\"   ✓ {entry['name']}: {entry['samples']:,} samples, {entry['features']} features\")\n",
+    "    print(f\"   \u2713 {entry['name']}: {entry['samples']:,} samples, {entry['features']} features\")\n",
     "\n",
     "print(f\"\"\"\n",
     "Next step:\n",
-    "  → 03_model_training.ipynb\n",
+    "  \u2192 03_model_training.ipynb\n",
     "\"\"\")\n",
     "print(\"=\" * 60)"
    ]
@@ -749,4 +653,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}