notebooks/01_data_acquisition.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "417baa98",
   "metadata": {},
   "source": [
    "# 01 - Data Acquisition\n",
    "\n",
    "## CyberForge AI - Cybersecurity Data Collection & Preparation\n",
    "\n",
    "This notebook handles all data acquisition for the CyberForge AI ML pipeline.\n",
    "\n",
    "### Data Sources:\n",
    "1. **Public Datasets** - Legal, publicly available cybersecurity datasets\n",
    "2. **Web Scraper API** - Real-time website security data collection\n",
    "3. **Hugging Face Datasets** - Pre-uploaded CyberForge datasets\n",
    "4. **Synthetic Data** - Generated data for edge cases\n",
    "\n",
    "### Output:\n",
    "- Cleaned, normalized datasets ready for feature engineering\n",
    "- Data validation reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e0aeada",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load configuration from environment setup\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import httpx\n",
    "import asyncio\n",
    "from datetime import datetime\n",
    "from typing import Dict, List, Any, Optional\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# Load notebook config (same directory as this notebook)\n",
    "config_path = Path(\"notebook_config.json\")\n",
    "if not config_path.exists():\n",
    "    # Try absolute path as fallback\n",
    "    config_path = Path(\"/home/user/app/notebooks/notebook_config.json\")\n",
    "    \n",
    "if config_path.exists():\n",
    "    with open(config_path) as f:\n",
    "        CONFIG = json.load(f)\n",
    "    print(\"\u2713 Configuration loaded\")\n",
    "else:\n",
    "    raise FileNotFoundError(\"Run 00_environment_setup.ipynb first!\")\n",
    "\n",
    "DATASETS_DIR = Path(CONFIG.get(\"datasets_dir\", \"/home/user/app/datasets\"))\n",
    "print(f\"\u2713 Datasets directory: {DATASETS_DIR}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9493ab50",
   "metadata": {},
   "source": [
    "## 1. Web Scraper API Data Collection\n",
    "\n",
    "Collect real-time website security data using the WebScrapper.live API."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "876b9e05",
   "metadata": {},
   "outputs": [],
   "source": [
    "class WebScraperDataCollector:\n",
    "    \"\"\"\n",
    "    Collects website security data via WebScrapper.live API.\n",
    "    Uses synchronous httpx for Jupyter compatibility.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.api_url = \"http://webscrapper.live/api/scrape\"\n",
    "        self.api_key = \"sk-fd14eaa7bceb478db7afc7256e514d2b\"\n",
    "        self.timeout = 60.0\n",
    "    \n",
    "    def scrape_website(self, url: str) -> Dict[str, Any]:\n",
    "        \"\"\"Scrape a single website and return security data (sync version)\"\"\"\n",
    "        try:\n",
    "            with httpx.Client(timeout=self.timeout) as client:\n",
    "                response = client.post(\n",
    "                    self.api_url,\n",
    "                    json={\"url\": url},\n",
    "                    headers={\n",
    "                        \"Content-Type\": \"application/json\",\n",
    "                        \"X-API-Key\": self.api_key\n",
    "                    }\n",
    "                )\n",
    "                \n",
    "                if response.status_code == 200:\n",
    "                    data = response.json()\n",
    "                    return {\n",
    "                        \"success\": True,\n",
    "                        \"url\": url,\n",
    "                        \"security_report\": data.get(\"security_report\", {}),\n",
    "                        \"network_requests\": data.get(\"network_requests\", []),\n",
    "                        \"console_logs\": data.get(\"console_logs\", []),\n",
    "                        \"performance_metrics\": data.get(\"performance_metrics\", {}),\n",
    "                        \"response_headers\": data.get(\"response_headers\", {}),\n",
    "                        \"html_length\": len(data.get(\"html\", \"\") or data.get(\"content\", \"\")),\n",
    "                        \"scraped_at\": datetime.now().isoformat()\n",
    "                    }\n",
    "                else:\n",
    "                    return {\"success\": False, \"url\": url, \"error\": f\"Status {response.status_code}\"}\n",
    "        except Exception as e:\n",
    "            return {\"success\": False, \"url\": url, \"error\": str(e)}\n",
    "    \n",
    "    def collect_batch(self, urls: List[str]) -> List[Dict]:\n",
    "        \"\"\"Collect data from multiple URLs (sync version)\"\"\"\n",
    "        import time\n",
    "        results = []\n",
    "        for url in urls:\n",
    "            print(f\"  Scraping: {url}\")\n",
    "            result = self.scrape_website(url)\n",
    "            results.append(result)\n",
    "            time.sleep(1)  # Rate limiting\n",
    "        return results\n",
    "    \n",
    "    def extract_security_features(self, data: Dict) -> Dict:\n",
    "        \"\"\"Extract security-relevant features from scraped data\"\"\"\n",
    "        if not data.get(\"success\"):\n",
    "            return None\n",
    "        \n",
    "        security_report = data.get(\"security_report\", {})\n",
    "        network_requests = data.get(\"network_requests\", [])\n",
    "        console_logs = data.get(\"console_logs\", [])\n",
    "        \n",
    "        return {\n",
    "            \"url\": data[\"url\"],\n",
    "            \"is_https\": security_report.get(\"is_https\", False),\n",
    "            \"has_mixed_content\": security_report.get(\"mixed_content\", False),\n",
    "            \"missing_headers_count\": len(security_report.get(\"missing_security_headers\", [])),\n",
    "            \"has_insecure_cookies\": security_report.get(\"insecure_cookies\", False),\n",
    "            \"total_requests\": len(network_requests),\n",
    "            \"external_requests\": sum(1 for r in network_requests if self._is_external(r, data[\"url\"])),\n",
    "            \"failed_requests\": sum(1 for r in network_requests if r.get(\"status\", 200) >= 400),\n",
    "            \"console_errors\": sum(1 for log in console_logs if log.get(\"level\") == \"error\"),\n",
    "            \"console_warnings\": sum(1 for log in console_logs if log.get(\"level\") == \"warning\"),\n",
    "            \"html_size\": data.get(\"html_length\", 0),\n",
    "            \"scraped_at\": data[\"scraped_at\"]\n",
    "        }\n",
    "    \n",
    "    def _is_external(self, request: Dict, base_url: str) -> bool:\n",
    "        \"\"\"Check if a request is to an external domain\"\"\"\n",
    "        try:\n",
    "            from urllib.parse import urlparse\n",
    "            base_domain = urlparse(base_url).netloc\n",
    "            req_domain = urlparse(request.get(\"url\", \"\")).netloc\n",
    "            return base_domain != req_domain\n",
    "        except:\n",
    "            return False\n",
    "\n",
    "scraper = WebScraperDataCollector()\n",
    "print(\"\u2713 Web Scraper Data Collector initialized\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aec0ba34",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Collect sample data from known safe/unsafe websites for training\n",
    "SAMPLE_URLS = [\n",
    "    # Known safe websites\n",
    "    \"https://www.google.com\",\n",
    "    \"https://github.com\",\n",
    "    \"https://www.microsoft.com\",\n",
    "    \"https://www.amazon.com\",\n",
    "    \"https://www.wikipedia.org\",\n",
    "    # Test sites\n",
    "    \"https://example.com\",\n",
    "    \"https://httpbin.org\",\n",
    "]\n",
    "\n",
    "print(f\"Collecting data from {len(SAMPLE_URLS)} URLs...\")\n",
    "\n",
    "# Run synchronous collection (Jupyter-compatible)\n",
    "scraped_data = scraper.collect_batch(SAMPLE_URLS[:3])  # Limited for demo\n",
    "\n",
    "# Extract features\n",
    "features_list = [scraper.extract_security_features(d) for d in scraped_data if d.get(\"success\")]\n",
    "if features_list:\n",
    "    web_scraper_df = pd.DataFrame(features_list)\n",
    "    print(f\"\\n\u2713 Collected {len(web_scraper_df)} website security profiles\")\n",
    "    display(web_scraper_df.head())\n",
    "else:\n",
    "    print(\"\u26a0 No data collected - API may be unavailable\")\n",
    "    web_scraper_df = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0dadb1ac",
   "metadata": {},
   "source": [
    "## 2. Load Hugging Face Datasets\n",
    "\n",
    "Load pre-uploaded CyberForge datasets from Hugging Face."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9be784df",
   "metadata": {},
   "outputs": [],
   "source": [
    "from huggingface_hub import hf_hub_download, list_repo_files\n",
    "import os\n",
    "\n",
    "HF_DATASET_REPO = \"Che237/cyberforge-datasets\"\n",
    "\n",
    "def list_available_datasets():\n",
    "    \"\"\"List all available datasets in the HF repository\"\"\"\n",
    "    try:\n",
    "        files = list_repo_files(HF_DATASET_REPO, repo_type=\"dataset\")\n",
    "        csv_files = [f for f in files if f.endswith('.csv')]\n",
    "        return csv_files\n",
    "    except Exception as e:\n",
    "        print(f\"\u26a0 Could not list HF datasets: {e}\")\n",
    "        return []\n",
    "\n",
    "def download_dataset(file_path: str, local_dir: Path = DATASETS_DIR) -> Optional[Path]:\n",
    "    \"\"\"Download a specific dataset from Hugging Face\"\"\"\n",
    "    try:\n",
    "        local_path = hf_hub_download(\n",
    "            repo_id=HF_DATASET_REPO,\n",
    "            filename=file_path,\n",
    "            repo_type=\"dataset\",\n",
    "            cache_dir=str(local_dir / \"cache\")\n",
    "        )\n",
    "        return Path(local_path)\n",
    "    except Exception as e:\n",
    "        print(f\"\u26a0 Could not download {file_path}: {e}\")\n",
    "        return None\n",
    "\n",
    "# List available datasets\n",
    "print(\"Available datasets on Hugging Face:\")\n",
    "available_files = list_available_datasets()\n",
    "for f in available_files[:20]:  # Show first 20\n",
    "    print(f\"  - {f}\")\n",
    "print(f\"  ... and {len(available_files) - 20} more\" if len(available_files) > 20 else \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc9ef68b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Priority datasets for training (aligned with backend requirements)\n",
    "PRIORITY_DATASETS = {\n",
    "    \"network_intrusion\": \"network_intrusion/network_intrusion_processed.csv\",\n",
    "    \"phishing_detection\": \"phishing_detection/phishing_detection_processed.csv\",\n",
    "    \"malware_detection\": \"malware_detection/malware_detection_processed.csv\",\n",
    "    \"anomaly_detection\": \"anomaly_detection/anomaly_detection_processed.csv\",\n",
    "    \"web_attack_detection\": \"web_attack_detection/web_attack_detection_processed.csv\",\n",
    "}\n",
    "\n",
    "loaded_datasets = {}\n",
    "\n",
    "print(\"Downloading priority datasets...\")\n",
    "for name, path in PRIORITY_DATASETS.items():\n",
    "    if path in available_files:\n",
    "        print(f\"  Downloading: {name}\")\n",
    "        local_path = download_dataset(path)\n",
    "        if local_path:\n",
    "            try:\n",
    "                df = pd.read_csv(local_path)\n",
    "                loaded_datasets[name] = df\n",
    "                print(f\"    \u2713 {name}: {len(df)} samples, {len(df.columns)} features\")\n",
    "            except Exception as e:\n",
    "                print(f\"    \u26a0 Could not load {name}: {e}\")\n",
    "    else:\n",
    "        print(f\"  \u26a0 {name}: Not found in repository\")\n",
    "\n",
    "print(f\"\\n\u2713 Loaded {len(loaded_datasets)} datasets\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fadcbfc",
   "metadata": {},
   "source": [
    "## 3. Load Local Datasets\n",
    "\n",
    "Load any datasets already present in the local datasets directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40cad5c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_local_datasets(datasets_dir: Path) -> Dict[str, pd.DataFrame]:\n",
    "    \"\"\"Load all CSV datasets from local directory\"\"\"\n",
    "    datasets = {}\n",
    "    \n",
    "    for csv_file in datasets_dir.rglob(\"*_processed.csv\"):\n",
    "        try:\n",
    "            name = csv_file.stem.replace(\"_processed\", \"\")\n",
    "            df = pd.read_csv(csv_file)\n",
    "            datasets[name] = df\n",
    "            print(f\"  \u2713 {name}: {len(df)} samples\")\n",
    "        except Exception as e:\n",
    "            print(f\"  \u26a0 {csv_file.name}: {e}\")\n",
    "    \n",
    "    return datasets\n",
    "\n",
    "print(\"Loading local datasets...\")\n",
    "local_datasets = load_local_datasets(DATASETS_DIR)\n",
    "\n",
    "# Merge with HF datasets (local takes precedence)\n",
    "for name, df in local_datasets.items():\n",
    "    if name not in loaded_datasets:\n",
    "        loaded_datasets[name] = df\n",
    "\n",
    "print(f\"\\n\u2713 Total datasets available: {len(loaded_datasets)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3584926",
   "metadata": {},
   "source": [
    "## 4. Data Validation & Quality Checks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4adefa74",
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_dataset(name: str, df: pd.DataFrame) -> Dict[str, Any]:\n",
    "    \"\"\"Validate dataset quality and return report\"\"\"\n",
    "    # Expanded label column detection\n",
    "    label_columns = ['label', 'target', 'class', 'is_malicious', 'attack_type', \n",
    "                     'attack', 'category', 'malware', 'phishing', 'threat', \n",
    "                     'classification', 'type', 'result', 'output', 'y']\n",
    "    has_label = any(col.lower() in [c.lower() for c in df.columns] for col in label_columns)\n",
    "    \n",
    "    report = {\n",
    "        \"name\": name,\n",
    "        \"samples\": len(df),\n",
    "        \"features\": len(df.columns),\n",
    "        \"missing_values\": df.isnull().sum().sum(),\n",
    "        \"missing_pct\": (df.isnull().sum().sum() / (len(df) * len(df.columns))) * 100 if len(df) > 0 else 0,\n",
    "        \"duplicate_rows\": df.duplicated().sum(),\n",
    "        \"numeric_columns\": len(df.select_dtypes(include=[np.number]).columns),\n",
    "        \"categorical_columns\": len(df.select_dtypes(include=['object', 'category']).columns),\n",
    "        \"memory_mb\": df.memory_usage(deep=True).sum() / (1024 * 1024),\n",
    "        \"has_label\": has_label,\n",
    "        \"valid\": True\n",
    "    }\n",
    "    \n",
    "    # More lenient validation - only fail on critical issues\n",
    "    issues = []\n",
    "    if report[\"samples\"] < 10:\n",
    "        issues.append(\"Too few samples (<10)\")\n",
    "    if report[\"missing_pct\"] > 80:\n",
    "        issues.append(\"Too many missing values (>80%)\")\n",
    "    \n",
    "    report[\"issues\"] = issues\n",
    "    report[\"valid\"] = len(issues) == 0\n",
    "    \n",
    "    return report\n",
    "\n",
    "# Validate all datasets\n",
    "validation_reports = []\n",
    "print(\"Validating datasets...\\n\")\n",
    "print(f\"{'Dataset':<30} {'Samples':>10} {'Features':>10} {'Missing %':>10} {'Valid':>8}\")\n",
    "print(\"-\" * 75)\n",
    "\n",
    "for name, df in loaded_datasets.items():\n",
    "    report = validate_dataset(name, df)\n",
    "    validation_reports.append(report)\n",
    "    status = \"\u2713\" if report[\"valid\"] else \"\u26a0\"\n",
    "    print(f\"{name:<30} {report['samples']:>10} {report['features']:>10} {report['missing_pct']:>9.2f}% {status:>8}\")\n",
    "\n",
    "valid_datasets = [r[\"name\"] for r in validation_reports if r[\"valid\"]]\n",
    "print(f\"\\n\u2713 {len(valid_datasets)} datasets passed validation\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0603447f",
   "metadata": {},
   "source": [
    "## 5. Data Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6544e100",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_dataset(df: pd.DataFrame, name: str) -> pd.DataFrame:\n",
    "    \"\"\"Normalize dataset for consistent processing\"\"\"\n",
    "    df = df.copy()\n",
    "    \n",
    "    # Standardize column names\n",
    "    df.columns = df.columns.str.lower().str.replace(' ', '_').str.replace('-', '_')\n",
    "    \n",
    "    # Find and standardize label column (expanded list)\n",
    "    label_columns = ['label', 'target', 'class', 'is_malicious', 'attack_type', \n",
    "                     'attack', 'category', 'malware', 'phishing', 'threat',\n",
    "                     'classification', 'type', 'result', 'output', 'y']\n",
    "    for col in label_columns:\n",
    "        matching_cols = [c for c in df.columns if c.lower() == col.lower()]\n",
    "        if matching_cols:\n",
    "            df = df.rename(columns={matching_cols[0]: 'label'})\n",
    "            break\n",
    "    \n",
    "    # Handle missing values\n",
    "    numeric_cols = df.select_dtypes(include=[np.number]).columns\n",
    "    if len(numeric_cols) > 0:\n",
    "        df[numeric_cols] = df[numeric_cols].fillna(df[numeric_cols].median())\n",
    "    \n",
    "    categorical_cols = df.select_dtypes(include=['object', 'category']).columns\n",
    "    for col in categorical_cols:\n",
    "        if col != 'label':\n",
    "            df[col] = df[col].fillna('unknown')\n",
    "    \n",
    "    # Remove duplicates\n",
    "    df = df.drop_duplicates()\n",
    "    \n",
    "    # Add metadata\n",
    "    df.attrs['dataset_name'] = name\n",
    "    df.attrs['processed_at'] = datetime.now().isoformat()\n",
    "    \n",
    "    return df\n",
    "\n",
    "# Normalize ALL loaded datasets (not just valid ones)\n",
    "normalized_datasets = {}\n",
    "print(\"Normalizing datasets...\")\n",
    "\n",
    "for name, df in loaded_datasets.items():\n",
    "    try:\n",
    "        normalized_df = normalize_dataset(df, name)\n",
    "        normalized_datasets[name] = normalized_df\n",
    "        print(f\"  \u2713 {name}: {len(normalized_df)} samples after normalization\")\n",
    "    except Exception as e:\n",
    "        print(f\"  \u26a0 {name}: Error during normalization - {e}\")\n",
    "\n",
    "print(f\"\\n\u2713 Normalized {len(normalized_datasets)} datasets\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9a016d4",
   "metadata": {},
   "source": [
    "## 6. Save Processed Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91248c3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create processed data directory\n",
    "PROCESSED_DIR = DATASETS_DIR / \"processed\"\n",
    "PROCESSED_DIR.mkdir(exist_ok=True)\n",
    "\n",
    "# Save each normalized dataset\n",
    "print(\"Saving processed datasets...\")\n",
    "dataset_manifest = []\n",
    "\n",
    "for name, df in normalized_datasets.items():\n",
    "    output_path = PROCESSED_DIR / f\"{name}_ready.csv\"\n",
    "    df.to_csv(output_path, index=False)\n",
    "    \n",
    "    manifest_entry = {\n",
    "        \"name\": name,\n",
    "        \"path\": str(output_path.relative_to(DATASETS_DIR.parent)),\n",
    "        \"samples\": len(df),\n",
    "        \"features\": len(df.columns),\n",
    "        \"has_label\": \"label\" in df.columns,\n",
    "        \"processed_at\": datetime.now().isoformat()\n",
    "    }\n",
    "    dataset_manifest.append(manifest_entry)\n",
    "    print(f\"  \u2713 Saved: {output_path.name}\")\n",
    "\n",
    "# Save manifest\n",
    "manifest_path = PROCESSED_DIR / \"manifest.json\"\n",
    "with open(manifest_path, \"w\") as f:\n",
    "    json.dump(dataset_manifest, f, indent=2)\n",
    "\n",
    "print(f\"\\n\u2713 Dataset manifest saved to: {manifest_path}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40ba332c",
   "metadata": {},
   "source": [
    "## 7. Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ef2f995",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n\" + \"=\" * 60)\n",
    "print(\"DATA ACQUISITION COMPLETE\")\n",
    "print(\"=\" * 60)\n",
    "\n",
    "total_samples = sum(len(df) for df in normalized_datasets.values())\n",
    "\n",
    "print(f\"\"\"\n",
    "\ud83d\udcca Data Collection Summary:\n",
    "   - Datasets processed: {len(normalized_datasets)}\n",
    "   - Total samples: {total_samples:,}\n",
    "   - Output directory: {PROCESSED_DIR}\n",
    "\n",
    "\ud83d\udcc1 Datasets Ready for Feature Engineering:\"\"\")\n",
    "\n",
    "for entry in dataset_manifest:\n",
    "    print(f\"   \u2713 {entry['name']}: {entry['samples']:,} samples\")\n",
    "\n",
    "print(f\"\"\"\n",
    "Next step:\n",
    "  \u2192 02_feature_engineering.ipynb\n",
    "\"\"\")\n",
    "print(\"=\" * 60)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}