Add notebook 08: train all models + upload to Che237/cyberforge-models

notebooks/08_upload_to_hub.ipynb

@@ -0,0 +1,344 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 5,
+ "metadata": {"kernelspec": {"display_name": "Python 3","language": "python","name": "python3"},"language_info": {"name": "python","version": "3.11.0"}},
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": ["# 08 - Upload Trained Models to HuggingFace Hub\n\nTrains all 4 CyberForge models from scratch (or loads existing ones) then uploads to `Che237/cyberforge-models`."]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os, json, joblib, logging\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from pathlib import Path\n",
+    "from datetime import datetime\n",
+    "from sklearn.ensemble import GradientBoostingClassifier, IsolationForest\n",
+    "from sklearn.preprocessing import StandardScaler\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.metrics import accuracy_score, f1_score\n",
+    "from huggingface_hub import HfApi, create_repo\n",
+    "\n",
+    "logging.basicConfig(level=logging.INFO, format='%(levelname)s | %(message)s')\n",
+    "log = logging.getLogger(__name__)\n",
+    "\n",
+    "HF_TOKEN   = os.environ.get('HF_TOKEN', '')\n",
+    "MODEL_REPO = 'Che237/cyberforge-models'\n",
+    "NB_DIR     = Path('.').absolute()\n",
+    "MODELS_DIR = NB_DIR.parent / 'models'\n",
+    "DATASETS   = NB_DIR.parent / 'datasets'\n",
+    "UPLOAD_DIR = NB_DIR.parent / 'trained_models'\n",
+    "UPLOAD_DIR.mkdir(exist_ok=True)\n",
+    "\n",
+    "FEATURE_NAMES = [\n",
+    "    'url_length','hostname_length','path_length','is_https',\n",
+    "    'has_ip_address','has_suspicious_tld','subdomain_count',\n",
+    "    'has_port','query_params_count','has_at_symbol',\n",
+    "    'has_double_slash','special_char_count'\n",
+    "]\n",
+    "rng = np.random.default_rng(42)\n",
+    "print(f'Working dir: {NB_DIR}')\n",
+    "print(f'Models dir:  {MODELS_DIR} (exists={MODELS_DIR.exists()})')\n",
+    "print(f'Upload dir:  {UPLOAD_DIR}')\n",
+    "print(f'HF_TOKEN set: {bool(HF_TOKEN)}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ── Synthetic data generators ───────────────────────────────────────────────\n",
+    "def synth_benign(n=1500):\n",
+    "    d = {\n",
+    "        'url_length':         rng.integers(15, 60, n),\n",
+    "        'hostname_length':    rng.integers(5, 25, n),\n",
+    "        'path_length':        rng.integers(0, 30, n),\n",
+    "        'is_https':           rng.choice([1,1,1,0], n),\n",
+    "        'has_ip_address':     rng.choice([0,0,0,0,1], n),\n",
+    "        'has_suspicious_tld': rng.choice([0,0,0,1], n),\n",
+    "        'subdomain_count':    rng.integers(0, 2, n),\n",
+    "        'has_port':           rng.choice([0,0,0,1], n),\n",
+    "        'query_params_count': rng.integers(0, 3, n),\n",
+    "        'has_at_symbol':      rng.choice([0,0,0,0,1], n),\n",
+    "        'has_double_slash':   rng.choice([0,0,0,1], n),\n",
+    "        'special_char_count': rng.integers(0, 4, n),\n",
+    "    }\n",
+    "    return pd.DataFrame(d), np.zeros(n, dtype=int)\n",
+    "\n",
+    "def synth_malicious(n=1500):\n",
+    "    d = {\n",
+    "        'url_length':         rng.integers(60, 300, n),\n",
+    "        'hostname_length':    rng.integers(20, 80, n),\n",
+    "        'path_length':        rng.integers(10, 120, n),\n",
+    "        'is_https':           rng.choice([1,0,0], n),\n",
+    "        'has_ip_address':     rng.choice([0,0,1,1], n),\n",
+    "        'has_suspicious_tld': rng.choice([0,1,1,1], n),\n",
+    "        'subdomain_count':    rng.integers(1, 5, n),\n",
+    "        'has_port':           rng.choice([0,0,1,1], n),\n",
+    "        'query_params_count': rng.integers(2, 10, n),\n",
+    "        'has_at_symbol':      rng.choice([0,0,0,1,1], n),\n",
+    "        'has_double_slash':   rng.choice([0,0,1,1], n),\n",
+    "        'special_char_count': rng.integers(5, 25, n),\n",
+    "    }\n",
+    "    return pd.DataFrame(d), np.ones(n, dtype=int)\n",
+    "\n",
+    "print('✓ Synthetic data generators ready')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ── Load real phishing dataset ───────────────────────────────────────────────\n",
+    "def load_phishing():\n",
+    "    csv_path = DATASETS / 'phishing_detection' / 'phishing_detection_processed.csv'\n",
+    "    X_b, y_b = synth_benign(2000)\n",
+    "    X_m, y_m = synth_malicious(2000)\n",
+    "    if csv_path.exists():\n",
+    "        df = pd.read_csv(csv_path)\n",
+    "        mapped = pd.DataFrame()\n",
+    "        mapped['url_length']          = df.get('url_length', rng.integers(15,200,len(df)))\n",
+    "        mapped['hostname_length']     = (df.get('url_length',40)*0.3).astype(int)\n",
+    "        mapped['path_length']         = (df.get('url_length',40)*0.4).astype(int)\n",
+    "        mapped['is_https']            = df.get('https',1)\n",
+    "        mapped['has_ip_address']      = rng.integers(0,2,len(df))\n",
+    "        mapped['has_suspicious_tld']  = (df.get('suspicious_words',0)>3).astype(int)\n",
+    "        mapped['subdomain_count']     = df.get('subdomain_level',rng.integers(0,3,len(df)))\n",
+    "        mapped['has_port']            = rng.choice([0,1],len(df),p=[0.85,0.15])\n",
+    "        mapped['query_params_count']  = rng.integers(0,6,len(df))\n",
+    "        mapped['has_at_symbol']       = rng.choice([0,1],len(df),p=[0.9,0.1])\n",
+    "        mapped['has_double_slash']    = rng.choice([0,1],len(df),p=[0.85,0.15])\n",
+    "        mapped['special_char_count']  = df.get('suspicious_words',rng.integers(0,15,len(df)))\n",
+    "        y_real = df['is_phishing'].values\n",
+    "        X = pd.concat([mapped, X_b, X_m], ignore_index=True)\n",
+    "        y = np.concatenate([y_real, y_b, y_m])\n",
+    "        print(f'Phishing: {len(X)} samples (real CSV + synthetic)')\n",
+    "    else:\n",
+    "        X = pd.concat([X_b, X_m], ignore_index=True)\n",
+    "        y = np.concatenate([y_b, y_m])\n",
+    "        print(f'Phishing: {len(X)} samples (synthetic only)')\n",
+    "    return X, y\n",
+    "\n",
+    "def load_malware():\n",
+    "    X_b, y_b = synth_benign(2000)\n",
+    "    X_m, y_m = synth_malicious(2000)\n",
+    "    csv_path = DATASETS / 'malware_detection' / 'malware_detection_processed.csv'\n",
+    "    if csv_path.exists():\n",
+    "        df = pd.read_csv(csv_path)\n",
+    "        mapped = pd.DataFrame()\n",
+    "        mapped['url_length']          = (df.get('file_size',50000)/1000).clip(10,300).astype(int)\n",
+    "        mapped['hostname_length']     = (df.get('entropy',4)*5).clip(5,40).astype(int)\n",
+    "        mapped['path_length']         = (df.get('strings_count',500)/100).clip(0,80).astype(int)\n",
+    "        mapped['is_https']            = rng.choice([0,1],len(df),p=[0.6,0.4])\n",
+    "        mapped['has_ip_address']      = (df.get('entropy',0)>6).astype(int)\n",
+    "        mapped['has_suspicious_tld']  = rng.integers(0,2,len(df))\n",
+    "        mapped['subdomain_count']     = df.get('pe_sections',rng.integers(0,4,len(df))).clip(0,6).astype(int)\n",
+    "        mapped['has_port']            = rng.choice([0,1],len(df),p=[0.7,0.3])\n",
+    "        mapped['query_params_count']  = (df.get('exports',0)/20).clip(0,10).astype(int)\n",
+    "        mapped['has_at_symbol']       = rng.choice([0,1],len(df),p=[0.85,0.15])\n",
+    "        mapped['has_double_slash']    = rng.choice([0,1],len(df),p=[0.8,0.2])\n",
+    "        mapped['special_char_count']  = (df.get('entropy',4)*2).clip(0,25).astype(int)\n",
+    "        y_real = df['is_malware'].values\n",
+    "        X = pd.concat([mapped, X_b, X_m], ignore_index=True)\n",
+    "        y = np.concatenate([y_real, y_b, y_m])\n",
+    "        print(f'Malware: {len(X)} samples (real CSV + synthetic)')\n",
+    "    else:\n",
+    "        X = pd.concat([X_b, X_m], ignore_index=True)\n",
+    "        y = np.concatenate([y_b, y_m])\n",
+    "        print(f'Malware: {len(X)} samples (synthetic only)')\n",
+    "    return X, y\n",
+    "\n",
+    "def load_web_attack():\n",
+    "    X_b, y_b = synth_benign(2000)\n",
+    "    X_m, y_m = synth_malicious(2000)\n",
+    "    X = pd.concat([X_b, X_m], ignore_index=True)\n",
+    "    y = np.concatenate([y_b, y_m])\n",
+    "    print(f'WebAttack: {len(X)} samples (synthetic)')\n",
+    "    return X, y\n",
+    "\n",
+    "def load_anomaly():\n",
+    "    X_b, y_b = synth_benign(3000)\n",
+    "    X_m, y_m = synth_malicious(600)\n",
+    "    X = pd.concat([X_b, X_m], ignore_index=True)\n",
+    "    y = np.concatenate([y_b, y_m])\n",
+    "    print(f'Anomaly: {len(X)} samples (synthetic)')\n",
+    "    return X, y\n",
+    "\n",
+    "print('✓ Dataset loaders ready')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ── Train one model ──────────────────────────────────────────────────────────\n",
+    "def train_model(name, X, y, isolation_forest=False):\n",
+    "    for col in FEATURE_NAMES:\n",
+    "        if col not in X.columns:\n",
+    "            X[col] = 0\n",
+    "    X = X[FEATURE_NAMES].fillna(0).astype(float)\n",
+    "\n",
+    "    X_tr, X_te, y_tr, y_te = train_test_split(\n",
+    "        X, y, test_size=0.2, random_state=42, stratify=y\n",
+    "    )\n",
+    "    scaler = StandardScaler()\n",
+    "    X_tr_s = scaler.fit_transform(X_tr)\n",
+    "    X_te_s = scaler.transform(X_te)\n",
+    "\n",
+    "    if isolation_forest:\n",
+    "        X_benign = X_tr_s[y_tr == 0]\n",
+    "        model = IsolationForest(n_estimators=200, contamination=0.1, random_state=42)\n",
+    "        model.fit(X_benign)\n",
+    "        preds = model.predict(X_te_s)\n",
+    "        y_pred = (preds == -1).astype(int)\n",
+    "    else:\n",
+    "        model = GradientBoostingClassifier(\n",
+    "            n_estimators=200, learning_rate=0.1, max_depth=5,\n",
+    "            subsample=0.8, random_state=42\n",
+    "        )\n",
+    "        model.fit(X_tr_s, y_tr)\n",
+    "        y_pred = model.predict(X_te_s)\n",
+    "\n",
+    "    acc = accuracy_score(y_te, y_pred)\n",
+    "    f1  = f1_score(y_te, y_pred, zero_division=0)\n",
+    "\n",
+    "    # Save to UPLOAD_DIR (= trained_models/) for app.py to pick up\n",
+    "    model_dir = UPLOAD_DIR / name\n",
+    "    model_dir.mkdir(parents=True, exist_ok=True)\n",
+    "    joblib.dump(model,  model_dir / 'best_model.pkl')\n",
+    "    joblib.dump(scaler, model_dir / 'scaler.pkl')\n",
+    "    meta = {\n",
+    "        'name': name, 'trained_at': datetime.utcnow().isoformat(),\n",
+    "        'samples': int(len(X)), 'threat_rate': float(y.mean()),\n",
+    "        'accuracy': float(acc), 'f1': float(f1),\n",
+    "        'feature_names': FEATURE_NAMES,\n",
+    "        'model_type': 'IsolationForest' if isolation_forest else 'GradientBoostingClassifier',\n",
+    "    }\n",
+    "    with open(model_dir / 'metadata.json', 'w') as f:\n",
+    "        json.dump(meta, f, indent=2)\n",
+    "\n",
+    "    print(f'  ✓ {name}: acc={acc:.3f} f1={f1:.3f} ({len(X)} samples)')\n",
+    "    return meta\n",
+    "\n",
+    "print('✓ Trainer ready — starting training pipeline')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ── Run training ─────────────────────────────────────────────────────────────\n",
+    "results = {}\n",
+    "print('Training phishing_detection...')\n",
+    "X, y = load_phishing()\n",
+    "results['phishing_detection'] = train_model('phishing_detection', X, y)\n",
+    "\n",
+    "print('Training malware_detection...')\n",
+    "X, y = load_malware()\n",
+    "results['malware_detection'] = train_model('malware_detection', X, y)\n",
+    "\n",
+    "print('Training web_attack_detection...')\n",
+    "X, y = load_web_attack()\n",
+    "results['web_attack_detection'] = train_model('web_attack_detection', X, y)\n",
+    "\n",
+    "print('Training anomaly_detection...')\n",
+    "X, y = load_anomaly()\n",
+    "results['anomaly_detection'] = train_model('anomaly_detection', X, y, isolation_forest=True)\n",
+    "\n",
+    "print()\n",
+    "print('='*50)\n",
+    "print('TRAINING COMPLETE')\n",
+    "for name, m in results.items():\n",
+    "    print(f'  {name}: acc={m[\"accuracy\"]:.3f} f1={m[\"f1\"]:.3f}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ── Upload to HuggingFace model repo ─────────────────────────────────────────\n",
+    "if not HF_TOKEN:\n",
+    "    print('⚠ HF_TOKEN not set — skipping upload. Models saved locally only.')\n",
+    "else:\n",
+    "    api = HfApi(token=HF_TOKEN)\n",
+    "    try:\n",
+    "        create_repo(MODEL_REPO, repo_type='model', token=HF_TOKEN, exist_ok=True, private=False)\n",
+    "        print(f'✓ Repo ready: {MODEL_REPO}')\n",
+    "    except Exception as e:\n",
+    "        print(f'Repo create: {e}')\n",
+    "\n",
+    "    uploaded = 0\n",
+    "    for name in results:\n",
+    "        model_dir = UPLOAD_DIR / name\n",
+    "        for fname in ['best_model.pkl', 'scaler.pkl', 'metadata.json']:\n",
+    "            fpath = model_dir / fname\n",
+    "            if not fpath.exists():\n",
+    "                print(f'  Missing: {fpath}')\n",
+    "                continue\n",
+    "            try:\n",
+    "                api.upload_file(\n",
+    "                    path_or_fileobj=str(fpath),\n",
+    "                    path_in_repo=f'{name}/{fname}',\n",
+    "                    repo_id=MODEL_REPO,\n",
+    "                    repo_type='model',\n",
+    "                    token=HF_TOKEN,\n",
+    "                )\n",
+    "                uploaded += 1\n",
+    "                print(f'  ✅ {name}/{fname}')\n",
+    "            except Exception as e:\n",
+    "                print(f'  ❌ {name}/{fname}: {e}')\n",
+    "\n",
+    "    print()\n",
+    "    print(f'Upload complete: {uploaded} files → {MODEL_REPO}')\n",
+    "    print(f'View: https://huggingface.co/{MODEL_REPO}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ── Verify models are accessible ────────────────────────────────────────��────\n",
+    "print('Verifying models in trained_models/')\n",
+    "for name in results:\n",
+    "    model_path = UPLOAD_DIR / name / 'best_model.pkl'\n",
+    "    if model_path.exists():\n",
+    "        m = joblib.load(model_path)\n",
+    "        # Quick test prediction\n",
+    "        import numpy as np\n",
+    "        X_test = np.array([[100,20,30,0,1,1,2,1,3,0,1,8]])  # suspicious URL features\n",
+    "        try:\n",
+    "            scaler_path = UPLOAD_DIR / name / 'scaler.pkl'\n",
+    "            if scaler_path.exists():\n",
+    "                sc = joblib.load(scaler_path)\n",
+    "                X_test = sc.transform(X_test)\n",
+    "            pred = m.predict(X_test)\n",
+    "            label = 'THREAT' if pred[0] == 1 else 'BENIGN'\n",
+    "            print(f'  ✓ {name}: predict={label} (model loaded OK)')\n",
+    "        except Exception as e:\n",
+    "            print(f'  ✓ {name}: model loaded (predict error: {e})')\n",
+    "    else:\n",
+    "        print(f'  ✗ {name}: model not found at {model_path}')\n",
+    "print()\n",
+    "print('All done! Models available at:', str(UPLOAD_DIR))"
+   ]
+  }
+ ]
+}