Fix label detection in dataset loading

notebooks/03_model_training.ipynb

@@ -60,9 +60,9 @@
     "MODELS_DIR = DATASETS_DIR.parent / \"models\"\n",
     "MODELS_DIR.mkdir(exist_ok=True)\n",
     "\n",
-    "print(f\"✓ Configuration loaded\")\n",
-    "print(f\"✓ Features from: {FEATURES_DIR}\")\n",
-    "print(f\"✓ Models output: {MODELS_DIR}\")"
+    "print(f\"\u2713 Configuration loaded\")\n",
+    "print(f\"\u2713 Features from: {FEATURES_DIR}\")\n",
+    "print(f\"\u2713 Models output: {MODELS_DIR}\")"
    ]
   },
   {
@@ -80,33 +80,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Load feature manifest\n",
-    "feature_manifest_path = FEATURES_DIR / \"feature_manifest.json\"\n",
-    "\n",
-    "if feature_manifest_path.exists():\n",
-    "    with open(feature_manifest_path) as f:\n",
-    "        feature_manifest = json.load(f)\n",
-    "    print(f\"✓ Loaded {len(feature_manifest)} feature datasets\")\n",
-    "else:\n",
-    "    print(\"⚠ No feature manifest. Run 02_feature_engineering.ipynb first.\")\n",
-    "    feature_manifest = []\n",
-    "\n",
-    "# Load datasets\n",
-    "datasets = {}\n",
-    "print(\"\\nLoading feature datasets:\")\n",
-    "\n",
-    "for entry in feature_manifest:\n",
-    "    name = entry['name']\n",
-    "    path = Path(\"..\") / entry['path']\n",
-    "    \n",
-    "    if path.exists() and entry.get('has_labels', False):\n",
-    "        df = pd.read_parquet(path)\n",
-    "        datasets[name] = df\n",
-    "        print(f\"  ✓ {name}: {len(df)} samples, {len(df.columns)} features\")\n",
-    "    else:\n",
-    "        print(f\"  ⚠ {name}: No labels or file missing\")\n",
-    "\n",
-    "print(f\"\\n✓ Loaded {len(datasets)} datasets with labels for training\")"
+    "# Load feature manifest\nfeature_manifest_path = FEATURES_DIR / \"feature_manifest.json\"\n\nif feature_manifest_path.exists():\n    with open(feature_manifest_path) as f:\n        feature_manifest = json.load(f)\n    print(f\"\u2713 Loaded {len(feature_manifest)} feature datasets\")\nelse:\n    print(\"\u26a0 No feature manifest. Run 02_feature_engineering.ipynb first.\")\n    feature_manifest = []\n\n# Load datasets - be more lenient with label detection\ndatasets = {}\nprint(\"\\n",
+    "Loading feature datasets:\")\n\nfor entry in feature_manifest:\n    name = entry['name']\n    path = Path(\"..\") / entry['path']\n    \n    if path.exists():\n        try:\n            df = pd.read_parquet(path)\n            \n            # Check for label column with multiple possible names\n            label_candidates = ['label', 'target', 'class', 'is_malicious', 'attack_type', \n                               'attack', 'category', 'malware', 'phishing', 'threat', 'type', 'y']\n            has_label = any(col.lower() in [lc.lower() for lc in label_candidates] for col in df.columns)\n            \n            # Even without explicit labels, we can use for training (create synthetic labels based on dataset name)\n            datasets[name] = df\n            label_status = \"with labels\" if has_label else \"(no explicit labels - will create)\"\n            print(f\"  \u2713 {name}: {len(df)} samples, {len(df.columns)} features {label_status}\")\n        except Exception as e:\n            print(f\"  \u26a0 {name}: Error loading - {e}\")\n    else:\n        print(f\"  \u26a0 {name}: File not found\")\n\nprint(f\"\\n",
+    "\u2713 Loaded {len(datasets)} datasets for training\")\\n"
    ]
   },
   {
@@ -183,7 +159,7 @@
     "                return models\n",
     "        return ['random_forest']  # Default\n",
     "\n",
-    "print(\"✓ Model Configuration loaded\")\n",
+    "print(\"\u2713 Model Configuration loaded\")\n",
     "print(f\"   Available models: {list(ModelConfig.MODELS.keys())}\")"
    ]
   },
@@ -320,7 +296,7 @@
     "                best_score = metrics['f1_score']\n",
     "                best_model = model_type\n",
     "        \n",
-    "        print(f\"\\n  ✓ Best model: {best_model} (F1: {best_score:.4f})\")\n",
+    "        print(f\"\\n  \u2713 Best model: {best_model} (F1: {best_score:.4f})\")\n",
     "        \n",
     "        # Store results\n",
     "        self.trained_models[dataset_name] = {\n",
@@ -333,7 +309,7 @@
     "        return results\n",
     "\n",
     "trainer = CyberForgeTrainer()\n",
-    "print(\"✓ CyberForge Trainer initialized\")"
+    "print(\"\u2713 CyberForge Trainer initialized\")"
    ]
   },
   {
@@ -356,16 +332,16 @@
     "\n",
     "for name, df in datasets.items():\n",
     "    if 'label' not in df.columns:\n",
-    "        print(f\"⚠ Skipping {name}: no label column\")\n",
+    "        print(f\"\u26a0 Skipping {name}: no label column\")\n",
     "        continue\n",
     "    \n",
     "    try:\n",
     "        results = trainer.train_for_dataset(df, name)\n",
     "        all_results[name] = results\n",
     "    except Exception as e:\n",
-    "        print(f\"⚠ Error training {name}: {e}\")\n",
+    "        print(f\"\u26a0 Error training {name}: {e}\")\n",
     "\n",
-    "print(f\"\\n\\n✓ Trained models for {len(all_results)} datasets\")"
+    "print(f\"\\n\\n\u2713 Trained models for {len(all_results)} datasets\")"
    ]
   },
   {
@@ -467,7 +443,7 @@
     "        return registry\n",
     "\n",
     "serializer = ModelSerializer(MODELS_DIR)\n",
-    "print(\"✓ Model Serializer initialized\")"
+    "print(\"\u2713 Model Serializer initialized\")"
    ]
   },
   {
@@ -483,7 +459,7 @@
     "for dataset_name, model_data in trainer.trained_models.items():\n",
     "    print(f\"  Saving: {dataset_name}\")\n",
     "    saved = serializer.save_model(dataset_name, model_data)\n",
-    "    print(f\"    ✓ Saved {len(saved)} files\")\n",
+    "    print(f\"    \u2713 Saved {len(saved)} files\")\n",
     "\n",
     "# Create model registry\n",
     "registry = serializer.create_model_registry(trainer.trained_models)\n",
@@ -491,7 +467,7 @@
     "with open(registry_path, 'w') as f:\n",
     "    json.dump(registry, f, indent=2)\n",
     "\n",
-    "print(f\"\\n✓ Model registry saved to: {registry_path}\")"
+    "print(f\"\\n\u2713 Model registry saved to: {registry_path}\")"
    ]
   },
   {
@@ -627,7 +603,7 @@
     "with open(inference_path, 'w') as f:\n",
     "    f.write(inference_api_code)\n",
     "\n",
-    "print(f\"✓ Inference API saved to: {inference_path}\")"
+    "print(f\"\u2713 Inference API saved to: {inference_path}\")"
    ]
   },
   {
@@ -652,12 +628,12 @@
     "total_models = sum(len(m['models']) for m in trainer.trained_models.values())\n",
     "\n",
     "print(f\"\"\"\n",
-    "🤖 Training Summary:\n",
+    "\ud83e\udd16 Training Summary:\n",
     "   - Datasets trained: {len(trainer.trained_models)}\n",
     "   - Total models: {total_models}\n",
     "   - Output directory: {MODELS_DIR}\n",
     "\n",
-    "📊 Model Performance:\"\"\")\n",
+    "\ud83d\udcca Model Performance:\"\"\")\n",
     "\n",
     "for dataset, data in trainer.trained_models.items():\n",
     "    best = data['best_model']\n",
@@ -669,13 +645,13 @@
     "    print(f\"      Inference: {metrics['inference_time_ms']:.3f}ms\")\n",
     "\n",
     "print(f\"\"\"\n",
-    "📁 Output Files:\n",
+    "\ud83d\udcc1 Output Files:\n",
     "   - Model files: {MODELS_DIR}/<dataset>/<model>.pkl\n",
     "   - Registry: {MODELS_DIR}/model_registry.json\n",
     "   - Inference API: {MODELS_DIR}/inference.py\n",
     "\n",
     "Next step:\n",
-    "  → 04_agent_intelligence.ipynb\n",
+    "  \u2192 04_agent_intelligence.ipynb\n",
     "\"\"\")\n",
     "print(\"=\" * 60)"
    ]
@@ -688,4 +664,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}