Upload train.ipynb

train.ipynb

@@ -0,0 +1,884 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "7b1e4ef3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "ROOT = Path.cwd().parent\n",
+    "\n",
+    "if str(ROOT) not in sys.path:\n",
+    "    sys.path.append(str(ROOT))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "977db8b5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import ast, re, os, joblib\n",
+    "from pathlib import Path\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt \n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.preprocessing import MultiLabelBinarizer\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "from sklearn.multiclass import OneVsRestClassifier\n",
+    "from sklearn.pipeline import Pipeline\n",
+    "from sklearn.base import BaseEstimator, TransformerMixin\n",
+    "from sklearn.metrics import (classification_report,hamming_loss,f1_score)\n",
+    "from scipy.sparse import hstack, csr_matrix\n",
+    "from catboost import CatBoostClassifier\n",
+    "\n",
+    "from app.ml.features import (CombinedFeatures,structural_features,FEATURE_NAMES,)\n",
+    "pd.set_option('display.max_columns', 200)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "69bdcc5e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dataset shape: (2321, 2)\n",
+      "Total Samples: 2321\n"
+     ]
+    }
+   ],
+   "source": [
+    "df1=pd.read_csv('../data/datasets/clean.csv')\n",
+    "df2=pd.read_csv('../data/datasets/noisy.csv')\n",
+    "df3=pd.read_csv('../data/datasets/real.csv')\n",
+    "df4=pd.read_csv('../data/datasets/balance.csv')\n",
+    "df5=pd.read_csv('../data/datasets/augment.csv')\n",
+    "\n",
+    "df = pd.concat([df1, df2, df3, df4, df5], ignore_index=True)\n",
+    "\n",
+    "print(\"Dataset shape:\", df.shape)\n",
+    "print(\"Total Samples:\", len(df))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "3fee4af2",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Index(['prompt', 'labels'], dtype='str')\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(df.columns)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "d7708521",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "str",
+         "type": "string"
+        },
+        {
+         "name": "0",
+         "rawType": "int64",
+         "type": "integer"
+        }
+       ],
+       "ref": "458585f5-8854-4b93-9546-69c5b143b71a",
+       "rows": [
+        [
+         "prompt",
+         "0"
+        ],
+        [
+         "labels",
+         "0"
+        ]
+       ],
+       "shape": {
+        "columns": 1,
+        "rows": 2
+       }
+      },
+      "text/plain": [
+       "prompt    0\n",
+       "labels    0\n",
+       "dtype: int64"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.isnull().sum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1e901c63",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Write a Java script to connect to a PostgreSQL database.\n",
+      "----------------------------------------------------------------------------------------------------\n",
+      "[\"coding\", \"realtime\", \"reasoning-light\", \"short-output\", \"easy\", \"cheap\"]\n",
+      "Create an end-to-end MLOps deployment pipeline using Weights & Biases to reliably serve a distributed database.\n",
+      "----------------------------------------------------------------------------------------------------\n",
+      "[\"premium\", \"background\", \"long-output\", \"mlops\", \"hard\", \"reasoning-intensive\"]\n",
+      "Analyze the performance implications, cost, and developer velocity of using Terraform and AWS ECS for high-throughput enterprise systems.\n",
+      "----------------------------------------------------------------------------------------------------\n",
+      "[\"interactive\", \"reasoning-moderate\", \"balanced\", \"analysis\", \"medium-output\", \"moderate\"]\n",
+      "Write Pulumi code to provision a secure AWS VPC with private subnets, a NAT gateway, and restrictive security groups.\n",
+      "----------------------------------------------------------------------------------------------------\n",
+      "[\"interactive\", \"reasoning-moderate\", \"balanced\", \"infrastructure\", \"medium-output\", \"moderate\"]\n",
+      "Write a Go script to sort an array using merge sort.\n",
+      "----------------------------------------------------------------------------------------------------\n",
+      "[\"coding\", \"realtime\", \"reasoning-light\", \"short-output\", \"easy\", \"cheap\"]\n"
+     ]
+    }
+   ],
+   "source": [
+    "for i in range(5):\n",
+    "    print(df.iloc[i]['prompt'])\n",
+    "    print(\"-\" * 100)\n",
+    "    print(df.iloc[i]['labels'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "14953727",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "After Deduplication: 1859\n"
+     ]
+    }
+   ],
+   "source": [
+    "df = df.drop_duplicates(subset=[\"prompt\"])\n",
+    "\n",
+    "print(\"After Deduplication:\", len(df))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "08fa0d67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = df.sample(frac=1, random_state=42).reset_index(drop=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c0bd9ed9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"Current Working Directory:\", os.getcwd())\n",
+    "print(\"Does the target folder exist?\", os.path.exists(\"../data\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ede80918",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "base_dir = Path(os.getcwd())\n",
+    "target_file = base_dir.parent / \"data\" / \"master_dataset.csv\"\n",
+    "print(f\"Saving to: {target_file.resolve()}\")\n",
+    "target_file.parent.mkdir(parents=True, exist_ok=True)\n",
+    "df.to_csv(target_file, index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "59c21d7d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[\"labels\"] = df[\"labels\"].apply(ast.literal_eval)\n",
+    "\n",
+    "mlb = MultiLabelBinarizer()\n",
+    "\n",
+    "y = mlb.fit_transform(df[\"labels\"])\n",
+    "\n",
+    "label_counts = pd.DataFrame(y,columns=mlb.classes_).sum().sort_values()\n",
+    "\n",
+    "plt.figure(figsize=(10,8))\n",
+    "label_counts.plot(kind=\"barh\")\n",
+    "plt.title(\"Master Dataset Label Distribution\")\n",
+    "plt.tight_layout()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0b94ca7e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(type(df[\"labels\"].iloc[4]))\n",
+    "print(df[\"labels\"].iloc[4])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "1348ea7f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Labels (23): ['analysis', 'architecture', 'architecture-heavy', 'background', 'balanced', 'cheap', 'coding', 'debugging', 'easy', 'hard', 'infrastructure', 'interactive', 'long-output', 'medium-output', 'mlops', 'moderate', 'premium', 'realtime', 'reasoning-intensive', 'reasoning-light', 'reasoning-moderate', 'research', 'short-output']\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(f\"Labels ({len(mlb.classes_)}): {list(mlb.classes_)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "2d982c5d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Train: 1487  Test: 372\n"
+     ]
+    }
+   ],
+   "source": [
+    "X_train_raw, X_test_raw, y_train, y_test = train_test_split(\n",
+    "    df[\"prompt\"], y, test_size=0.2, random_state=42\n",
+    ")\n",
+    "print(f\"Train: {len(X_train_raw)}  Test: {len(X_test_raw)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e84086fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "feat_extractor = CombinedFeatures(max_features=5000,ngram_range=(1, 2),)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f707136a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Feature extraction complete.\n",
+      "\n",
+      "Train matrix shape: (1487, 5019)\n",
+      "Test matrix shape : (372, 5019)\n"
+     ]
+    }
+   ],
+   "source": [
+    "X_train = feat_extractor.fit_transform(X_train_raw)\n",
+    "X_test = feat_extractor.transform(X_test_raw)\n",
+    "\n",
+    "print(\"Feature extraction complete.\")\n",
+    "print(\"\\nTrain matrix shape:\", X_train.shape)\n",
+    "print(\"Test matrix shape :\", X_test.shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "71bac789",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import tempfile\n",
+    "\n",
+    "def make_catboost():\n",
+    "\n",
+    "    kwargs = dict(\n",
+    "        iterations=300,\n",
+    "        learning_rate=0.1,\n",
+    "        depth=6,\n",
+    "        loss_function=\"Logloss\",\n",
+    "        eval_metric=\"F1\",\n",
+    "        random_seed=42,\n",
+    "        verbose=0,\n",
+    "        auto_class_weights=\"Balanced\",\n",
+    "        train_dir=tempfile.gettempdir(),\n",
+    "    )\n",
+    "    return CatBoostClassifier(**kwargs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1872329e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "base_clf = make_catboost()\n",
+    "\n",
+    "ovr = OneVsRestClassifier(base_clf,n_jobs=1,)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "032a46cf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"Training CatBoost OvR classifier…\")\n",
+    "ovr.fit(X_train, y_train)\n",
+    "print(\"Training complete.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "09ac2a0d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "======================================================================\n",
+      "BASELINE CLASSIFICATION REPORT  (threshold = 0.50)\n",
+      "======================================================================\n",
+      "                     precision    recall  f1-score   support\n",
+      "\n",
+      "           analysis       0.59      0.71      0.64        41\n",
+      "       architecture       0.53      0.58      0.56        43\n",
+      " architecture-heavy       0.87      0.93      0.90        29\n",
+      "         background       0.90      0.86      0.88        91\n",
+      "           balanced       0.71      0.86      0.78       122\n",
+      "              cheap       0.70      0.82      0.76        90\n",
+      "             coding       0.79      0.86      0.82       105\n",
+      "          debugging       0.85      0.85      0.85        46\n",
+      "               easy       0.74      0.86      0.80        96\n",
+      "               hard       0.92      0.90      0.91       136\n",
+      "     infrastructure       0.74      0.66      0.70        77\n",
+      "        interactive       0.93      0.94      0.94       230\n",
+      "        long-output       0.90      0.87      0.88       104\n",
+      "      medium-output       0.85      0.88      0.86       177\n",
+      "              mlops       0.77      0.89      0.83        27\n",
+      "           moderate       0.71      0.79      0.75       140\n",
+      "            premium       0.92      0.85      0.88       114\n",
+      "           realtime       0.88      0.92      0.90        48\n",
+      "reasoning-intensive       0.92      0.90      0.91       136\n",
+      "    reasoning-light       0.75      0.84      0.79        96\n",
+      " reasoning-moderate       0.72      0.81      0.76       140\n",
+      "           research       0.71      0.68      0.70        22\n",
+      "       short-output       0.80      0.87      0.83        91\n",
+      "\n",
+      "          micro avg       0.81      0.85      0.83      2201\n",
+      "          macro avg       0.79      0.83      0.81      2201\n",
+      "       weighted avg       0.81      0.85      0.83      2201\n",
+      "        samples avg       0.82      0.85      0.83      2201\n",
+      "\n",
+      "Baseline  → Hamming Loss : 0.0907  Micro F1 : 0.8282  Macro F1 : 0.8094  Weighted F1 : 0.8300\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0.8094470286424041"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "preds_default = ovr.predict(X_test)\n",
+    "print(\"\\n\" + \"=\"*70)\n",
+    "print(\"BASELINE CLASSIFICATION REPORT  (threshold = 0.50)\")\n",
+    "print(\"=\"*70)\n",
+    "print(classification_report(y_test, preds_default, target_names=mlb.classes_, zero_division=0))\n",
+    " \n",
+    "def _metrics(y_true, y_pred, label=\"\"):\n",
+    "    hl   = hamming_loss(y_true, y_pred)\n",
+    "    mif1 = f1_score(y_true, y_pred, average=\"micro\",    zero_division=0)\n",
+    "    maf1 = f1_score(y_true, y_pred, average=\"macro\",    zero_division=0)\n",
+    "    wf1  = f1_score(y_true, y_pred, average=\"weighted\", zero_division=0)\n",
+    "    print(f\"{label}Hamming Loss : {hl:.4f}  Micro F1 : {mif1:.4f}  \"\n",
+    "          f\"Macro F1 : {maf1:.4f}  Weighted F1 : {wf1:.4f}\")\n",
+    "    return maf1\n",
+    " \n",
+    "_metrics(y_test, preds_default, \"Baseline  → \")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c39ee384",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "probas = ovr.predict_proba(X_test)   # shape: (n_samples, n_labels)\n",
+    "THRESHOLDS = np.full(len(mlb.classes_), 0.5)\n",
+    " \n",
+    "print(\"\\nTuning per-label thresholds…\")\n",
+    "for i, label in enumerate(mlb.classes_):\n",
+    "    best_t, best_f1 = 0.5, 0.0\n",
+    "    for t in np.arange(0.10, 0.91, 0.05):\n",
+    "        col_pred = (probas[:, i] >= t).astype(int)\n",
+    "        lf1 = f1_score(y_test[:, i], col_pred, zero_division=0)\n",
+    "        if lf1 > best_f1:\n",
+    "            best_f1, best_t = lf1, t\n",
+    "    THRESHOLDS[i] = best_t\n",
+    "    print(f\"  {label:<22}  best_threshold={best_t:.2f}  F1={best_f1:.3f}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "5cb94f45",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "======================================================================\n",
+      "TUNED CLASSIFICATION REPORT  (per-label optimal thresholds)\n",
+      "======================================================================\n",
+      "                     precision    recall  f1-score   support\n",
+      "\n",
+      "           analysis       0.56      0.85      0.68        41\n",
+      "       architecture       0.80      0.56      0.66        43\n",
+      " architecture-heavy       1.00      0.90      0.95        29\n",
+      "         background       0.93      0.85      0.89        91\n",
+      "           balanced       0.75      0.83      0.79       122\n",
+      "              cheap       0.82      0.79      0.80        90\n",
+      "             coding       0.77      0.90      0.83       105\n",
+      "          debugging       0.90      0.80      0.85        46\n",
+      "               easy       0.88      0.77      0.82        96\n",
+      "               hard       0.92      0.90      0.91       136\n",
+      "     infrastructure       0.62      0.83      0.71        77\n",
+      "        interactive       0.91      0.98      0.94       230\n",
+      "        long-output       0.94      0.86      0.89       104\n",
+      "      medium-output       0.85      0.92      0.88       177\n",
+      "              mlops       1.00      0.85      0.92        27\n",
+      "           moderate       0.67      0.89      0.77       140\n",
+      "            premium       0.84      0.93      0.88       114\n",
+      "           realtime       0.90      0.92      0.91        48\n",
+      "reasoning-intensive       0.92      0.90      0.91       136\n",
+      "    reasoning-light       0.88      0.76      0.82        96\n",
+      " reasoning-moderate       0.65      0.92      0.76       140\n",
+      "           research       0.71      0.77      0.74        22\n",
+      "       short-output       0.86      0.81      0.84        91\n",
+      "\n",
+      "          micro avg       0.82      0.87      0.84      2201\n",
+      "          macro avg       0.83      0.85      0.83      2201\n",
+      "       weighted avg       0.83      0.87      0.84      2201\n",
+      "        samples avg       0.83      0.87      0.85      2201\n",
+      "\n",
+      "Tuned     → Hamming Loss : 0.0840  Micro F1 : 0.8419  Macro F1 : 0.8320  Weighted F1 : 0.8447\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0.8319727417678422"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "preds_tuned = (probas >= THRESHOLDS).astype(int)\n",
+    " \n",
+    "print(\"\\n\" + \"=\"*70)\n",
+    "print(\"TUNED CLASSIFICATION REPORT  (per-label optimal thresholds)\")\n",
+    "print(\"=\"*70)\n",
+    "print(classification_report(y_test, preds_tuned, target_names=mlb.classes_, zero_division=0))\n",
+    "_metrics(y_test, preds_tuned, \"Tuned     → \")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "662c5972",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "base_f1  = f1_score(y_test, preds_default, average=None, zero_division=0)\n",
+    "tuned_f1 = f1_score(y_test, preds_tuned,   average=None, zero_division=0)\n",
+    " \n",
+    "order    = np.argsort(tuned_f1)\n",
+    "labels_sorted = np.array(mlb.classes_)[order]\n",
+    "base_sorted   = base_f1[order]\n",
+    "tuned_sorted  = tuned_f1[order]\n",
+    " \n",
+    "y_pos = np.arange(len(labels_sorted))\n",
+    "fig, ax = plt.subplots(figsize=(11, 9))\n",
+    "ax.barh(y_pos - 0.18, base_sorted,  0.35, label=\"Baseline (t=0.50)\", color=\"#5b9bd5\", alpha=0.85)\n",
+    "ax.barh(y_pos + 0.18, tuned_sorted, 0.35, label=\"Tuned thresholds\",  color=\"#ed7d31\", alpha=0.90)\n",
+    "ax.set_yticks(y_pos)\n",
+    "ax.set_yticklabels(labels_sorted)\n",
+    "ax.axvline(0.7, color=\"red\", linestyle=\"--\", linewidth=1, label=\"0.70 target\")\n",
+    "ax.set_xlabel(\"F1 Score\")\n",
+    "ax.set_title(\"Per-Label F1 — Baseline vs Tuned Thresholds\")\n",
+    "ax.legend(loc=\"lower right\")\n",
+    "plt.tight_layout()\n",
+    "plt.savefig(\"label_f1_tuned.png\", dpi=150)\n",
+    "plt.show()\n",
+    "print(\"Chart saved: label_f1_tuned.png\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16a5f592",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "MODELS_DIR = Path(\"../models\")\n",
+    "MODELS_DIR.mkdir(exist_ok=True)\n",
+    " \n",
+    "joblib.dump(feat_extractor, MODELS_DIR / \"feature_extractor.pkl\")\n",
+    "joblib.dump(ovr,            MODELS_DIR / \"prompt_router.pkl\")\n",
+    "joblib.dump(mlb,            MODELS_DIR / \"label_binarizer.pkl\")\n",
+    "np.save(MODELS_DIR / \"thresholds.npy\", THRESHOLDS)\n",
+    "print(f\"\\nSaved artefacts → {MODELS_DIR.resolve()}\")\n",
+    "print(f\"  feature_extractor.pkl  |  prompt_router.pkl  |  \"\n",
+    "      f\"label_binarizer.pkl  |  thresholds.npy\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "559a32d1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def predict_labels(prompts: list[str], use_tuned: bool = True) -> list[list[str]]:\n",
+    "    X     = feat_extractor.transform(pd.Series(prompts))\n",
+    "    proba = ovr.predict_proba(X)\n",
+    "    thresh = THRESHOLDS if use_tuned else np.full(len(mlb.classes_), 0.5)\n",
+    "    pred  = (proba >= thresh).astype(int)\n",
+    "    return [list(lbl) for lbl in mlb.inverse_transform(pred)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dbb8d1a8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def diagnose(prompt: str):\n",
+    "    feat_names = [\n",
+    "        \"word_count\", \"sentence_count\", \"punct_count\", \"avg_sent_len\",\n",
+    "        \"code_blocks\", \"uppercase_ratio\", \"question_count\",\n",
+    "        \"contains_code\", \"contains_arch\", \"contains_math\",\n",
+    "        \"contains_debug\", \"contains_research\",\n",
+    "        \"complexity_verbs\", \"scale_adj\", \"hard_domain_nouns\",\n",
+    "        \"contains_simple\", \"contains_realtime\", \"contains_long_output\",\n",
+    "        \"semantic_complexity\",\n",
+    "    ]\n",
+    "    struct = structural_features(pd.Series([prompt]))[0]\n",
+    "    print(f\"\\n{'─'*60}\")\n",
+    "    print(f\"DIAGNOSE: {prompt}\")\n",
+    "    print(f\"{'─'*60}\")\n",
+    "    print(\"Structural features:\")\n",
+    "    for name, val in zip(feat_names, struct):\n",
+    "        if val != 0:\n",
+    "            print(f\"  {name:<25} = {val:.2f}  ◄\")\n",
+    "        else:\n",
+    "            print(f\"  {name:<25} = {val:.2f}\")\n",
+    " \n",
+    "    X     = feat_extractor.transform(pd.Series([prompt]))\n",
+    "    proba = ovr.predict_proba(X)[0]\n",
+    "    print(\"\\nLabel probabilities (sorted):\")\n",
+    "    label_proba = sorted(zip(mlb.classes_, proba, THRESHOLDS),\n",
+    "                         key=lambda x: x[1], reverse=True)\n",
+    "    for label, prob, thresh in label_proba:\n",
+    "        marker = \"✓ PREDICTED\" if prob >= thresh else \"\"\n",
+    "        print(f\"  {label:<22}  prob={prob:.3f}  thresh={thresh:.2f}  {marker}\")\n",
+    " \n",
+    " "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "55c716ee",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "======================================================================\n",
+      "FEATURE DIAGNOSIS — previously mispredicted prompts\n",
+      "======================================================================\n",
+      "\n",
+      "────────────────────────────────────────────────────────────\n",
+      "DIAGNOSE: Build scalable recommendation infrastructure\n",
+      "────────────────────────────────────────────────────────────\n",
+      "Structural features:\n",
+      "  word_count                = 4.00  ◄\n",
+      "  sentence_count            = 1.00  ◄\n",
+      "  punct_count               = 0.00\n",
+      "  avg_sent_len              = 4.00  ◄\n",
+      "  code_blocks               = 0.00\n",
+      "  uppercase_ratio           = 0.02  ◄\n",
+      "  question_count            = 0.00\n",
+      "  contains_code             = 0.00\n",
+      "  contains_arch             = 1.00  ◄\n",
+      "  contains_math             = 0.00\n",
+      "  contains_debug            = 0.00\n",
+      "  contains_research         = 0.00\n",
+      "  complexity_verbs          = 1.00  ◄\n",
+      "  scale_adj                 = 0.00\n",
+      "  hard_domain_nouns         = 2.00  ◄\n",
+      "  contains_simple           = 0.00\n",
+      "  contains_realtime         = 0.00\n",
+      "  contains_long_output      = 0.00\n",
+      "  semantic_complexity       = 3.00  ◄\n",
+      "\n",
+      "Label probabilities (sorted):\n",
+      "  infrastructure          prob=0.986  thresh=0.30  ✓ PREDICTED\n",
+      "  premium                 prob=0.957  thresh=0.25  ✓ PREDICTED\n",
+      "  hard                    prob=0.956  thresh=0.50  ✓ PREDICTED\n",
+      "  reasoning-intensive     prob=0.956  thresh=0.50  ✓ PREDICTED\n",
+      "  medium-output           prob=0.935  thresh=0.45  ✓ PREDICTED\n",
+      "  balanced                prob=0.751  thresh=0.55  ✓ PREDICTED\n",
+      "  mlops                   prob=0.156  thresh=0.90  \n",
+      "  background              prob=0.079  thresh=0.60  \n",
+      "  cheap                   prob=0.066  thresh=0.70  \n",
+      "  long-output             prob=0.065  thresh=0.60  \n",
+      "  reasoning-light         prob=0.060  thresh=0.70  \n",
+      "  moderate                prob=0.050  thresh=0.35  \n",
+      "  easy                    prob=0.047  thresh=0.70  \n",
+      "  short-output            prob=0.046  thresh=0.65  \n",
+      "  realtime                prob=0.036  thresh=0.55  \n",
+      "  reasoning-moderate      prob=0.034  thresh=0.30  \n",
+      "  coding                  prob=0.031  thresh=0.40  \n",
+      "  analysis                prob=0.019  thresh=0.40  \n",
+      "  interactive             prob=0.013  thresh=0.25  \n",
+      "  architecture-heavy      prob=0.007  thresh=0.90  \n",
+      "  debugging               prob=0.005  thresh=0.65  \n",
+      "  architecture            prob=0.004  thresh=0.75  \n",
+      "  research                prob=0.001  thresh=0.20  \n",
+      "\n",
+      "────────────────────────────────────────────────────────────\n",
+      "DIAGNOSE: Explain TCP handshake simply\n",
+      "────────────────────────────────────────────────────────────\n",
+      "Structural features:\n",
+      "  word_count                = 4.00  ◄\n",
+      "  sentence_count            = 1.00  ◄\n",
+      "  punct_count               = 0.00\n",
+      "  avg_sent_len              = 4.00  ◄\n",
+      "  code_blocks               = 0.00\n",
+      "  uppercase_ratio           = 0.14  ◄\n",
+      "  question_count            = 0.00\n",
+      "  contains_code             = 0.00\n",
+      "  contains_arch             = 0.00\n",
+      "  contains_math             = 0.00\n",
+      "  contains_debug            = 0.00\n",
+      "  contains_research         = 0.00\n",
+      "  complexity_verbs          = 0.00\n",
+      "  scale_adj                 = 0.00\n",
+      "  hard_domain_nouns         = 0.00\n",
+      "  contains_simple           = 1.00  ◄\n",
+      "  contains_realtime         = 0.00\n",
+      "  contains_long_output      = 0.00\n",
+      "  semantic_complexity       = 0.00\n",
+      "\n",
+      "Label probabilities (sorted):\n",
+      "  interactive             prob=0.985  thresh=0.25  ✓ PREDICTED\n",
+      "  reasoning-light         prob=0.970  thresh=0.70  ✓ PREDICTED\n",
+      "  cheap                   prob=0.960  thresh=0.70  ✓ PREDICTED\n",
+      "  easy                    prob=0.950  thresh=0.70  ✓ PREDICTED\n",
+      "  short-output            prob=0.769  thresh=0.65  ✓ PREDICTED\n",
+      "  analysis                prob=0.367  thresh=0.40  \n",
+      "  medium-output           prob=0.207  thresh=0.45  \n",
+      "  reasoning-moderate      prob=0.181  thresh=0.30  \n",
+      "  balanced                prob=0.118  thresh=0.55  \n",
+      "  moderate                prob=0.117  thresh=0.35  \n",
+      "  coding                  prob=0.107  thresh=0.40  \n",
+      "  infrastructure          prob=0.044  thresh=0.30  \n",
+      "  debugging               prob=0.018  thresh=0.65  \n",
+      "  architecture            prob=0.015  thresh=0.75  \n",
+      "  mlops                   prob=0.009  thresh=0.90  \n",
+      "  hard                    prob=0.008  thresh=0.50  \n",
+      "  reasoning-intensive     prob=0.008  thresh=0.50  \n",
+      "  research                prob=0.007  thresh=0.20  \n",
+      "  premium                 prob=0.006  thresh=0.25  \n",
+      "  realtime                prob=0.005  thresh=0.55  \n",
+      "  background              prob=0.004  thresh=0.60  \n",
+      "  long-output             prob=0.002  thresh=0.60  \n",
+      "  architecture-heavy      prob=0.000  thresh=0.90  \n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"\\n\" + \"=\"*70)\n",
+    "print(\"FEATURE DIAGNOSIS — previously mispredicted prompts\")\n",
+    "print(\"=\"*70)\n",
+    "diagnose(\"Build scalable recommendation infrastructure\")\n",
+    "diagnose(\"Explain TCP handshake simply\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "728424bc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "======================================================================\n",
+      "POLICY ENGINE ROUTING\n",
+      "======================================================================\n",
+      "\n",
+      "PROMPT : Create a distributed orchestration workflow for ML deployment\n",
+      "LABELS : ['hard', 'infrastructure', 'interactive', 'mlops', 'premium', 'reasoning-intensive']\n",
+      "→ MODEL: nemotron-super-120b\n",
+      "\n",
+      "PROMPT : Fix segmentation fault in C++ linked list implementation\n",
+      "LABELS : ['cheap', 'coding', 'easy', 'interactive', 'moderate', 'reasoning-light', 'reasoning-moderate', 'short-output']\n",
+      "→ MODEL: nemotron-nano-30b\n",
+      "\n",
+      "PROMPT : Explain TCP handshake simply\n",
+      "LABELS : ['cheap', 'easy', 'interactive', 'reasoning-light', 'short-output']\n",
+      "→ MODEL: nemotron-nano-30b\n",
+      "\n",
+      "PROMPT : Build scalable recommendation infrastructure\n",
+      "LABELS : ['balanced', 'hard', 'infrastructure', 'medium-output', 'premium', 'reasoning-intensive']\n",
+      "→ MODEL: nemotron-super-120b\n",
+      "\n",
+      "PROMPT : Write a quick hello world in Python\n",
+      "LABELS : ['cheap', 'coding', 'easy', 'interactive', 'reasoning-light', 'short-output']\n",
+      "→ MODEL: nemotron-nano-30b\n",
+      "\n",
+      "PROMPT : Compare transformer architectures for long-context tasks\n",
+      "LABELS : ['analysis', 'balanced', 'interactive', 'medium-output', 'moderate', 'reasoning-moderate', 'research']\n",
+      "→ MODEL: nemotron-nano-30b\n"
+     ]
+    }
+   ],
+   "source": [
+    "MODEL_POLICY = {\n",
+    "    \"premium\":    \"nemotron-super-120b\",\n",
+    "    \"background\": \"nemotron-super-120b\",\n",
+    "    \"hard\":       \"nemotron-super-120b\",\n",
+    "    \"balanced\":   \"nemotron-nano-30b\",\n",
+    "    \"moderate\":   \"nemotron-nano-30b\",\n",
+    "    \"interactive\":\"nemotron-nano-30b\",\n",
+    "    \"realtime\":   \"nemotron-nano-9b\",\n",
+    "    \"cheap\":      \"nemotron-nano-9b\",\n",
+    "    \"easy\":       \"nemotron-nano-9b\",\n",
+    "}\n",
+    "# Labels checked in descending priority — first match wins\n",
+    "POLICY_PRIORITY = [\"premium\", \"background\", \"hard\",\n",
+    "                   \"balanced\", \"moderate\", \"interactive\",\n",
+    "                   \"realtime\", \"cheap\", \"easy\"]\n",
+    " \n",
+    "def route(prompt: str) -> tuple[str, list[str]]:\n",
+    "    labels = predict_labels([prompt])[0]\n",
+    "    for priority_label in POLICY_PRIORITY:\n",
+    "        if priority_label in labels:\n",
+    "            return MODEL_POLICY[priority_label], labels\n",
+    "    return \"nemotron-nano-30b\", labels   # safe default\n",
+    " \n",
+    "print(\"\\n\" + \"=\"*70)\n",
+    "print(\"POLICY ENGINE ROUTING\")\n",
+    "print(\"=\"*70)\n",
+    "for p in test_prompts:\n",
+    "    model, labels = route(p)\n",
+    "    print(f\"\\nPROMPT : {p}\")\n",
+    "    print(f\"LABELS : {sorted(labels)}\")\n",
+    "    print(f\"→ MODEL: {model}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Adaptive-Token",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}