{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "7b1e4ef3", "metadata": {}, "outputs": [], "source": [ "import sys\n", "from pathlib import Path\n", "\n", "ROOT = Path.cwd().parent\n", "\n", "if str(ROOT) not in sys.path:\n", " sys.path.append(str(ROOT))" ] }, { "cell_type": "code", "execution_count": null, "id": "977db8b5", "metadata": {}, "outputs": [], "source": [ "import ast, re, os, joblib\n", "from pathlib import Path\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt \n", "from sklearn.model_selection import train_test_split\n", "from sklearn.preprocessing import MultiLabelBinarizer\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from sklearn.multiclass import OneVsRestClassifier\n", "from sklearn.pipeline import Pipeline\n", "from sklearn.base import BaseEstimator, TransformerMixin\n", "from sklearn.metrics import (classification_report,hamming_loss,f1_score)\n", "from scipy.sparse import hstack, csr_matrix\n", "from catboost import CatBoostClassifier\n", "\n", "from app.ml.features import (CombinedFeatures,structural_features,FEATURE_NAMES,)\n", "pd.set_option('display.max_columns', 200)" ] }, { "cell_type": "code", "execution_count": null, "id": "69bdcc5e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Dataset shape: (2321, 2)\n", "Total Samples: 2321\n" ] } ], "source": [ "df1=pd.read_csv('../data/datasets/clean.csv')\n", "df2=pd.read_csv('../data/datasets/noisy.csv')\n", "df3=pd.read_csv('../data/datasets/real.csv')\n", "df4=pd.read_csv('../data/datasets/balance.csv')\n", "df5=pd.read_csv('../data/datasets/augment.csv')\n", "\n", "df = pd.concat([df1, df2, df3, df4, df5], ignore_index=True)\n", "\n", "print(\"Dataset shape:\", df.shape)\n", "print(\"Total Samples:\", len(df))" ] }, { "cell_type": "code", "execution_count": 4, "id": "3fee4af2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Index(['prompt', 'labels'], dtype='str')\n" ] } ], "source": [ "print(df.columns)" ] }, { "cell_type": "code", "execution_count": 5, "id": "d7708521", "metadata": {}, "outputs": [ { "data": { "application/vnd.microsoft.datawrangler.viewer.v0+json": { "columns": [ { "name": "index", "rawType": "str", "type": "string" }, { "name": "0", "rawType": "int64", "type": "integer" } ], "ref": "458585f5-8854-4b93-9546-69c5b143b71a", "rows": [ [ "prompt", "0" ], [ "labels", "0" ] ], "shape": { "columns": 1, "rows": 2 } }, "text/plain": [ "prompt 0\n", "labels 0\n", "dtype: int64" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.isnull().sum()" ] }, { "cell_type": "code", "execution_count": null, "id": "1e901c63", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Write a Java script to connect to a PostgreSQL database.\n", "----------------------------------------------------------------------------------------------------\n", "[\"coding\", \"realtime\", \"reasoning-light\", \"short-output\", \"easy\", \"cheap\"]\n", "Create an end-to-end MLOps deployment pipeline using Weights & Biases to reliably serve a distributed database.\n", "----------------------------------------------------------------------------------------------------\n", "[\"premium\", \"background\", \"long-output\", \"mlops\", \"hard\", \"reasoning-intensive\"]\n", "Analyze the performance implications, cost, and developer velocity of using Terraform and AWS ECS for high-throughput enterprise systems.\n", "----------------------------------------------------------------------------------------------------\n", "[\"interactive\", \"reasoning-moderate\", \"balanced\", \"analysis\", \"medium-output\", \"moderate\"]\n", "Write Pulumi code to provision a secure AWS VPC with private subnets, a NAT gateway, and restrictive security groups.\n", "----------------------------------------------------------------------------------------------------\n", "[\"interactive\", \"reasoning-moderate\", \"balanced\", \"infrastructure\", \"medium-output\", \"moderate\"]\n", "Write a Go script to sort an array using merge sort.\n", "----------------------------------------------------------------------------------------------------\n", "[\"coding\", \"realtime\", \"reasoning-light\", \"short-output\", \"easy\", \"cheap\"]\n" ] } ], "source": [ "for i in range(5):\n", " print(df.iloc[i]['prompt'])\n", " print(\"-\" * 100)\n", " print(df.iloc[i]['labels'])" ] }, { "cell_type": "code", "execution_count": 7, "id": "14953727", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "After Deduplication: 1859\n" ] } ], "source": [ "df = df.drop_duplicates(subset=[\"prompt\"])\n", "\n", "print(\"After Deduplication:\", len(df))" ] }, { "cell_type": "code", "execution_count": 8, "id": "08fa0d67", "metadata": {}, "outputs": [], "source": [ "df = df.sample(frac=1, random_state=42).reset_index(drop=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "c0bd9ed9", "metadata": {}, "outputs": [], "source": [ "print(\"Current Working Directory:\", os.getcwd())\n", "print(\"Does the target folder exist?\", os.path.exists(\"../data\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "ede80918", "metadata": {}, "outputs": [], "source": [ "base_dir = Path(os.getcwd())\n", "target_file = base_dir.parent / \"data\" / \"master_dataset.csv\"\n", "print(f\"Saving to: {target_file.resolve()}\")\n", "target_file.parent.mkdir(parents=True, exist_ok=True)\n", "df.to_csv(target_file, index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "59c21d7d", "metadata": {}, "outputs": [], "source": [ "df[\"labels\"] = df[\"labels\"].apply(ast.literal_eval)\n", "\n", "mlb = MultiLabelBinarizer()\n", "\n", "y = mlb.fit_transform(df[\"labels\"])\n", "\n", "label_counts = pd.DataFrame(y,columns=mlb.classes_).sum().sort_values()\n", "\n", "plt.figure(figsize=(10,8))\n", "label_counts.plot(kind=\"barh\")\n", "plt.title(\"Master Dataset Label Distribution\")\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "id": "0b94ca7e", "metadata": {}, "outputs": [], "source": [ "print(type(df[\"labels\"].iloc[4]))\n", "print(df[\"labels\"].iloc[4])" ] }, { "cell_type": "code", "execution_count": 13, "id": "1348ea7f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Labels (23): ['analysis', 'architecture', 'architecture-heavy', 'background', 'balanced', 'cheap', 'coding', 'debugging', 'easy', 'hard', 'infrastructure', 'interactive', 'long-output', 'medium-output', 'mlops', 'moderate', 'premium', 'realtime', 'reasoning-intensive', 'reasoning-light', 'reasoning-moderate', 'research', 'short-output']\n" ] } ], "source": [ "print(f\"Labels ({len(mlb.classes_)}): {list(mlb.classes_)}\")" ] }, { "cell_type": "code", "execution_count": 14, "id": "2d982c5d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Train: 1487 Test: 372\n" ] } ], "source": [ "X_train_raw, X_test_raw, y_train, y_test = train_test_split(\n", " df[\"prompt\"], y, test_size=0.2, random_state=42\n", ")\n", "print(f\"Train: {len(X_train_raw)} Test: {len(X_test_raw)}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "e84086fa", "metadata": {}, "outputs": [], "source": [ "feat_extractor = CombinedFeatures(max_features=5000,ngram_range=(1, 2),)" ] }, { "cell_type": "code", "execution_count": null, "id": "f707136a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Feature extraction complete.\n", "\n", "Train matrix shape: (1487, 5019)\n", "Test matrix shape : (372, 5019)\n" ] } ], "source": [ "X_train = feat_extractor.fit_transform(X_train_raw)\n", "X_test = feat_extractor.transform(X_test_raw)\n", "\n", "print(\"Feature extraction complete.\")\n", "print(\"\\nTrain matrix shape:\", X_train.shape)\n", "print(\"Test matrix shape :\", X_test.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "71bac789", "metadata": {}, "outputs": [], "source": [ "import tempfile\n", "\n", "def make_catboost():\n", "\n", " kwargs = dict(\n", " iterations=300,\n", " learning_rate=0.1,\n", " depth=6,\n", " loss_function=\"Logloss\",\n", " eval_metric=\"F1\",\n", " random_seed=42,\n", " verbose=0,\n", " auto_class_weights=\"Balanced\",\n", " train_dir=tempfile.gettempdir(),\n", " )\n", " return CatBoostClassifier(**kwargs)" ] }, { "cell_type": "code", "execution_count": null, "id": "1872329e", "metadata": {}, "outputs": [], "source": [ "base_clf = make_catboost()\n", "\n", "ovr = OneVsRestClassifier(base_clf,n_jobs=1,)" ] }, { "cell_type": "code", "execution_count": null, "id": "032a46cf", "metadata": {}, "outputs": [], "source": [ "print(\"Training CatBoost OvR classifier…\")\n", "ovr.fit(X_train, y_train)\n", "print(\"Training complete.\")" ] }, { "cell_type": "code", "execution_count": null, "id": "09ac2a0d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "======================================================================\n", "BASELINE CLASSIFICATION REPORT (threshold = 0.50)\n", "======================================================================\n", " precision recall f1-score support\n", "\n", " analysis 0.59 0.71 0.64 41\n", " architecture 0.53 0.58 0.56 43\n", " architecture-heavy 0.87 0.93 0.90 29\n", " background 0.90 0.86 0.88 91\n", " balanced 0.71 0.86 0.78 122\n", " cheap 0.70 0.82 0.76 90\n", " coding 0.79 0.86 0.82 105\n", " debugging 0.85 0.85 0.85 46\n", " easy 0.74 0.86 0.80 96\n", " hard 0.92 0.90 0.91 136\n", " infrastructure 0.74 0.66 0.70 77\n", " interactive 0.93 0.94 0.94 230\n", " long-output 0.90 0.87 0.88 104\n", " medium-output 0.85 0.88 0.86 177\n", " mlops 0.77 0.89 0.83 27\n", " moderate 0.71 0.79 0.75 140\n", " premium 0.92 0.85 0.88 114\n", " realtime 0.88 0.92 0.90 48\n", "reasoning-intensive 0.92 0.90 0.91 136\n", " reasoning-light 0.75 0.84 0.79 96\n", " reasoning-moderate 0.72 0.81 0.76 140\n", " research 0.71 0.68 0.70 22\n", " short-output 0.80 0.87 0.83 91\n", "\n", " micro avg 0.81 0.85 0.83 2201\n", " macro avg 0.79 0.83 0.81 2201\n", " weighted avg 0.81 0.85 0.83 2201\n", " samples avg 0.82 0.85 0.83 2201\n", "\n", "Baseline → Hamming Loss : 0.0907 Micro F1 : 0.8282 Macro F1 : 0.8094 Weighted F1 : 0.8300\n" ] }, { "data": { "text/plain": [ "0.8094470286424041" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "preds_default = ovr.predict(X_test)\n", "print(\"\\n\" + \"=\"*70)\n", "print(\"BASELINE CLASSIFICATION REPORT (threshold = 0.50)\")\n", "print(\"=\"*70)\n", "print(classification_report(y_test, preds_default, target_names=mlb.classes_, zero_division=0))\n", " \n", "def _metrics(y_true, y_pred, label=\"\"):\n", " hl = hamming_loss(y_true, y_pred)\n", " mif1 = f1_score(y_true, y_pred, average=\"micro\", zero_division=0)\n", " maf1 = f1_score(y_true, y_pred, average=\"macro\", zero_division=0)\n", " wf1 = f1_score(y_true, y_pred, average=\"weighted\", zero_division=0)\n", " print(f\"{label}Hamming Loss : {hl:.4f} Micro F1 : {mif1:.4f} \"\n", " f\"Macro F1 : {maf1:.4f} Weighted F1 : {wf1:.4f}\")\n", " return maf1\n", " \n", "_metrics(y_test, preds_default, \"Baseline → \")" ] }, { "cell_type": "code", "execution_count": null, "id": "c39ee384", "metadata": {}, "outputs": [], "source": [ "probas = ovr.predict_proba(X_test) # shape: (n_samples, n_labels)\n", "THRESHOLDS = np.full(len(mlb.classes_), 0.5)\n", " \n", "print(\"\\nTuning per-label thresholds…\")\n", "for i, label in enumerate(mlb.classes_):\n", " best_t, best_f1 = 0.5, 0.0\n", " for t in np.arange(0.10, 0.91, 0.05):\n", " col_pred = (probas[:, i] >= t).astype(int)\n", " lf1 = f1_score(y_test[:, i], col_pred, zero_division=0)\n", " if lf1 > best_f1:\n", " best_f1, best_t = lf1, t\n", " THRESHOLDS[i] = best_t\n", " print(f\" {label:<22} best_threshold={best_t:.2f} F1={best_f1:.3f}\")" ] }, { "cell_type": "code", "execution_count": 24, "id": "5cb94f45", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "======================================================================\n", "TUNED CLASSIFICATION REPORT (per-label optimal thresholds)\n", "======================================================================\n", " precision recall f1-score support\n", "\n", " analysis 0.56 0.85 0.68 41\n", " architecture 0.80 0.56 0.66 43\n", " architecture-heavy 1.00 0.90 0.95 29\n", " background 0.93 0.85 0.89 91\n", " balanced 0.75 0.83 0.79 122\n", " cheap 0.82 0.79 0.80 90\n", " coding 0.77 0.90 0.83 105\n", " debugging 0.90 0.80 0.85 46\n", " easy 0.88 0.77 0.82 96\n", " hard 0.92 0.90 0.91 136\n", " infrastructure 0.62 0.83 0.71 77\n", " interactive 0.91 0.98 0.94 230\n", " long-output 0.94 0.86 0.89 104\n", " medium-output 0.85 0.92 0.88 177\n", " mlops 1.00 0.85 0.92 27\n", " moderate 0.67 0.89 0.77 140\n", " premium 0.84 0.93 0.88 114\n", " realtime 0.90 0.92 0.91 48\n", "reasoning-intensive 0.92 0.90 0.91 136\n", " reasoning-light 0.88 0.76 0.82 96\n", " reasoning-moderate 0.65 0.92 0.76 140\n", " research 0.71 0.77 0.74 22\n", " short-output 0.86 0.81 0.84 91\n", "\n", " micro avg 0.82 0.87 0.84 2201\n", " macro avg 0.83 0.85 0.83 2201\n", " weighted avg 0.83 0.87 0.84 2201\n", " samples avg 0.83 0.87 0.85 2201\n", "\n", "Tuned → Hamming Loss : 0.0840 Micro F1 : 0.8419 Macro F1 : 0.8320 Weighted F1 : 0.8447\n" ] }, { "data": { "text/plain": [ "0.8319727417678422" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "preds_tuned = (probas >= THRESHOLDS).astype(int)\n", " \n", "print(\"\\n\" + \"=\"*70)\n", "print(\"TUNED CLASSIFICATION REPORT (per-label optimal thresholds)\")\n", "print(\"=\"*70)\n", "print(classification_report(y_test, preds_tuned, target_names=mlb.classes_, zero_division=0))\n", "_metrics(y_test, preds_tuned, \"Tuned → \")" ] }, { "cell_type": "code", "execution_count": null, "id": "662c5972", "metadata": {}, "outputs": [], "source": [ "base_f1 = f1_score(y_test, preds_default, average=None, zero_division=0)\n", "tuned_f1 = f1_score(y_test, preds_tuned, average=None, zero_division=0)\n", " \n", "order = np.argsort(tuned_f1)\n", "labels_sorted = np.array(mlb.classes_)[order]\n", "base_sorted = base_f1[order]\n", "tuned_sorted = tuned_f1[order]\n", " \n", "y_pos = np.arange(len(labels_sorted))\n", "fig, ax = plt.subplots(figsize=(11, 9))\n", "ax.barh(y_pos - 0.18, base_sorted, 0.35, label=\"Baseline (t=0.50)\", color=\"#5b9bd5\", alpha=0.85)\n", "ax.barh(y_pos + 0.18, tuned_sorted, 0.35, label=\"Tuned thresholds\", color=\"#ed7d31\", alpha=0.90)\n", "ax.set_yticks(y_pos)\n", "ax.set_yticklabels(labels_sorted)\n", "ax.axvline(0.7, color=\"red\", linestyle=\"--\", linewidth=1, label=\"0.70 target\")\n", "ax.set_xlabel(\"F1 Score\")\n", "ax.set_title(\"Per-Label F1 — Baseline vs Tuned Thresholds\")\n", "ax.legend(loc=\"lower right\")\n", "plt.tight_layout()\n", "plt.savefig(\"label_f1_tuned.png\", dpi=150)\n", "plt.show()\n", "print(\"Chart saved: label_f1_tuned.png\")" ] }, { "cell_type": "code", "execution_count": null, "id": "16a5f592", "metadata": {}, "outputs": [], "source": [ "MODELS_DIR = Path(\"../models\")\n", "MODELS_DIR.mkdir(exist_ok=True)\n", " \n", "joblib.dump(feat_extractor, MODELS_DIR / \"feature_extractor.pkl\")\n", "joblib.dump(ovr, MODELS_DIR / \"prompt_router.pkl\")\n", "joblib.dump(mlb, MODELS_DIR / \"label_binarizer.pkl\")\n", "np.save(MODELS_DIR / \"thresholds.npy\", THRESHOLDS)\n", "print(f\"\\nSaved artefacts → {MODELS_DIR.resolve()}\")\n", "print(f\" feature_extractor.pkl | prompt_router.pkl | \"\n", " f\"label_binarizer.pkl | thresholds.npy\")" ] }, { "cell_type": "code", "execution_count": null, "id": "559a32d1", "metadata": {}, "outputs": [], "source": [ "def predict_labels(prompts: list[str], use_tuned: bool = True) -> list[list[str]]:\n", " X = feat_extractor.transform(pd.Series(prompts))\n", " proba = ovr.predict_proba(X)\n", " thresh = THRESHOLDS if use_tuned else np.full(len(mlb.classes_), 0.5)\n", " pred = (proba >= thresh).astype(int)\n", " return [list(lbl) for lbl in mlb.inverse_transform(pred)]" ] }, { "cell_type": "code", "execution_count": null, "id": "dbb8d1a8", "metadata": {}, "outputs": [], "source": [ "def diagnose(prompt: str):\n", " feat_names = [\n", " \"word_count\", \"sentence_count\", \"punct_count\", \"avg_sent_len\",\n", " \"code_blocks\", \"uppercase_ratio\", \"question_count\",\n", " \"contains_code\", \"contains_arch\", \"contains_math\",\n", " \"contains_debug\", \"contains_research\",\n", " \"complexity_verbs\", \"scale_adj\", \"hard_domain_nouns\",\n", " \"contains_simple\", \"contains_realtime\", \"contains_long_output\",\n", " \"semantic_complexity\",\n", " ]\n", " struct = structural_features(pd.Series([prompt]))[0]\n", " print(f\"\\n{'─'*60}\")\n", " print(f\"DIAGNOSE: {prompt}\")\n", " print(f\"{'─'*60}\")\n", " print(\"Structural features:\")\n", " for name, val in zip(feat_names, struct):\n", " if val != 0:\n", " print(f\" {name:<25} = {val:.2f} ◄\")\n", " else:\n", " print(f\" {name:<25} = {val:.2f}\")\n", " \n", " X = feat_extractor.transform(pd.Series([prompt]))\n", " proba = ovr.predict_proba(X)[0]\n", " print(\"\\nLabel probabilities (sorted):\")\n", " label_proba = sorted(zip(mlb.classes_, proba, THRESHOLDS),\n", " key=lambda x: x[1], reverse=True)\n", " for label, prob, thresh in label_proba:\n", " marker = \"✓ PREDICTED\" if prob >= thresh else \"\"\n", " print(f\" {label:<22} prob={prob:.3f} thresh={thresh:.2f} {marker}\")\n", " \n", " " ] }, { "cell_type": "code", "execution_count": 30, "id": "55c716ee", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "======================================================================\n", "FEATURE DIAGNOSIS — previously mispredicted prompts\n", "======================================================================\n", "\n", "────────────────────────────────────────────────────────────\n", "DIAGNOSE: Build scalable recommendation infrastructure\n", "────────────────────────────────────────────────────────────\n", "Structural features:\n", " word_count = 4.00 ◄\n", " sentence_count = 1.00 ◄\n", " punct_count = 0.00\n", " avg_sent_len = 4.00 ◄\n", " code_blocks = 0.00\n", " uppercase_ratio = 0.02 ◄\n", " question_count = 0.00\n", " contains_code = 0.00\n", " contains_arch = 1.00 ◄\n", " contains_math = 0.00\n", " contains_debug = 0.00\n", " contains_research = 0.00\n", " complexity_verbs = 1.00 ◄\n", " scale_adj = 0.00\n", " hard_domain_nouns = 2.00 ◄\n", " contains_simple = 0.00\n", " contains_realtime = 0.00\n", " contains_long_output = 0.00\n", " semantic_complexity = 3.00 ◄\n", "\n", "Label probabilities (sorted):\n", " infrastructure prob=0.986 thresh=0.30 ✓ PREDICTED\n", " premium prob=0.957 thresh=0.25 ✓ PREDICTED\n", " hard prob=0.956 thresh=0.50 ✓ PREDICTED\n", " reasoning-intensive prob=0.956 thresh=0.50 ✓ PREDICTED\n", " medium-output prob=0.935 thresh=0.45 ✓ PREDICTED\n", " balanced prob=0.751 thresh=0.55 ✓ PREDICTED\n", " mlops prob=0.156 thresh=0.90 \n", " background prob=0.079 thresh=0.60 \n", " cheap prob=0.066 thresh=0.70 \n", " long-output prob=0.065 thresh=0.60 \n", " reasoning-light prob=0.060 thresh=0.70 \n", " moderate prob=0.050 thresh=0.35 \n", " easy prob=0.047 thresh=0.70 \n", " short-output prob=0.046 thresh=0.65 \n", " realtime prob=0.036 thresh=0.55 \n", " reasoning-moderate prob=0.034 thresh=0.30 \n", " coding prob=0.031 thresh=0.40 \n", " analysis prob=0.019 thresh=0.40 \n", " interactive prob=0.013 thresh=0.25 \n", " architecture-heavy prob=0.007 thresh=0.90 \n", " debugging prob=0.005 thresh=0.65 \n", " architecture prob=0.004 thresh=0.75 \n", " research prob=0.001 thresh=0.20 \n", "\n", "────────────────────────────────────────────────────────────\n", "DIAGNOSE: Explain TCP handshake simply\n", "────────────────────────────────────────────────────────────\n", "Structural features:\n", " word_count = 4.00 ◄\n", " sentence_count = 1.00 ◄\n", " punct_count = 0.00\n", " avg_sent_len = 4.00 ◄\n", " code_blocks = 0.00\n", " uppercase_ratio = 0.14 ◄\n", " question_count = 0.00\n", " contains_code = 0.00\n", " contains_arch = 0.00\n", " contains_math = 0.00\n", " contains_debug = 0.00\n", " contains_research = 0.00\n", " complexity_verbs = 0.00\n", " scale_adj = 0.00\n", " hard_domain_nouns = 0.00\n", " contains_simple = 1.00 ◄\n", " contains_realtime = 0.00\n", " contains_long_output = 0.00\n", " semantic_complexity = 0.00\n", "\n", "Label probabilities (sorted):\n", " interactive prob=0.985 thresh=0.25 ✓ PREDICTED\n", " reasoning-light prob=0.970 thresh=0.70 ✓ PREDICTED\n", " cheap prob=0.960 thresh=0.70 ✓ PREDICTED\n", " easy prob=0.950 thresh=0.70 ✓ PREDICTED\n", " short-output prob=0.769 thresh=0.65 ✓ PREDICTED\n", " analysis prob=0.367 thresh=0.40 \n", " medium-output prob=0.207 thresh=0.45 \n", " reasoning-moderate prob=0.181 thresh=0.30 \n", " balanced prob=0.118 thresh=0.55 \n", " moderate prob=0.117 thresh=0.35 \n", " coding prob=0.107 thresh=0.40 \n", " infrastructure prob=0.044 thresh=0.30 \n", " debugging prob=0.018 thresh=0.65 \n", " architecture prob=0.015 thresh=0.75 \n", " mlops prob=0.009 thresh=0.90 \n", " hard prob=0.008 thresh=0.50 \n", " reasoning-intensive prob=0.008 thresh=0.50 \n", " research prob=0.007 thresh=0.20 \n", " premium prob=0.006 thresh=0.25 \n", " realtime prob=0.005 thresh=0.55 \n", " background prob=0.004 thresh=0.60 \n", " long-output prob=0.002 thresh=0.60 \n", " architecture-heavy prob=0.000 thresh=0.90 \n" ] } ], "source": [ "print(\"\\n\" + \"=\"*70)\n", "print(\"FEATURE DIAGNOSIS — previously mispredicted prompts\")\n", "print(\"=\"*70)\n", "diagnose(\"Build scalable recommendation infrastructure\")\n", "diagnose(\"Explain TCP handshake simply\")" ] }, { "cell_type": "code", "execution_count": 31, "id": "728424bc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "======================================================================\n", "POLICY ENGINE ROUTING\n", "======================================================================\n", "\n", "PROMPT : Create a distributed orchestration workflow for ML deployment\n", "LABELS : ['hard', 'infrastructure', 'interactive', 'mlops', 'premium', 'reasoning-intensive']\n", "→ MODEL: nemotron-super-120b\n", "\n", "PROMPT : Fix segmentation fault in C++ linked list implementation\n", "LABELS : ['cheap', 'coding', 'easy', 'interactive', 'moderate', 'reasoning-light', 'reasoning-moderate', 'short-output']\n", "→ MODEL: nemotron-nano-30b\n", "\n", "PROMPT : Explain TCP handshake simply\n", "LABELS : ['cheap', 'easy', 'interactive', 'reasoning-light', 'short-output']\n", "→ MODEL: nemotron-nano-30b\n", "\n", "PROMPT : Build scalable recommendation infrastructure\n", "LABELS : ['balanced', 'hard', 'infrastructure', 'medium-output', 'premium', 'reasoning-intensive']\n", "→ MODEL: nemotron-super-120b\n", "\n", "PROMPT : Write a quick hello world in Python\n", "LABELS : ['cheap', 'coding', 'easy', 'interactive', 'reasoning-light', 'short-output']\n", "→ MODEL: nemotron-nano-30b\n", "\n", "PROMPT : Compare transformer architectures for long-context tasks\n", "LABELS : ['analysis', 'balanced', 'interactive', 'medium-output', 'moderate', 'reasoning-moderate', 'research']\n", "→ MODEL: nemotron-nano-30b\n" ] } ], "source": [ "MODEL_POLICY = {\n", " \"premium\": \"nemotron-super-120b\",\n", " \"background\": \"nemotron-super-120b\",\n", " \"hard\": \"nemotron-super-120b\",\n", " \"balanced\": \"nemotron-nano-30b\",\n", " \"moderate\": \"nemotron-nano-30b\",\n", " \"interactive\":\"nemotron-nano-30b\",\n", " \"realtime\": \"nemotron-nano-9b\",\n", " \"cheap\": \"nemotron-nano-9b\",\n", " \"easy\": \"nemotron-nano-9b\",\n", "}\n", "# Labels checked in descending priority — first match wins\n", "POLICY_PRIORITY = [\"premium\", \"background\", \"hard\",\n", " \"balanced\", \"moderate\", \"interactive\",\n", " \"realtime\", \"cheap\", \"easy\"]\n", " \n", "def route(prompt: str) -> tuple[str, list[str]]:\n", " labels = predict_labels([prompt])[0]\n", " for priority_label in POLICY_PRIORITY:\n", " if priority_label in labels:\n", " return MODEL_POLICY[priority_label], labels\n", " return \"nemotron-nano-30b\", labels # safe default\n", " \n", "print(\"\\n\" + \"=\"*70)\n", "print(\"POLICY ENGINE ROUTING\")\n", "print(\"=\"*70)\n", "for p in test_prompts:\n", " model, labels = route(p)\n", " print(f\"\\nPROMPT : {p}\")\n", " print(f\"LABELS : {sorted(labels)}\")\n", " print(f\"→ MODEL: {model}\")" ] } ], "metadata": { "kernelspec": { "display_name": "Adaptive-Token", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.12" } }, "nbformat": 4, "nbformat_minor": 5 }