Sync from GitHub Actions

.gitattributes

@@ -1,5 +1 @@
-notebooks/** linguist-documentation
-notebooks/*.ipynb linguist-documentation
-notebooks/**/*.ipynb linguist-documentation
-notebooks/** linguist-vendored
 *.ipynb linguist-documentation

notebooks/autocatalog-model-comparsion.ipynb

@@ -0,0 +1,2557 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "6c8e59c3",
+   "metadata": {},
+   "source": [
+    "### AutoCatalogAI — Reproducible Model Comparison\n",
+    "\n",
+    "This notebook creates a fair comparison between:\n",
+    "\n",
+    "1. **Majority Baseline**\n",
+    "2. **Frozen CLIP + Multi-task Heads (V1)**\n",
+    "3. **AutoCatalogAI V2 (fine-tuned production model)**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4ce28167",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [],
+   "source": [
+    "%pip uninstall -y torch torchvision torchaudio\n",
+    "\n",
+    "%pip install -q --no-cache-dir \\\n",
+    "    torch==2.5.1 \\\n",
+    "    torchvision==0.20.1 \\\n",
+    "    torchaudio==2.5.1 \\\n",
+    "    --index-url https://download.pytorch.org/whl/cu121\n",
+    "\n",
+    "%pip install -q \\\n",
+    "    transformers==4.46.3 \\\n",
+    "    datasets==3.1.0 \\\n",
+    "    huggingface-hub==0.26.2 \\\n",
+    "    scikit-learn==1.5.2 \\\n",
+    "    pandas==2.2.3 \\\n",
+    "    numpy==1.26.4 \\\n",
+    "    Pillow==11.0.0 \\\n",
+    "    tqdm==4.67.1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5aade65d",
+   "metadata": {},
+   "source": [
+    "## 2. Imports and Configuration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "6f8acb93",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:50:58.717046Z",
+     "iopub.status.busy": "2026-07-06T14:50:58.716189Z",
+     "iopub.status.idle": "2026-07-06T14:51:09.100500Z",
+     "shell.execute_reply": "2026-07-06T14:51:09.099494Z",
+     "shell.execute_reply.started": "2026-07-06T14:50:58.717016Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Python: 3.12.13\n",
+      "Torch: 2.5.1+cu121\n",
+      "Transformers: 4.46.3\n",
+      "CUDA available: True\n",
+      "GPU: Tesla P100-PCIE-16GB\n",
+      "CUDA runtime: 12.1\n"
+     ]
+    }
+   ],
+   "source": [
+    "import gc\n",
+    "import hashlib\n",
+    "import json\n",
+    "import os\n",
+    "import platform\n",
+    "import random\n",
+    "import time\n",
+    "from datetime import datetime, timezone\n",
+    "from pathlib import Path\n",
+    "\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "import torch.nn.functional as F\n",
+    "import transformers\n",
+    "from datasets import load_dataset\n",
+    "from huggingface_hub import hf_hub_download\n",
+    "from PIL import Image\n",
+    "from sklearn.metrics import accuracy_score, f1_score\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from torch.utils.data import DataLoader, Dataset\n",
+    "from tqdm.auto import tqdm\n",
+    "from transformers import CLIPImageProcessor, CLIPModel\n",
+    "\n",
+    "print(\"Python:\", platform.python_version())\n",
+    "print(\"Torch:\", torch.__version__)\n",
+    "print(\"Transformers:\", transformers.__version__)\n",
+    "print(\"CUDA available:\", torch.cuda.is_available())\n",
+    "\n",
+    "if torch.cuda.is_available():\n",
+    "    print(\"GPU:\", torch.cuda.get_device_name(0))\n",
+    "    print(\"CUDA runtime:\", torch.version.cuda)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bb00e756",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:51:12.581470Z",
+     "iopub.status.busy": "2026-07-06T14:51:12.580398Z",
+     "iopub.status.idle": "2026-07-06T14:51:12.589409Z",
+     "shell.execute_reply": "2026-07-06T14:51:12.588347Z",
+     "shell.execute_reply.started": "2026-07-06T14:51:12.581433Z"
+    },
+    "trusted": true
+   },
+   "outputs": [],
+   "source": [
+    "DATASET_NAME = \"ashraq/fashion-product-images-small\"\n",
+    "V1_REPO_ID = \"mohsin416/autocatalogai-clip-multitask\"\n",
+    "V2_REPO_ID = \"mohsin416/autocatalogai-clip-multitask-v2\"\n",
+    "\n",
+    "TASKS = [\n",
+    "    \"gender\",\n",
+    "    \"masterCategory\",\n",
+    "    \"subCategory\",\n",
+    "    \"articleType\",\n",
+    "    \"baseColour\",\n",
+    "    \"season\",\n",
+    "    \"usage\",\n",
+    "]\n",
+    "\n",
+    "SEED = 42\n",
+    "TRAIN_RATIO = 0.70\n",
+    "VAL_RATIO = 0.15\n",
+    "TEST_RATIO = 0.15\n",
+    "\n",
+    "BATCH_SIZE = 64\n",
+    "NUM_WORKERS = 0\n",
+    "\n",
+    "LATENCY_WARMUP_RUNS = 20\n",
+    "LATENCY_MEASURED_RUNS = 100\n",
+    "\n",
+    "COLOR_IMAGE_SIZE = 128\n",
+    "COLOR_FEATURE_DIM = 37\n",
+    "\n",
+    "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+    "ROOT_DIR = Path(\".\")\n",
+    "OUTPUT_DIR = ROOT_DIR / \"artifacts\" / \"evaluation\" / \"model_comparison\"\n",
+    "PROCESSED_DIR = ROOT_DIR / \"data\" / \"processed\"\n",
+    "\n",
+    "OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "PROCESSED_DIR.mkdir(parents=True, exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b03f3ae6",
+   "metadata": {},
+   "source": [
+    "## 3. Reproducibility"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "5027507b",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:51:14.492941Z",
+     "iopub.status.busy": "2026-07-06T14:51:14.492441Z",
+     "iopub.status.idle": "2026-07-06T14:51:14.500479Z",
+     "shell.execute_reply": "2026-07-06T14:51:14.499674Z",
+     "shell.execute_reply.started": "2026-07-06T14:51:14.492906Z"
+    },
+    "trusted": true
+   },
+   "outputs": [],
+   "source": [
+    "def set_seed(seed):\n",
+    "    random.seed(seed)\n",
+    "    np.random.seed(seed)\n",
+    "    torch.manual_seed(seed)\n",
+    "\n",
+    "    if torch.cuda.is_available():\n",
+    "        torch.cuda.manual_seed_all(seed)\n",
+    "\n",
+    "    torch.backends.cudnn.benchmark = False\n",
+    "    torch.backends.cudnn.deterministic = True\n",
+    "\n",
+    "\n",
+    "set_seed(SEED)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "08513a1e",
+   "metadata": {},
+   "source": [
+    "## 4. Download Published Model Metadata\n",
+    "\n",
+    "The comparison uses the exact published V1 and V2 checkpoints.  \n",
+    "V1 represents the frozen-CLIP baseline. V2 represents the production model.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "1ac434ae",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:51:15.077911Z",
+     "iopub.status.busy": "2026-07-06T14:51:15.076996Z",
+     "iopub.status.idle": "2026-07-06T14:51:16.662367Z",
+     "shell.execute_reply": "2026-07-06T14:51:16.661468Z",
+     "shell.execute_reply.started": "2026-07-06T14:51:15.077876Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Base model: openai/clip-vit-base-patch32\n",
+      "Task classes: {'gender': 5, 'masterCategory': 7, 'subCategory': 45, 'articleType': 141, 'baseColour': 46, 'season': 4, 'usage': 8}\n"
+     ]
+    }
+   ],
+   "source": [
+    "def load_json(path):\n",
+    "    with open(path, \"r\", encoding=\"utf-8\") as file:\n",
+    "        return json.load(file)\n",
+    "\n",
+    "\n",
+    "def safe_torch_load(path, map_location=\"cpu\"):\n",
+    "    try:\n",
+    "        return torch.load(\n",
+    "            path,\n",
+    "            map_location=map_location,\n",
+    "            weights_only=True,\n",
+    "        )\n",
+    "    except (TypeError, RuntimeError):\n",
+    "        return torch.load(\n",
+    "            path,\n",
+    "            map_location=map_location,\n",
+    "        )\n",
+    "\n",
+    "\n",
+    "def download_repo_artifacts(repo_id, include_rules=False):\n",
+    "    filenames = [\n",
+    "        \"model.pt\",\n",
+    "        \"config.json\",\n",
+    "        \"label_maps.json\",\n",
+    "    ]\n",
+    "\n",
+    "    if include_rules:\n",
+    "        filenames.append(\"consistency_rules.json\")\n",
+    "\n",
+    "    paths = {\n",
+    "        filename: hf_hub_download(\n",
+    "            repo_id=repo_id,\n",
+    "            filename=filename,\n",
+    "            repo_type=\"model\",\n",
+    "        )\n",
+    "        for filename in filenames\n",
+    "    }\n",
+    "\n",
+    "    artifacts = {\n",
+    "        \"checkpoint\": safe_torch_load(paths[\"model.pt\"]),\n",
+    "        \"config\": load_json(paths[\"config.json\"]),\n",
+    "        \"label_maps\": load_json(paths[\"label_maps.json\"]),\n",
+    "    }\n",
+    "\n",
+    "    if include_rules:\n",
+    "        artifacts[\"consistency_rules\"] = load_json(\n",
+    "            paths[\"consistency_rules.json\"]\n",
+    "        )\n",
+    "\n",
+    "    return artifacts\n",
+    "\n",
+    "\n",
+    "v1_artifacts = download_repo_artifacts(V1_REPO_ID)\n",
+    "v2_artifacts = download_repo_artifacts(\n",
+    "    V2_REPO_ID,\n",
+    "    include_rules=True,\n",
+    ")\n",
+    "\n",
+    "if v1_artifacts[\"label_maps\"] != v2_artifacts[\"label_maps\"]:\n",
+    "    raise ValueError(\n",
+    "        \"V1 and V2 label maps are different. \"\n",
+    "        \"A direct comparison would not be valid.\"\n",
+    "    )\n",
+    "\n",
+    "label_maps = v2_artifacts[\"label_maps\"]\n",
+    "v1_config = v1_artifacts[\"config\"]\n",
+    "v2_config = v2_artifacts[\"config\"]\n",
+    "\n",
+    "MODEL_NAME = (\n",
+    "    v2_config.get(\"base_model_name\")\n",
+    "    or v2_config.get(\"model_name\")\n",
+    "    or \"openai/clip-vit-base-patch32\"\n",
+    ")\n",
+    "\n",
+    "HIDDEN_DIM = int(v2_config.get(\"hidden_dim\", 512))\n",
+    "DROPOUT = float(v2_config.get(\"dropout\", 0.2))\n",
+    "\n",
+    "task_num_classes = {\n",
+    "    task: len(label_maps[task][\"label2id\"])\n",
+    "    for task in TASKS\n",
+    "}\n",
+    "\n",
+    "print(\"Base model:\", MODEL_NAME)\n",
+    "print(\"Task classes:\", task_num_classes)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "279e5552",
+   "metadata": {},
+   "source": [
+    "## 5. Load and Clean the Dataset\n",
+    "\n",
+    "The same validation rules used in the training notebook are applied here.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "bf1f57de",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:51:16.664194Z",
+     "iopub.status.busy": "2026-07-06T14:51:16.663758Z",
+     "iopub.status.idle": "2026-07-06T14:51:31.386130Z",
+     "shell.execute_reply": "2026-07-06T14:51:31.385294Z",
+     "shell.execute_reply.started": "2026-07-06T14:51:16.664164Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "daa46350514b4fd99e16003161697efd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Filter:   0%|          | 0/44072 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Raw samples: 44072\n",
+      "Clean samples: 44072\n",
+      "Dataset fingerprint: 5cacd020bfdb9ce5\n"
+     ]
+    }
+   ],
+   "source": [
+    "raw_dataset = load_dataset(\n",
+    "    DATASET_NAME,\n",
+    "    split=\"train\",\n",
+    ")\n",
+    "\n",
+    "missing_columns = [\n",
+    "    task\n",
+    "    for task in TASKS\n",
+    "    if task not in raw_dataset.column_names\n",
+    "]\n",
+    "\n",
+    "if \"image\" not in raw_dataset.column_names:\n",
+    "    raise ValueError(\"Dataset must contain an image column.\")\n",
+    "\n",
+    "if missing_columns:\n",
+    "    raise ValueError(\n",
+    "        f\"Dataset is missing task columns: {missing_columns}\"\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "def is_valid_row(row):\n",
+    "    if row.get(\"image\") is None:\n",
+    "        return False\n",
+    "\n",
+    "    for task in TASKS:\n",
+    "        value = row.get(task)\n",
+    "\n",
+    "        if value is None:\n",
+    "            return False\n",
+    "\n",
+    "        value = str(value).strip()\n",
+    "\n",
+    "        if not value:\n",
+    "            return False\n",
+    "\n",
+    "        if value not in label_maps[task][\"label2id\"]:\n",
+    "            return False\n",
+    "\n",
+    "    return True\n",
+    "\n",
+    "\n",
+    "clean_dataset = raw_dataset.filter(is_valid_row)\n",
+    "\n",
+    "print(\"Raw samples:\", len(raw_dataset))\n",
+    "print(\"Clean samples:\", len(clean_dataset))\n",
+    "print(\"Dataset fingerprint:\", clean_dataset._fingerprint)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c683a237",
+   "metadata": {},
+   "source": [
+    "## 6. Recreate the Exact 70/15/15 Split\n",
+    "\n",
+    "The split uses seed `42` and article-type stratification, matching the V2 training notebook.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "2f6aa87a",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:51:31.388042Z",
+     "iopub.status.busy": "2026-07-06T14:51:31.387795Z",
+     "iopub.status.idle": "2026-07-06T14:51:33.455071Z",
+     "shell.execute_reply": "2026-07-06T14:51:33.454144Z",
+     "shell.execute_reply.started": "2026-07-06T14:51:31.388018Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Train: 30850\n",
+      "Validation: 6611\n",
+      "Test: 6611\n",
+      "Test split SHA256: 106737acc60a436248d35e8a375a014d8b6e2f26f74b147b6444540b7781c0ec\n"
+     ]
+    }
+   ],
+   "source": [
+    "metadata = {\n",
+    "    task: [\n",
+    "        str(value).strip()\n",
+    "        for value in clean_dataset[task]\n",
+    "    ]\n",
+    "    for task in TASKS\n",
+    "}\n",
+    "\n",
+    "df = pd.DataFrame(metadata)\n",
+    "df[\"dataset_idx\"] = np.arange(len(clean_dataset))\n",
+    "\n",
+    "\n",
+    "def make_safe_stratify_labels(series):\n",
+    "    counts = series.value_counts()\n",
+    "\n",
+    "    return series.apply(\n",
+    "        lambda value: (\n",
+    "            value\n",
+    "            if counts[value] >= 2\n",
+    "            else \"__rare__\"\n",
+    "        )\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "all_indices = df.index.to_numpy()\n",
+    "\n",
+    "try:\n",
+    "    train_idx, temporary_idx = train_test_split(\n",
+    "        all_indices,\n",
+    "        test_size=VAL_RATIO + TEST_RATIO,\n",
+    "        random_state=SEED,\n",
+    "        stratify=make_safe_stratify_labels(\n",
+    "            df[\"articleType\"]\n",
+    "        ),\n",
+    "    )\n",
+    "except ValueError:\n",
+    "    train_idx, temporary_idx = train_test_split(\n",
+    "        all_indices,\n",
+    "        test_size=VAL_RATIO + TEST_RATIO,\n",
+    "        random_state=SEED,\n",
+    "    )\n",
+    "\n",
+    "temporary_df = df.loc[temporary_idx]\n",
+    "\n",
+    "try:\n",
+    "    val_idx, test_idx = train_test_split(\n",
+    "        temporary_idx,\n",
+    "        test_size=TEST_RATIO / (VAL_RATIO + TEST_RATIO),\n",
+    "        random_state=SEED,\n",
+    "        stratify=make_safe_stratify_labels(\n",
+    "            temporary_df[\"articleType\"]\n",
+    "        ),\n",
+    "    )\n",
+    "except ValueError:\n",
+    "    val_idx, test_idx = train_test_split(\n",
+    "        temporary_idx,\n",
+    "        test_size=TEST_RATIO / (VAL_RATIO + TEST_RATIO),\n",
+    "        random_state=SEED,\n",
+    "    )\n",
+    "\n",
+    "train_df = df.loc[train_idx].copy()\n",
+    "val_df = df.loc[val_idx].copy()\n",
+    "test_df = df.loc[test_idx].copy()\n",
+    "\n",
+    "assert set(train_idx).isdisjoint(val_idx)\n",
+    "assert set(train_idx).isdisjoint(test_idx)\n",
+    "assert set(val_idx).isdisjoint(test_idx)\n",
+    "\n",
+    "train_df.to_csv(\n",
+    "    PROCESSED_DIR / \"train_v2.csv\",\n",
+    "    index=False,\n",
+    ")\n",
+    "val_df.to_csv(\n",
+    "    PROCESSED_DIR / \"val_v2.csv\",\n",
+    "    index=False,\n",
+    ")\n",
+    "test_df.to_csv(\n",
+    "    PROCESSED_DIR / \"test_v2.csv\",\n",
+    "    index=False,\n",
+    ")\n",
+    "\n",
+    "test_index_bytes = np.asarray(\n",
+    "    sorted(test_idx),\n",
+    "    dtype=np.int64,\n",
+    ").tobytes()\n",
+    "\n",
+    "test_split_sha256 = hashlib.sha256(\n",
+    "    test_index_bytes\n",
+    ").hexdigest()\n",
+    "\n",
+    "print(\"Train:\", len(train_df))\n",
+    "print(\"Validation:\", len(val_df))\n",
+    "print(\"Test:\", len(test_df))\n",
+    "print(\"Test split SHA256:\", test_split_sha256)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3e08e2d0",
+   "metadata": {},
+   "source": [
+    "## 7. Test Dataset and DataLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "ee239b3e",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:51:33.456387Z",
+     "iopub.status.busy": "2026-07-06T14:51:33.456134Z",
+     "iopub.status.idle": "2026-07-06T14:51:33.553429Z",
+     "shell.execute_reply": "2026-07-06T14:51:33.552537Z",
+     "shell.execute_reply.started": "2026-07-06T14:51:33.456361Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Test samples: 6611\n"
+     ]
+    }
+   ],
+   "source": [
+    "processor = CLIPImageProcessor.from_pretrained(\n",
+    "    MODEL_NAME\n",
+    ")\n",
+    "\n",
+    "\n",
+    "def extract_color_features(\n",
+    "    image,\n",
+    "    image_size=COLOR_IMAGE_SIZE,\n",
+    "):\n",
+    "    image = image.convert(\"RGB\").resize(\n",
+    "        (image_size, image_size)\n",
+    "    )\n",
+    "\n",
+    "    margin = int(image_size * 0.10)\n",
+    "\n",
+    "    image = image.crop(\n",
+    "        (\n",
+    "            margin,\n",
+    "            margin,\n",
+    "            image_size - margin,\n",
+    "            image_size - margin,\n",
+    "        )\n",
+    "    )\n",
+    "\n",
+    "    rgb = np.asarray(\n",
+    "        image,\n",
+    "        dtype=np.float32,\n",
+    "    ) / 255.0\n",
+    "\n",
+    "    hsv = np.asarray(\n",
+    "        image.convert(\"HSV\"),\n",
+    "        dtype=np.float32,\n",
+    "    ) / 255.0\n",
+    "\n",
+    "    rgb_flat = rgb.reshape(-1, 3)\n",
+    "    hsv_flat = hsv.reshape(-1, 3)\n",
+    "\n",
+    "    saturation = hsv_flat[:, 1]\n",
+    "    value = hsv_flat[:, 2]\n",
+    "\n",
+    "    foreground_mask = (\n",
+    "        (saturation > 0.08)\n",
+    "        | (value < 0.92)\n",
+    "    )\n",
+    "\n",
+    "    if foreground_mask.sum() < 256:\n",
+    "        foreground_mask = np.ones(\n",
+    "            len(hsv_flat),\n",
+    "            dtype=bool,\n",
+    "        )\n",
+    "\n",
+    "    selected_rgb = rgb_flat[foreground_mask]\n",
+    "    selected_hsv = hsv_flat[foreground_mask]\n",
+    "\n",
+    "    hue_hist, _ = np.histogram(\n",
+    "        selected_hsv[:, 0],\n",
+    "        bins=12,\n",
+    "        range=(0.0, 1.0),\n",
+    "    )\n",
+    "\n",
+    "    saturation_hist, _ = np.histogram(\n",
+    "        selected_hsv[:, 1],\n",
+    "        bins=8,\n",
+    "        range=(0.0, 1.0),\n",
+    "    )\n",
+    "\n",
+    "    value_hist, _ = np.histogram(\n",
+    "        selected_hsv[:, 2],\n",
+    "        bins=8,\n",
+    "        range=(0.0, 1.0),\n",
+    "    )\n",
+    "\n",
+    "    hue_hist = hue_hist.astype(np.float32)\n",
+    "    saturation_hist = saturation_hist.astype(np.float32)\n",
+    "    value_hist = value_hist.astype(np.float32)\n",
+    "\n",
+    "    hue_hist /= max(hue_hist.sum(), 1.0)\n",
+    "    saturation_hist /= max(\n",
+    "        saturation_hist.sum(),\n",
+    "        1.0,\n",
+    "    )\n",
+    "    value_hist /= max(value_hist.sum(), 1.0)\n",
+    "\n",
+    "    rgb_mean = selected_rgb.mean(\n",
+    "        axis=0\n",
+    "    ).astype(np.float32)\n",
+    "\n",
+    "    rgb_std = selected_rgb.std(\n",
+    "        axis=0\n",
+    "    ).astype(np.float32)\n",
+    "\n",
+    "    rgb_median = np.median(\n",
+    "        selected_rgb,\n",
+    "        axis=0,\n",
+    "    ).astype(np.float32)\n",
+    "\n",
+    "    features = np.concatenate(\n",
+    "        [\n",
+    "            hue_hist,\n",
+    "            saturation_hist,\n",
+    "            value_hist,\n",
+    "            rgb_mean,\n",
+    "            rgb_std,\n",
+    "            rgb_median,\n",
+    "        ]\n",
+    "    ).astype(np.float32)\n",
+    "\n",
+    "    if features.shape[0] != COLOR_FEATURE_DIM:\n",
+    "        raise ValueError(\n",
+    "            f\"Expected {COLOR_FEATURE_DIM} color features, \"\n",
+    "            f\"got {features.shape[0]}\"\n",
+    "        )\n",
+    "\n",
+    "    return features\n",
+    "\n",
+    "\n",
+    "class ComparisonDataset(Dataset):\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        source_dataset,\n",
+    "        indices,\n",
+    "        processor,\n",
+    "        label_maps,\n",
+    "    ):\n",
+    "        self.source_dataset = source_dataset\n",
+    "        self.indices = list(map(int, indices))\n",
+    "        self.processor = processor\n",
+    "        self.label_maps = label_maps\n",
+    "\n",
+    "    def __len__(self):\n",
+    "        return len(self.indices)\n",
+    "\n",
+    "    def __getitem__(self, index):\n",
+    "        global_index = self.indices[index]\n",
+    "        item = self.source_dataset[global_index]\n",
+    "\n",
+    "        image = item[\"image\"]\n",
+    "\n",
+    "        if not isinstance(image, Image.Image):\n",
+    "            image = Image.open(image)\n",
+    "\n",
+    "        image = image.convert(\"RGB\")\n",
+    "\n",
+    "        pixel_values = self.processor(\n",
+    "            images=image,\n",
+    "            return_tensors=\"pt\",\n",
+    "        )[\"pixel_values\"].squeeze(0)\n",
+    "\n",
+    "        color_features = torch.tensor(\n",
+    "            extract_color_features(image),\n",
+    "            dtype=torch.float32,\n",
+    "        )\n",
+    "\n",
+    "        labels = {\n",
+    "            task: torch.tensor(\n",
+    "                self.label_maps[task][\"label2id\"][\n",
+    "                    str(item[task]).strip()\n",
+    "                ],\n",
+    "                dtype=torch.long,\n",
+    "            )\n",
+    "            for task in TASKS\n",
+    "        }\n",
+    "\n",
+    "        return {\n",
+    "            \"pixel_values\": pixel_values,\n",
+    "            \"color_features\": color_features,\n",
+    "            \"labels\": labels,\n",
+    "            \"global_index\": global_index,\n",
+    "        }\n",
+    "\n",
+    "\n",
+    "def collate_batch(batch):\n",
+    "    return {\n",
+    "        \"pixel_values\": torch.stack(\n",
+    "            [item[\"pixel_values\"] for item in batch]\n",
+    "        ),\n",
+    "        \"color_features\": torch.stack(\n",
+    "            [item[\"color_features\"] for item in batch]\n",
+    "        ),\n",
+    "        \"labels\": {\n",
+    "            task: torch.stack(\n",
+    "                [item[\"labels\"][task] for item in batch]\n",
+    "            )\n",
+    "            for task in TASKS\n",
+    "        },\n",
+    "        \"global_indices\": [\n",
+    "            item[\"global_index\"]\n",
+    "            for item in batch\n",
+    "        ],\n",
+    "    }\n",
+    "\n",
+    "\n",
+    "test_dataset = ComparisonDataset(\n",
+    "    clean_dataset,\n",
+    "    test_df[\"dataset_idx\"],\n",
+    "    processor,\n",
+    "    label_maps,\n",
+    ")\n",
+    "\n",
+    "test_loader = DataLoader(\n",
+    "    test_dataset,\n",
+    "    batch_size=BATCH_SIZE,\n",
+    "    shuffle=False,\n",
+    "    num_workers=NUM_WORKERS,\n",
+    "    pin_memory=torch.cuda.is_available(),\n",
+    "    collate_fn=collate_batch,\n",
+    ")\n",
+    "\n",
+    "print(\"Test samples:\", len(test_dataset))\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c97a0b5b",
+   "metadata": {},
+   "source": [
+    "## 8. Model Architectures"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "65d75793",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:51:33.555302Z",
+     "iopub.status.busy": "2026-07-06T14:51:33.554960Z",
+     "iopub.status.idle": "2026-07-06T14:51:33.569958Z",
+     "shell.execute_reply": "2026-07-06T14:51:33.568888Z",
+     "shell.execute_reply.started": "2026-07-06T14:51:33.555271Z"
+    },
+    "trusted": true
+   },
+   "outputs": [],
+   "source": [
+    "class ClassificationHead(nn.Module):\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        embedding_dim,\n",
+    "        num_classes,\n",
+    "        hidden_dim=512,\n",
+    "        dropout=0.2,\n",
+    "    ):\n",
+    "        super().__init__()\n",
+    "\n",
+    "        self.net = nn.Sequential(\n",
+    "            nn.LayerNorm(embedding_dim),\n",
+    "            nn.Linear(embedding_dim, hidden_dim),\n",
+    "            nn.GELU(),\n",
+    "            nn.Dropout(dropout),\n",
+    "            nn.Linear(hidden_dim, num_classes),\n",
+    "        )\n",
+    "\n",
+    "    def forward(self, features):\n",
+    "        return self.net(features)\n",
+    "\n",
+    "\n",
+    "class CLIPMultiTaskClassifier(nn.Module):\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        model_name,\n",
+    "        task_num_classes,\n",
+    "        hidden_dim=512,\n",
+    "        dropout=0.2,\n",
+    "    ):\n",
+    "        super().__init__()\n",
+    "\n",
+    "        self.clip = CLIPModel.from_pretrained(\n",
+    "            model_name\n",
+    "        )\n",
+    "\n",
+    "        embedding_dim = (\n",
+    "            self.clip.config.projection_dim\n",
+    "        )\n",
+    "\n",
+    "        self.heads = nn.ModuleDict(\n",
+    "            {\n",
+    "                task: ClassificationHead(\n",
+    "                    embedding_dim,\n",
+    "                    num_classes,\n",
+    "                    hidden_dim,\n",
+    "                    dropout,\n",
+    "                )\n",
+    "                for task, num_classes\n",
+    "                in task_num_classes.items()\n",
+    "            }\n",
+    "        )\n",
+    "\n",
+    "    def forward(self, pixel_values):\n",
+    "        image_features = (\n",
+    "            self.clip.get_image_features(\n",
+    "                pixel_values=pixel_values\n",
+    "            )\n",
+    "        )\n",
+    "\n",
+    "        image_features = F.normalize(\n",
+    "            image_features,\n",
+    "            dim=-1,\n",
+    "        )\n",
+    "\n",
+    "        return {\n",
+    "            task: head(image_features)\n",
+    "            for task, head in self.heads.items()\n",
+    "        }\n",
+    "\n",
+    "\n",
+    "class CLIPMultiTaskClassifierV2(nn.Module):\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        model_name,\n",
+    "        task_num_classes,\n",
+    "        hidden_dim=512,\n",
+    "        dropout=0.2,\n",
+    "        color_feature_dim=37,\n",
+    "    ):\n",
+    "        super().__init__()\n",
+    "\n",
+    "        self.clip = CLIPModel.from_pretrained(\n",
+    "            model_name\n",
+    "        )\n",
+    "\n",
+    "        embedding_dim = (\n",
+    "            self.clip.config.projection_dim\n",
+    "        )\n",
+    "\n",
+    "        self.heads = nn.ModuleDict(\n",
+    "            {\n",
+    "                task: ClassificationHead(\n",
+    "                    embedding_dim,\n",
+    "                    num_classes,\n",
+    "                    hidden_dim,\n",
+    "                    dropout,\n",
+    "                )\n",
+    "                for task, num_classes\n",
+    "                in task_num_classes.items()\n",
+    "            }\n",
+    "        )\n",
+    "\n",
+    "        self.master_to_sub = nn.Linear(\n",
+    "            task_num_classes[\"masterCategory\"],\n",
+    "            task_num_classes[\"subCategory\"],\n",
+    "            bias=False,\n",
+    "        )\n",
+    "\n",
+    "        self.sub_to_article = nn.Linear(\n",
+    "            task_num_classes[\"subCategory\"],\n",
+    "            task_num_classes[\"articleType\"],\n",
+    "            bias=False,\n",
+    "        )\n",
+    "\n",
+    "        self.article_to_season = nn.Linear(\n",
+    "            task_num_classes[\"articleType\"],\n",
+    "            task_num_classes[\"season\"],\n",
+    "            bias=False,\n",
+    "        )\n",
+    "\n",
+    "        self.article_to_usage = nn.Linear(\n",
+    "            task_num_classes[\"articleType\"],\n",
+    "            task_num_classes[\"usage\"],\n",
+    "            bias=False,\n",
+    "        )\n",
+    "\n",
+    "        self.color_branch = nn.Sequential(\n",
+    "            nn.LayerNorm(color_feature_dim),\n",
+    "            nn.Linear(color_feature_dim, 64),\n",
+    "            nn.GELU(),\n",
+    "            nn.Dropout(0.10),\n",
+    "            nn.Linear(\n",
+    "                64,\n",
+    "                task_num_classes[\"baseColour\"],\n",
+    "            ),\n",
+    "        )\n",
+    "\n",
+    "    def forward(\n",
+    "        self,\n",
+    "        pixel_values,\n",
+    "        color_features,\n",
+    "    ):\n",
+    "        image_features = (\n",
+    "            self.clip.get_image_features(\n",
+    "                pixel_values=pixel_values\n",
+    "            )\n",
+    "        )\n",
+    "\n",
+    "        image_features = F.normalize(\n",
+    "            image_features,\n",
+    "            dim=-1,\n",
+    "        )\n",
+    "\n",
+    "        outputs = {\n",
+    "            task: head(image_features)\n",
+    "            for task, head in self.heads.items()\n",
+    "        }\n",
+    "\n",
+    "        master_probs = torch.softmax(\n",
+    "            outputs[\"masterCategory\"].detach(),\n",
+    "            dim=1,\n",
+    "        )\n",
+    "\n",
+    "        outputs[\"subCategory\"] = (\n",
+    "            outputs[\"subCategory\"]\n",
+    "            + self.master_to_sub(master_probs)\n",
+    "        )\n",
+    "\n",
+    "        sub_probs = torch.softmax(\n",
+    "            outputs[\"subCategory\"].detach(),\n",
+    "            dim=1,\n",
+    "        )\n",
+    "\n",
+    "        outputs[\"articleType\"] = (\n",
+    "            outputs[\"articleType\"]\n",
+    "            + self.sub_to_article(sub_probs)\n",
+    "        )\n",
+    "\n",
+    "        article_probs = torch.softmax(\n",
+    "            outputs[\"articleType\"].detach(),\n",
+    "            dim=1,\n",
+    "        )\n",
+    "\n",
+    "        outputs[\"season\"] = (\n",
+    "            outputs[\"season\"]\n",
+    "            + self.article_to_season(article_probs)\n",
+    "        )\n",
+    "\n",
+    "        outputs[\"usage\"] = (\n",
+    "            outputs[\"usage\"]\n",
+    "            + self.article_to_usage(article_probs)\n",
+    "        )\n",
+    "\n",
+    "        outputs[\"baseColour\"] = (\n",
+    "            outputs[\"baseColour\"]\n",
+    "            + self.color_branch(color_features)\n",
+    "        )\n",
+    "\n",
+    "        return outputs\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6bcdc281",
+   "metadata": {},
+   "source": [
+    "## 9. Shared Metric Functions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "d76d6326",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:51:33.571811Z",
+     "iopub.status.busy": "2026-07-06T14:51:33.571394Z",
+     "iopub.status.idle": "2026-07-06T14:51:33.589627Z",
+     "shell.execute_reply": "2026-07-06T14:51:33.588833Z",
+     "shell.execute_reply.started": "2026-07-06T14:51:33.571769Z"
+    },
+    "trusted": true
+   },
+   "outputs": [],
+   "source": [
+    "def evaluate_predictions(\n",
+    "    y_true,\n",
+    "    y_pred,\n",
+    "    y_top3,\n",
+    "):\n",
+    "    task_metrics = {}\n",
+    "\n",
+    "    for task in TASKS:\n",
+    "        top3_matches = [\n",
+    "            int(true_label)\n",
+    "            in set(map(int, top3_labels))\n",
+    "            for true_label, top3_labels\n",
+    "            in zip(\n",
+    "                y_true[task],\n",
+    "                y_top3[task],\n",
+    "            )\n",
+    "        ]\n",
+    "\n",
+    "        task_metrics[task] = {\n",
+    "            \"accuracy\": float(\n",
+    "                accuracy_score(\n",
+    "                    y_true[task],\n",
+    "                    y_pred[task],\n",
+    "                )\n",
+    "            ),\n",
+    "            \"macro_f1\": float(\n",
+    "                f1_score(\n",
+    "                    y_true[task],\n",
+    "                    y_pred[task],\n",
+    "                    average=\"macro\",\n",
+    "                    zero_division=0,\n",
+    "                )\n",
+    "            ),\n",
+    "            \"top3_accuracy\": float(\n",
+    "                np.mean(top3_matches)\n",
+    "            ),\n",
+    "        }\n",
+    "\n",
+    "    exact_matches = np.ones(\n",
+    "        len(y_true[TASKS[0]]),\n",
+    "        dtype=bool,\n",
+    "    )\n",
+    "\n",
+    "    for task in TASKS:\n",
+    "        exact_matches &= (\n",
+    "            np.asarray(y_true[task])\n",
+    "            == np.asarray(y_pred[task])\n",
+    "        )\n",
+    "\n",
+    "    overall = {\n",
+    "        \"average_accuracy\": float(\n",
+    "            np.mean(\n",
+    "                [\n",
+    "                    task_metrics[task][\"accuracy\"]\n",
+    "                    for task in TASKS\n",
+    "                ]\n",
+    "            )\n",
+    "        ),\n",
+    "        \"average_macro_f1\": float(\n",
+    "            np.mean(\n",
+    "                [\n",
+    "                    task_metrics[task][\"macro_f1\"]\n",
+    "                    for task in TASKS\n",
+    "                ]\n",
+    "            )\n",
+    "        ),\n",
+    "        \"average_top3_accuracy\": float(\n",
+    "            np.mean(\n",
+    "                [\n",
+    "                    task_metrics[task][\"top3_accuracy\"]\n",
+    "                    for task in TASKS\n",
+    "                ]\n",
+    "            )\n",
+    "        ),\n",
+    "        \"exact_match_accuracy\": float(\n",
+    "            exact_matches.mean()\n",
+    "        ),\n",
+    "        \"samples\": int(len(exact_matches)),\n",
+    "    }\n",
+    "\n",
+    "    return {\n",
+    "        \"task_metrics\": task_metrics,\n",
+    "        \"overall_metrics\": overall,\n",
+    "    }\n",
+    "\n",
+    "\n",
+    "@torch.inference_mode()\n",
+    "def collect_model_predictions(\n",
+    "    model,\n",
+    "    loader,\n",
+    "    device,\n",
+    "    uses_color_features,\n",
+    "):\n",
+    "    model.eval()\n",
+    "\n",
+    "    y_true = {\n",
+    "        task: []\n",
+    "        for task in TASKS\n",
+    "    }\n",
+    "\n",
+    "    y_pred = {\n",
+    "        task: []\n",
+    "        for task in TASKS\n",
+    "    }\n",
+    "\n",
+    "    y_top3 = {\n",
+    "        task: []\n",
+    "        for task in TASKS\n",
+    "    }\n",
+    "\n",
+    "    global_indices = []\n",
+    "\n",
+    "    for batch in tqdm(\n",
+    "        loader,\n",
+    "        desc=\"Evaluating\",\n",
+    "        leave=False,\n",
+    "    ):\n",
+    "        pixel_values = batch[\n",
+    "            \"pixel_values\"\n",
+    "        ].to(device)\n",
+    "\n",
+    "        if uses_color_features:\n",
+    "            outputs = model(\n",
+    "                pixel_values,\n",
+    "                batch[\"color_features\"].to(device),\n",
+    "            )\n",
+    "        else:\n",
+    "            outputs = model(pixel_values)\n",
+    "\n",
+    "        for task in TASKS:\n",
+    "            probabilities = torch.softmax(\n",
+    "                outputs[task],\n",
+    "                dim=1,\n",
+    "            )\n",
+    "\n",
+    "            top_k = min(\n",
+    "                3,\n",
+    "                probabilities.shape[1],\n",
+    "            )\n",
+    "\n",
+    "            top_values, top_indices = torch.topk(\n",
+    "                probabilities,\n",
+    "                k=top_k,\n",
+    "                dim=1,\n",
+    "            )\n",
+    "\n",
+    "            predictions = top_indices[:, 0]\n",
+    "\n",
+    "            y_true[task].extend(\n",
+    "                batch[\"labels\"][task]\n",
+    "                .numpy()\n",
+    "                .tolist()\n",
+    "            )\n",
+    "\n",
+    "            y_pred[task].extend(\n",
+    "                predictions\n",
+    "                .cpu()\n",
+    "                .numpy()\n",
+    "                .tolist()\n",
+    "            )\n",
+    "\n",
+    "            y_top3[task].extend(\n",
+    "                top_indices\n",
+    "                .cpu()\n",
+    "                .numpy()\n",
+    "                .tolist()\n",
+    "            )\n",
+    "\n",
+    "        global_indices.extend(\n",
+    "            batch[\"global_indices\"]\n",
+    "        )\n",
+    "\n",
+    "    for task in TASKS:\n",
+    "        y_true[task] = np.asarray(\n",
+    "            y_true[task],\n",
+    "            dtype=np.int64,\n",
+    "        )\n",
+    "\n",
+    "        y_pred[task] = np.asarray(\n",
+    "            y_pred[task],\n",
+    "            dtype=np.int64,\n",
+    "        )\n",
+    "\n",
+    "        y_top3[task] = np.asarray(\n",
+    "            y_top3[task],\n",
+    "            dtype=np.int64,\n",
+    "        )\n",
+    "\n",
+    "    return (\n",
+    "        y_true,\n",
+    "        y_pred,\n",
+    "        y_top3,\n",
+    "        global_indices,\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "def get_state_dict(checkpoint):\n",
+    "    if \"model_state_dict\" in checkpoint:\n",
+    "        return checkpoint[\"model_state_dict\"]\n",
+    "\n",
+    "    if \"state_dict\" in checkpoint:\n",
+    "        return checkpoint[\"state_dict\"]\n",
+    "\n",
+    "    return checkpoint\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "07ff619d",
+   "metadata": {},
+   "source": [
+    "## 10. Majority Baseline\n",
+    "\n",
+    "Top-1 uses the most frequent training label for each task.  \n",
+    "Top-3 uses the three most frequent training labels for each task.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "3f5ca12c",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:51:39.992449Z",
+     "iopub.status.busy": "2026-07-06T14:51:39.991906Z",
+     "iopub.status.idle": "2026-07-06T14:51:40.154821Z",
+     "shell.execute_reply": "2026-07-06T14:51:40.154030Z",
+     "shell.execute_reply.started": "2026-07-06T14:51:39.992414Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "  \"average_accuracy\": 0.42628087386822827,\n",
+      "  \"average_macro_f1\": 0.07927692465823664,\n",
+      "  \"average_top3_accuracy\": 0.7344469174752037,\n",
+      "  \"exact_match_accuracy\": 0.009529571925578581,\n",
+      "  \"samples\": 6611\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "majority_top1 = {}\n",
+    "majority_top3 = {}\n",
+    "\n",
+    "for task in TASKS:\n",
+    "    counts = train_df[task].value_counts()\n",
+    "\n",
+    "    majority_top1[task] = (\n",
+    "        label_maps[task][\"label2id\"][\n",
+    "            counts.index[0]\n",
+    "        ]\n",
+    "    )\n",
+    "\n",
+    "    majority_top3[task] = [\n",
+    "        label_maps[task][\"label2id\"][label]\n",
+    "        for label in counts.index[:3]\n",
+    "    ]\n",
+    "\n",
+    "\n",
+    "majority_y_true = {\n",
+    "    task: test_df[task]\n",
+    "    .map(label_maps[task][\"label2id\"])\n",
+    "    .to_numpy(dtype=np.int64)\n",
+    "    for task in TASKS\n",
+    "}\n",
+    "\n",
+    "majority_y_pred = {\n",
+    "    task: np.full(\n",
+    "        len(test_df),\n",
+    "        majority_top1[task],\n",
+    "        dtype=np.int64,\n",
+    "    )\n",
+    "    for task in TASKS\n",
+    "}\n",
+    "\n",
+    "majority_y_top3 = {\n",
+    "    task: np.tile(\n",
+    "        np.asarray(\n",
+    "            majority_top3[task],\n",
+    "            dtype=np.int64,\n",
+    "        ),\n",
+    "        (len(test_df), 1),\n",
+    "    )\n",
+    "    for task in TASKS\n",
+    "}\n",
+    "\n",
+    "majority_metrics = evaluate_predictions(\n",
+    "    majority_y_true,\n",
+    "    majority_y_pred,\n",
+    "    majority_y_top3,\n",
+    ")\n",
+    "\n",
+    "print(\n",
+    "    json.dumps(\n",
+    "        majority_metrics[\"overall_metrics\"],\n",
+    "        indent=2,\n",
+    "    )\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "74860669",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:51:42.144938Z",
+     "iopub.status.busy": "2026-07-06T14:51:42.144442Z",
+     "iopub.status.idle": "2026-07-06T14:51:42.156211Z",
+     "shell.execute_reply": "2026-07-06T14:51:42.155268Z",
+     "shell.execute_reply.started": "2026-07-06T14:51:42.144902Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Majority lookup latency: 0.000565 ms\n"
+     ]
+    }
+   ],
+   "source": [
+    "def benchmark_majority_lookup(\n",
+    "    measured_runs=10000,\n",
+    "):\n",
+    "    start = time.perf_counter()\n",
+    "\n",
+    "    for _ in range(measured_runs):\n",
+    "        _ = {\n",
+    "            task: majority_top1[task]\n",
+    "            for task in TASKS\n",
+    "        }\n",
+    "\n",
+    "    elapsed_ms = (\n",
+    "        time.perf_counter() - start\n",
+    "    ) * 1000.0\n",
+    "\n",
+    "    return elapsed_ms / measured_runs\n",
+    "\n",
+    "\n",
+    "majority_latency_ms = benchmark_majority_lookup()\n",
+    "\n",
+    "print(\n",
+    "    \"Majority lookup latency:\",\n",
+    "    f\"{majority_latency_ms:.6f} ms\",\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b4db8d68",
+   "metadata": {},
+   "source": [
+    "## 11. Evaluate Frozen CLIP + Heads (V1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "f0ec76a3",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:51:44.397516Z",
+     "iopub.status.busy": "2026-07-06T14:51:44.396710Z",
+     "iopub.status.idle": "2026-07-06T14:52:39.006069Z",
+     "shell.execute_reply": "2026-07-06T14:52:39.005197Z",
+     "shell.execute_reply.started": "2026-07-06T14:51:44.397482Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Evaluating:   0%|          | 0/104 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "  \"average_accuracy\": 0.8335026038852995,\n",
+      "  \"average_macro_f1\": 0.6568447666058456,\n",
+      "  \"average_top3_accuracy\": 0.9711087581303888,\n",
+      "  \"exact_match_accuracy\": 0.27938284677053393,\n",
+      "  \"samples\": 6611\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "v1_checkpoint = v1_artifacts[\"checkpoint\"]\n",
+    "\n",
+    "v1_model_name = (\n",
+    "    v1_config.get(\"base_model_name\")\n",
+    "    or v1_config.get(\"model_name\")\n",
+    "    or v1_checkpoint.get(\"model_name\")\n",
+    "    or MODEL_NAME\n",
+    ")\n",
+    "\n",
+    "v1_hidden_dim = int(\n",
+    "    v1_config.get(\n",
+    "        \"hidden_dim\",\n",
+    "        v1_checkpoint.get(\n",
+    "            \"hidden_dim\",\n",
+    "            512,\n",
+    "        ),\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "v1_dropout = float(\n",
+    "    v1_config.get(\n",
+    "        \"dropout\",\n",
+    "        v1_checkpoint.get(\n",
+    "            \"dropout\",\n",
+    "            0.2,\n",
+    "        ),\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "v1_model = CLIPMultiTaskClassifier(\n",
+    "    model_name=v1_model_name,\n",
+    "    task_num_classes=task_num_classes,\n",
+    "    hidden_dim=v1_hidden_dim,\n",
+    "    dropout=v1_dropout,\n",
+    ")\n",
+    "\n",
+    "v1_model.load_state_dict(\n",
+    "    get_state_dict(v1_checkpoint),\n",
+    "    strict=True,\n",
+    ")\n",
+    "\n",
+    "v1_model.to(DEVICE)\n",
+    "v1_model.eval()\n",
+    "\n",
+    "(\n",
+    "    v1_y_true,\n",
+    "    v1_y_pred,\n",
+    "    v1_y_top3,\n",
+    "    v1_global_indices,\n",
+    ") = collect_model_predictions(\n",
+    "    v1_model,\n",
+    "    test_loader,\n",
+    "    DEVICE,\n",
+    "    uses_color_features=False,\n",
+    ")\n",
+    "\n",
+    "v1_metrics = evaluate_predictions(\n",
+    "    v1_y_true,\n",
+    "    v1_y_pred,\n",
+    "    v1_y_top3,\n",
+    ")\n",
+    "\n",
+    "print(\n",
+    "    json.dumps(\n",
+    "        v1_metrics[\"overall_metrics\"],\n",
+    "        indent=2,\n",
+    "    )\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "99fde9e8",
+   "metadata": {},
+   "source": [
+    "## 12. Fair Latency Benchmark"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "bc279ecd",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:52:43.888474Z",
+     "iopub.status.busy": "2026-07-06T14:52:43.888075Z",
+     "iopub.status.idle": "2026-07-06T14:52:44.676522Z",
+     "shell.execute_reply": "2026-07-06T14:52:44.675520Z",
+     "shell.execute_reply.started": "2026-07-06T14:52:43.888445Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'average_ms': 6.226557110003341, 'p50_ms': 6.155085000045801, 'p95_ms': 6.70613664992743, 'warmup_runs': 20, 'measured_runs': 100}\n"
+     ]
+    }
+   ],
+   "source": [
+    "@torch.inference_mode()\n",
+    "def benchmark_model_latency(\n",
+    "    model,\n",
+    "    sample,\n",
+    "    device,\n",
+    "    uses_color_features,\n",
+    "    warmup_runs=20,\n",
+    "    measured_runs=100,\n",
+    "):\n",
+    "    model.eval()\n",
+    "\n",
+    "    pixel_values = sample[\n",
+    "        \"pixel_values\"\n",
+    "    ].unsqueeze(0).to(device)\n",
+    "\n",
+    "    color_features = sample[\n",
+    "        \"color_features\"\n",
+    "    ].unsqueeze(0).to(device)\n",
+    "\n",
+    "    for _ in range(warmup_runs):\n",
+    "        if uses_color_features:\n",
+    "            model(\n",
+    "                pixel_values,\n",
+    "                color_features,\n",
+    "            )\n",
+    "        else:\n",
+    "            model(pixel_values)\n",
+    "\n",
+    "    if device.type == \"cuda\":\n",
+    "        torch.cuda.synchronize()\n",
+    "\n",
+    "    times = []\n",
+    "\n",
+    "    for _ in range(measured_runs):\n",
+    "        if device.type == \"cuda\":\n",
+    "            torch.cuda.synchronize()\n",
+    "\n",
+    "        start = time.perf_counter()\n",
+    "\n",
+    "        if uses_color_features:\n",
+    "            model(\n",
+    "                pixel_values,\n",
+    "                color_features,\n",
+    "            )\n",
+    "        else:\n",
+    "            model(pixel_values)\n",
+    "\n",
+    "        if device.type == \"cuda\":\n",
+    "            torch.cuda.synchronize()\n",
+    "\n",
+    "        times.append(\n",
+    "            (\n",
+    "                time.perf_counter() - start\n",
+    "            )\n",
+    "            * 1000.0\n",
+    "        )\n",
+    "\n",
+    "    return {\n",
+    "        \"average_ms\": float(\n",
+    "            np.mean(times)\n",
+    "        ),\n",
+    "        \"p50_ms\": float(\n",
+    "            np.percentile(times, 50)\n",
+    "        ),\n",
+    "        \"p95_ms\": float(\n",
+    "            np.percentile(times, 95)\n",
+    "        ),\n",
+    "        \"warmup_runs\": warmup_runs,\n",
+    "        \"measured_runs\": measured_runs,\n",
+    "    }\n",
+    "\n",
+    "\n",
+    "latency_sample = test_dataset[0]\n",
+    "\n",
+    "v1_latency = benchmark_model_latency(\n",
+    "    v1_model,\n",
+    "    latency_sample,\n",
+    "    DEVICE,\n",
+    "    uses_color_features=False,\n",
+    "    warmup_runs=LATENCY_WARMUP_RUNS,\n",
+    "    measured_runs=LATENCY_MEASURED_RUNS,\n",
+    ")\n",
+    "\n",
+    "print(v1_latency)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c5593fe6",
+   "metadata": {},
+   "source": [
+    "## 13. Evaluate AutoCatalogAI V2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "86188832",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:52:47.240600Z",
+     "iopub.status.busy": "2026-07-06T14:52:47.240043Z",
+     "iopub.status.idle": "2026-07-06T14:53:39.405651Z",
+     "shell.execute_reply": "2026-07-06T14:53:39.404397Z",
+     "shell.execute_reply.started": "2026-07-06T14:52:47.240567Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Evaluating:   0%|          | 0/104 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "  \"average_accuracy\": 0.8747758065561727,\n",
+      "  \"average_macro_f1\": 0.6740687827687263,\n",
+      "  \"average_top3_accuracy\": 0.981502690321326,\n",
+      "  \"exact_match_accuracy\": 0.4046286492209953,\n",
+      "  \"samples\": 6611\n",
+      "}\n",
+      "{'average_ms': 7.4737231199719645, 'p50_ms': 7.384413499949005, 'p95_ms': 8.047834450019309, 'warmup_runs': 20, 'measured_runs': 100}\n"
+     ]
+    }
+   ],
+   "source": [
+    "del v1_model\n",
+    "\n",
+    "gc.collect()\n",
+    "\n",
+    "if torch.cuda.is_available():\n",
+    "    torch.cuda.empty_cache()\n",
+    "\n",
+    "\n",
+    "v2_checkpoint = v2_artifacts[\"checkpoint\"]\n",
+    "\n",
+    "v2_model = CLIPMultiTaskClassifierV2(\n",
+    "    model_name=(\n",
+    "        v2_checkpoint.get(\n",
+    "            \"model_name\",\n",
+    "            MODEL_NAME,\n",
+    "        )\n",
+    "    ),\n",
+    "    task_num_classes=(\n",
+    "        v2_checkpoint.get(\n",
+    "            \"task_num_classes\",\n",
+    "            task_num_classes,\n",
+    "        )\n",
+    "    ),\n",
+    "    hidden_dim=int(\n",
+    "        v2_checkpoint.get(\n",
+    "            \"hidden_dim\",\n",
+    "            HIDDEN_DIM,\n",
+    "        )\n",
+    "    ),\n",
+    "    dropout=float(\n",
+    "        v2_checkpoint.get(\n",
+    "            \"dropout\",\n",
+    "            DROPOUT,\n",
+    "        )\n",
+    "    ),\n",
+    "    color_feature_dim=int(\n",
+    "        v2_checkpoint.get(\n",
+    "            \"color_feature_dim\",\n",
+    "            COLOR_FEATURE_DIM,\n",
+    "        )\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "v2_model.load_state_dict(\n",
+    "    get_state_dict(v2_checkpoint),\n",
+    "    strict=True,\n",
+    ")\n",
+    "\n",
+    "v2_model.to(DEVICE)\n",
+    "v2_model.eval()\n",
+    "\n",
+    "(\n",
+    "    v2_y_true,\n",
+    "    v2_y_pred,\n",
+    "    v2_y_top3,\n",
+    "    v2_global_indices,\n",
+    ") = collect_model_predictions(\n",
+    "    v2_model,\n",
+    "    test_loader,\n",
+    "    DEVICE,\n",
+    "    uses_color_features=True,\n",
+    ")\n",
+    "\n",
+    "v2_metrics = evaluate_predictions(\n",
+    "    v2_y_true,\n",
+    "    v2_y_pred,\n",
+    "    v2_y_top3,\n",
+    ")\n",
+    "\n",
+    "v2_latency = benchmark_model_latency(\n",
+    "    v2_model,\n",
+    "    latency_sample,\n",
+    "    DEVICE,\n",
+    "    uses_color_features=True,\n",
+    "    warmup_runs=LATENCY_WARMUP_RUNS,\n",
+    "    measured_runs=LATENCY_MEASURED_RUNS,\n",
+    ")\n",
+    "\n",
+    "print(\n",
+    "    json.dumps(\n",
+    "        v2_metrics[\"overall_metrics\"],\n",
+    "        indent=2,\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "print(v2_latency)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f8f35b7f",
+   "metadata": {},
+   "source": [
+    "## 14. Build the Comparison Table\n",
+    "\n",
+    "The table uses **raw model predictions** for a fair comparison.  \n",
+    "V2 consistency correction is not applied here because the other baselines do not use post-processing.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "303f42be",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:53:47.885588Z",
+     "iopub.status.busy": "2026-07-06T14:53:47.885021Z",
+     "iopub.status.idle": "2026-07-06T14:53:47.904148Z",
+     "shell.execute_reply": "2026-07-06T14:53:47.902890Z",
+     "shell.execute_reply.started": "2026-07-06T14:53:47.885552Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Model</th>\n",
+       "      <th>Avg Accuracy</th>\n",
+       "      <th>Avg Macro F1</th>\n",
+       "      <th>Top-3 Accuracy</th>\n",
+       "      <th>Exact Match</th>\n",
+       "      <th>Latency (ms)</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Majority Baseline</td>\n",
+       "      <td>42.63%</td>\n",
+       "      <td>7.93%</td>\n",
+       "      <td>73.44%</td>\n",
+       "      <td>0.95%</td>\n",
+       "      <td>0.001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Frozen CLIP + Heads (V1)</td>\n",
+       "      <td>83.35%</td>\n",
+       "      <td>65.68%</td>\n",
+       "      <td>97.11%</td>\n",
+       "      <td>27.94%</td>\n",
+       "      <td>6.227</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>AutoCatalogAI V2</td>\n",
+       "      <td>87.48%</td>\n",
+       "      <td>67.41%</td>\n",
+       "      <td>98.15%</td>\n",
+       "      <td>40.46%</td>\n",
+       "      <td>7.474</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                      Model Avg Accuracy Avg Macro F1 Top-3 Accuracy  \\\n",
+       "0         Majority Baseline       42.63%        7.93%         73.44%   \n",
+       "1  Frozen CLIP + Heads (V1)       83.35%       65.68%         97.11%   \n",
+       "2          AutoCatalogAI V2       87.48%       67.41%         98.15%   \n",
+       "\n",
+       "  Exact Match Latency (ms)  \n",
+       "0       0.95%        0.001  \n",
+       "1      27.94%        6.227  \n",
+       "2      40.46%        7.474  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "comparison_rows = []\n",
+    "\n",
+    "\n",
+    "def add_comparison_row(\n",
+    "    model_name,\n",
+    "    metrics,\n",
+    "    latency_ms,\n",
+    "):\n",
+    "    overall = metrics[\"overall_metrics\"]\n",
+    "\n",
+    "    comparison_rows.append(\n",
+    "        {\n",
+    "            \"Model\": model_name,\n",
+    "            \"Avg Accuracy\": overall[\n",
+    "                \"average_accuracy\"\n",
+    "            ],\n",
+    "            \"Avg Macro F1\": overall[\n",
+    "                \"average_macro_f1\"\n",
+    "            ],\n",
+    "            \"Top-3 Accuracy\": overall[\n",
+    "                \"average_top3_accuracy\"\n",
+    "            ],\n",
+    "            \"Exact Match\": overall[\n",
+    "                \"exact_match_accuracy\"\n",
+    "            ],\n",
+    "            \"Latency (ms)\": latency_ms,\n",
+    "        }\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "add_comparison_row(\n",
+    "    \"Majority Baseline\",\n",
+    "    majority_metrics,\n",
+    "    majority_latency_ms,\n",
+    ")\n",
+    "\n",
+    "add_comparison_row(\n",
+    "    \"Frozen CLIP + Heads (V1)\",\n",
+    "    v1_metrics,\n",
+    "    v1_latency[\"average_ms\"],\n",
+    ")\n",
+    "\n",
+    "add_comparison_row(\n",
+    "    \"AutoCatalogAI V2\",\n",
+    "    v2_metrics,\n",
+    "    v2_latency[\"average_ms\"],\n",
+    ")\n",
+    "\n",
+    "\n",
+    "comparison_df = pd.DataFrame(\n",
+    "    comparison_rows\n",
+    ")\n",
+    "\n",
+    "display_df = comparison_df.copy()\n",
+    "\n",
+    "for column in [\n",
+    "    \"Avg Accuracy\",\n",
+    "    \"Avg Macro F1\",\n",
+    "    \"Top-3 Accuracy\",\n",
+    "    \"Exact Match\",\n",
+    "]:\n",
+    "    display_df[column] = (\n",
+    "        display_df[column] * 100\n",
+    "    ).map(lambda value: f\"{value:.2f}%\")\n",
+    "\n",
+    "display_df[\"Latency (ms)\"] = (\n",
+    "    display_df[\"Latency (ms)\"]\n",
+    "    .map(lambda value: f\"{value:.3f}\")\n",
+    ")\n",
+    "\n",
+    "display(display_df)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3f4e3ac7",
+   "metadata": {},
+   "source": [
+    "## 15. Per-Task Metrics"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "6c7e45bc",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:54:06.403485Z",
+     "iopub.status.busy": "2026-07-06T14:54:06.402755Z",
+     "iopub.status.idle": "2026-07-06T14:54:06.419062Z",
+     "shell.execute_reply": "2026-07-06T14:54:06.418316Z",
+     "shell.execute_reply.started": "2026-07-06T14:54:06.403449Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Model</th>\n",
+       "      <th>Task</th>\n",
+       "      <th>Accuracy</th>\n",
+       "      <th>Macro F1</th>\n",
+       "      <th>Top-3 Accuracy</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Majority Baseline</td>\n",
+       "      <td>gender</td>\n",
+       "      <td>0.499168</td>\n",
+       "      <td>0.133185</td>\n",
+       "      <td>0.969899</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Majority Baseline</td>\n",
+       "      <td>masterCategory</td>\n",
+       "      <td>0.484496</td>\n",
+       "      <td>0.108790</td>\n",
+       "      <td>0.948571</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Majority Baseline</td>\n",
+       "      <td>subCategory</td>\n",
+       "      <td>0.349115</td>\n",
+       "      <td>0.012939</td>\n",
+       "      <td>0.584329</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Majority Baseline</td>\n",
+       "      <td>articleType</td>\n",
+       "      <td>0.160339</td>\n",
+       "      <td>0.002176</td>\n",
+       "      <td>0.297837</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Majority Baseline</td>\n",
+       "      <td>baseColour</td>\n",
+       "      <td>0.218726</td>\n",
+       "      <td>0.008158</td>\n",
+       "      <td>0.456209</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>Majority Baseline</td>\n",
+       "      <td>season</td>\n",
+       "      <td>0.489033</td>\n",
+       "      <td>0.164212</td>\n",
+       "      <td>0.939949</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>Majority Baseline</td>\n",
+       "      <td>usage</td>\n",
+       "      <td>0.783089</td>\n",
+       "      <td>0.125479</td>\n",
+       "      <td>0.944335</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>Frozen CLIP + Heads (V1)</td>\n",
+       "      <td>gender</td>\n",
+       "      <td>0.888670</td>\n",
+       "      <td>0.776600</td>\n",
+       "      <td>0.995613</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>Frozen CLIP + Heads (V1)</td>\n",
+       "      <td>masterCategory</td>\n",
+       "      <td>0.993344</td>\n",
+       "      <td>0.862994</td>\n",
+       "      <td>0.999395</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>Frozen CLIP + Heads (V1)</td>\n",
+       "      <td>subCategory</td>\n",
+       "      <td>0.942974</td>\n",
+       "      <td>0.710894</td>\n",
+       "      <td>0.994706</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>Frozen CLIP + Heads (V1)</td>\n",
+       "      <td>articleType</td>\n",
+       "      <td>0.841023</td>\n",
+       "      <td>0.663970</td>\n",
+       "      <td>0.971109</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>Frozen CLIP + Heads (V1)</td>\n",
+       "      <td>baseColour</td>\n",
+       "      <td>0.601119</td>\n",
+       "      <td>0.340958</td>\n",
+       "      <td>0.853880</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>Frozen CLIP + Heads (V1)</td>\n",
+       "      <td>season</td>\n",
+       "      <td>0.707003</td>\n",
+       "      <td>0.728013</td>\n",
+       "      <td>0.984874</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>Frozen CLIP + Heads (V1)</td>\n",
+       "      <td>usage</td>\n",
+       "      <td>0.860384</td>\n",
+       "      <td>0.514485</td>\n",
+       "      <td>0.998185</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>AutoCatalogAI V2</td>\n",
+       "      <td>gender</td>\n",
+       "      <td>0.919226</td>\n",
+       "      <td>0.812002</td>\n",
+       "      <td>0.997429</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>AutoCatalogAI V2</td>\n",
+       "      <td>masterCategory</td>\n",
+       "      <td>0.994403</td>\n",
+       "      <td>0.846441</td>\n",
+       "      <td>0.999395</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>AutoCatalogAI V2</td>\n",
+       "      <td>subCategory</td>\n",
+       "      <td>0.963546</td>\n",
+       "      <td>0.759314</td>\n",
+       "      <td>0.997277</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>AutoCatalogAI V2</td>\n",
+       "      <td>articleType</td>\n",
+       "      <td>0.876418</td>\n",
+       "      <td>0.663665</td>\n",
+       "      <td>0.981546</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>AutoCatalogAI V2</td>\n",
+       "      <td>baseColour</td>\n",
+       "      <td>0.697171</td>\n",
+       "      <td>0.364965</td>\n",
+       "      <td>0.906973</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>AutoCatalogAI V2</td>\n",
+       "      <td>season</td>\n",
+       "      <td>0.754803</td>\n",
+       "      <td>0.767844</td>\n",
+       "      <td>0.989714</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>AutoCatalogAI V2</td>\n",
+       "      <td>usage</td>\n",
+       "      <td>0.917864</td>\n",
+       "      <td>0.504250</td>\n",
+       "      <td>0.998185</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       Model            Task  Accuracy  Macro F1  \\\n",
+       "0          Majority Baseline          gender  0.499168  0.133185   \n",
+       "1          Majority Baseline  masterCategory  0.484496  0.108790   \n",
+       "2          Majority Baseline     subCategory  0.349115  0.012939   \n",
+       "3          Majority Baseline     articleType  0.160339  0.002176   \n",
+       "4          Majority Baseline      baseColour  0.218726  0.008158   \n",
+       "5          Majority Baseline          season  0.489033  0.164212   \n",
+       "6          Majority Baseline           usage  0.783089  0.125479   \n",
+       "7   Frozen CLIP + Heads (V1)          gender  0.888670  0.776600   \n",
+       "8   Frozen CLIP + Heads (V1)  masterCategory  0.993344  0.862994   \n",
+       "9   Frozen CLIP + Heads (V1)     subCategory  0.942974  0.710894   \n",
+       "10  Frozen CLIP + Heads (V1)     articleType  0.841023  0.663970   \n",
+       "11  Frozen CLIP + Heads (V1)      baseColour  0.601119  0.340958   \n",
+       "12  Frozen CLIP + Heads (V1)          season  0.707003  0.728013   \n",
+       "13  Frozen CLIP + Heads (V1)           usage  0.860384  0.514485   \n",
+       "14          AutoCatalogAI V2          gender  0.919226  0.812002   \n",
+       "15          AutoCatalogAI V2  masterCategory  0.994403  0.846441   \n",
+       "16          AutoCatalogAI V2     subCategory  0.963546  0.759314   \n",
+       "17          AutoCatalogAI V2     articleType  0.876418  0.663665   \n",
+       "18          AutoCatalogAI V2      baseColour  0.697171  0.364965   \n",
+       "19          AutoCatalogAI V2          season  0.754803  0.767844   \n",
+       "20          AutoCatalogAI V2           usage  0.917864  0.504250   \n",
+       "\n",
+       "    Top-3 Accuracy  \n",
+       "0         0.969899  \n",
+       "1         0.948571  \n",
+       "2         0.584329  \n",
+       "3         0.297837  \n",
+       "4         0.456209  \n",
+       "5         0.939949  \n",
+       "6         0.944335  \n",
+       "7         0.995613  \n",
+       "8         0.999395  \n",
+       "9         0.994706  \n",
+       "10        0.971109  \n",
+       "11        0.853880  \n",
+       "12        0.984874  \n",
+       "13        0.998185  \n",
+       "14        0.997429  \n",
+       "15        0.999395  \n",
+       "16        0.997277  \n",
+       "17        0.981546  \n",
+       "18        0.906973  \n",
+       "19        0.989714  \n",
+       "20        0.998185  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "per_task_rows = []\n",
+    "\n",
+    "models_and_metrics = {\n",
+    "    \"Majority Baseline\": majority_metrics,\n",
+    "    \"Frozen CLIP + Heads (V1)\": v1_metrics,\n",
+    "    \"AutoCatalogAI V2\": v2_metrics,\n",
+    "}\n",
+    "\n",
+    "for model_name, metrics in models_and_metrics.items():\n",
+    "    for task in TASKS:\n",
+    "        task_result = metrics[\n",
+    "            \"task_metrics\"\n",
+    "        ][task]\n",
+    "\n",
+    "        per_task_rows.append(\n",
+    "            {\n",
+    "                \"Model\": model_name,\n",
+    "                \"Task\": task,\n",
+    "                \"Accuracy\": task_result[\n",
+    "                    \"accuracy\"\n",
+    "                ],\n",
+    "                \"Macro F1\": task_result[\n",
+    "                    \"macro_f1\"\n",
+    "                ],\n",
+    "                \"Top-3 Accuracy\": task_result[\n",
+    "                    \"top3_accuracy\"\n",
+    "                ],\n",
+    "            }\n",
+    "        )\n",
+    "\n",
+    "\n",
+    "per_task_df = pd.DataFrame(\n",
+    "    per_task_rows\n",
+    ")\n",
+    "\n",
+    "display(per_task_df)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a877244a",
+   "metadata": {},
+   "source": [
+    "## 16. Save Reproducibility Artifacts"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "2f752616",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-07-06T14:54:13.885906Z",
+     "iopub.status.busy": "2026-07-06T14:54:13.885429Z",
+     "iopub.status.idle": "2026-07-06T14:54:13.905372Z",
+     "shell.execute_reply": "2026-07-06T14:54:13.904349Z",
+     "shell.execute_reply.started": "2026-07-06T14:54:13.885872Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "| Model | Avg Accuracy | Avg Macro F1 | Top-3 Accuracy | Exact Match | Latency |\n",
+      "|---|---:|---:|---:|---:|---:|\n",
+      "| Majority Baseline | 42.63% | 7.93% | 73.44% | 0.95% | 0.001 ms |\n",
+      "| Frozen CLIP + Heads (V1) | 83.35% | 65.68% | 97.11% | 27.94% | 6.227 ms |\n",
+      "| AutoCatalogAI V2 | 87.48% | 67.41% | 98.15% | 40.46% | 7.474 ms |\n",
+      "\n",
+      "> All models were evaluated on the same 6,611-image held-out test split. Metrics use raw model predictions. Latency is batch-size-1 model-forward time with preprocessing excluded.\n",
+      "\n",
+      "Saved files:\n",
+      "- artifacts/evaluation/model_comparison/README_model_comparison.md\n",
+      "- artifacts/evaluation/model_comparison/benchmark_metadata.json\n",
+      "- artifacts/evaluation/model_comparison/model_comparison.csv\n",
+      "- artifacts/evaluation/model_comparison/model_comparison.json\n",
+      "- artifacts/evaluation/model_comparison/model_comparison_per_task.csv\n"
+     ]
+    }
+   ],
+   "source": [
+    "comparison_csv_path = (\n",
+    "    OUTPUT_DIR\n",
+    "    / \"model_comparison.csv\"\n",
+    ")\n",
+    "\n",
+    "per_task_csv_path = (\n",
+    "    OUTPUT_DIR\n",
+    "    / \"model_comparison_per_task.csv\"\n",
+    ")\n",
+    "\n",
+    "comparison_json_path = (\n",
+    "    OUTPUT_DIR\n",
+    "    / \"model_comparison.json\"\n",
+    ")\n",
+    "\n",
+    "metadata_path = (\n",
+    "    OUTPUT_DIR\n",
+    "    / \"benchmark_metadata.json\"\n",
+    ")\n",
+    "\n",
+    "readme_table_path = (\n",
+    "    OUTPUT_DIR\n",
+    "    / \"README_model_comparison.md\"\n",
+    ")\n",
+    "\n",
+    "comparison_df.to_csv(\n",
+    "    comparison_csv_path,\n",
+    "    index=False,\n",
+    ")\n",
+    "\n",
+    "per_task_df.to_csv(\n",
+    "    per_task_csv_path,\n",
+    "    index=False,\n",
+    ")\n",
+    "\n",
+    "\n",
+    "comparison_payload = {\n",
+    "    row[\"Model\"]: {\n",
+    "        \"average_accuracy\": float(\n",
+    "            row[\"Avg Accuracy\"]\n",
+    "        ),\n",
+    "        \"average_macro_f1\": float(\n",
+    "            row[\"Avg Macro F1\"]\n",
+    "        ),\n",
+    "        \"average_top3_accuracy\": float(\n",
+    "            row[\"Top-3 Accuracy\"]\n",
+    "        ),\n",
+    "        \"exact_match_accuracy\": float(\n",
+    "            row[\"Exact Match\"]\n",
+    "        ),\n",
+    "        \"latency_ms\": float(\n",
+    "            row[\"Latency (ms)\"]\n",
+    "        ),\n",
+    "    }\n",
+    "    for row in comparison_rows\n",
+    "}\n",
+    "\n",
+    "with open(\n",
+    "    comparison_json_path,\n",
+    "    \"w\",\n",
+    "    encoding=\"utf-8\",\n",
+    ") as file:\n",
+    "    json.dump(\n",
+    "        comparison_payload,\n",
+    "        file,\n",
+    "        indent=2,\n",
+    "        ensure_ascii=False,\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "benchmark_metadata = {\n",
+    "    \"created_at_utc\": datetime.now(\n",
+    "        timezone.utc\n",
+    "    ).isoformat(),\n",
+    "    \"dataset_name\": DATASET_NAME,\n",
+    "    \"dataset_fingerprint\": (\n",
+    "        clean_dataset._fingerprint\n",
+    "    ),\n",
+    "    \"v1_repo_id\": V1_REPO_ID,\n",
+    "    \"v2_repo_id\": V2_REPO_ID,\n",
+    "    \"seed\": SEED,\n",
+    "    \"train_samples\": int(\n",
+    "        len(train_df)\n",
+    "    ),\n",
+    "    \"validation_samples\": int(\n",
+    "        len(val_df)\n",
+    "    ),\n",
+    "    \"test_samples\": int(\n",
+    "        len(test_df)\n",
+    "    ),\n",
+    "    \"test_split_sha256\": (\n",
+    "        test_split_sha256\n",
+    "    ),\n",
+    "    \"tasks\": TASKS,\n",
+    "    \"metric_policy\": (\n",
+    "        \"Raw model predictions for all rows\"\n",
+    "    ),\n",
+    "    \"latency_policy\": {\n",
+    "        \"batch_size\": 1,\n",
+    "        \"model_forward_only\": True,\n",
+    "        \"preprocessing_excluded\": True,\n",
+    "        \"warmup_runs\": (\n",
+    "            LATENCY_WARMUP_RUNS\n",
+    "        ),\n",
+    "        \"measured_runs\": (\n",
+    "            LATENCY_MEASURED_RUNS\n",
+    "        ),\n",
+    "    },\n",
+    "    \"environment\": {\n",
+    "        \"python\": platform.python_version(),\n",
+    "        \"torch\": torch.__version__,\n",
+    "        \"transformers\": (\n",
+    "            transformers.__version__\n",
+    "        ),\n",
+    "        \"device\": str(DEVICE),\n",
+    "        \"gpu\": (\n",
+    "            torch.cuda.get_device_name(0)\n",
+    "            if torch.cuda.is_available()\n",
+    "            else None\n",
+    "        ),\n",
+    "        \"cuda_runtime\": torch.version.cuda,\n",
+    "    },\n",
+    "}\n",
+    "\n",
+    "with open(\n",
+    "    metadata_path,\n",
+    "    \"w\",\n",
+    "    encoding=\"utf-8\",\n",
+    ") as file:\n",
+    "    json.dump(\n",
+    "        benchmark_metadata,\n",
+    "        file,\n",
+    "        indent=2,\n",
+    "        ensure_ascii=False,\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "def markdown_percent(value):\n",
+    "    return f\"{value * 100:.2f}%\"\n",
+    "\n",
+    "\n",
+    "markdown_lines = [\n",
+    "    \"| Model | Avg Accuracy | Avg Macro F1 | Top-3 Accuracy | Exact Match | Latency |\",\n",
+    "    \"|---|---:|---:|---:|---:|---:|\",\n",
+    "]\n",
+    "\n",
+    "for row in comparison_rows:\n",
+    "    latency_text = (\n",
+    "        f\"{row['Latency (ms)']:.3f} ms\"\n",
+    "    )\n",
+    "\n",
+    "    markdown_lines.append(\n",
+    "        \"| \"\n",
+    "        + \" | \".join(\n",
+    "            [\n",
+    "                row[\"Model\"],\n",
+    "                markdown_percent(\n",
+    "                    row[\"Avg Accuracy\"]\n",
+    "                ),\n",
+    "                markdown_percent(\n",
+    "                    row[\"Avg Macro F1\"]\n",
+    "                ),\n",
+    "                markdown_percent(\n",
+    "                    row[\"Top-3 Accuracy\"]\n",
+    "                ),\n",
+    "                markdown_percent(\n",
+    "                    row[\"Exact Match\"]\n",
+    "                ),\n",
+    "                latency_text,\n",
+    "            ]\n",
+    "        )\n",
+    "        + \" |\"\n",
+    "    )\n",
+    "\n",
+    "markdown_lines.extend(\n",
+    "    [\n",
+    "        \"\",\n",
+    "        (\n",
+    "            \"> All models were evaluated on the same \"\n",
+    "            f\"{len(test_df):,}-image held-out test split. \"\n",
+    "            \"Metrics use raw model predictions. \"\n",
+    "            \"Latency is batch-size-1 model-forward time \"\n",
+    "            \"with preprocessing excluded.\"\n",
+    "        ),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "readme_markdown = \"\\n\".join(\n",
+    "    markdown_lines\n",
+    ")\n",
+    "\n",
+    "readme_table_path.write_text(\n",
+    "    readme_markdown,\n",
+    "    encoding=\"utf-8\",\n",
+    ")\n",
+    "\n",
+    "print(readme_markdown)\n",
+    "\n",
+    "print(\"\\nSaved files:\")\n",
+    "\n",
+    "for path in sorted(\n",
+    "    OUTPUT_DIR.iterdir()\n",
+    "):\n",
+    "    print(\"-\", path)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}