{ "cells": [ { "cell_type": "markdown", "id": "85c327d2", "metadata": {}, "source": [ "# PISCES quickstart\n", "\n", "PISCES (ARIES #1) against the official e-means of `faiss.contrib.e_means`. Both start from the same\n", "k rows; we compare mean squared error on held-out rows and wall-clock time. This uses a 200,000-row\n", "slice of DEEP\\* (downloaded with fingerprint checks) and k = 2,048, so it takes a few minutes on a laptop CPU." ] }, { "cell_type": "code", "execution_count": 1, "id": "7f427f79", "metadata": { "execution": { "iopub.execute_input": "2026-11-18T00:38:04.054586Z", "iopub.status.busy": "2026-11-18T00:38:04.054465Z", "iopub.status.idle": "2026-11-18T00:38:04.435329Z", "shell.execute_reply": "2026-11-18T00:38:04.433700Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "ready\n" ] } ], "source": [ "# Get the code (skip when running inside a clone of this repository)\n", "import os, sys, subprocess\n", "if not os.path.exists('aries') and os.path.exists('../aries'):\n", " os.chdir('..') # opened from notebooks/ inside a clone\n", "if not os.path.exists('aries'):\n", " subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', 'huggingface_hub'], check=True)\n", " from huggingface_hub import snapshot_download\n", " root = snapshot_download('Nabidnur/PISCES', local_dir='pisces_repo')\n", " os.chdir(root)\n", "subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', '--disable-pip-version-check', '-r', 'requirements.txt'], check=True)\n", "sys.path.insert(0, os.getcwd()); print('ready')" ] }, { "cell_type": "code", "execution_count": 2, "id": "09f24a74", "metadata": { "execution": { "iopub.execute_input": "2026-11-18T00:38:04.436803Z", "iopub.status.busy": "2026-11-18T00:38:04.436660Z", "iopub.status.idle": "2026-11-18T00:38:04.595549Z", "shell.execute_reply": "2026-11-18T00:38:04.594889Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "DEEP* 1M already in data/deep1b; fingerprints match the paper's\n", "(200000, 96) (5000, 96)\n" ] } ], "source": [ "import os, subprocess, sys\n", "subprocess.run([sys.executable, 'benchmarks/get_deep1m.py', 'data/deep1b'], check=True)\n", "import numpy as np\n", "x = np.ascontiguousarray(np.load('data/deep1b/train_1M.npy', mmap_mode='r')[:200_000])\n", "val = np.load('data/deep1b/val.npy')\n", "print(x.shape, val.shape)" ] }, { "cell_type": "code", "execution_count": 3, "id": "577a520a", "metadata": { "execution": { "iopub.execute_input": "2026-11-18T00:38:04.597106Z", "iopub.status.busy": "2026-11-18T00:38:04.596661Z", "iopub.status.idle": "2026-11-18T00:38:28.950319Z", "shell.execute_reply": "2026-11-18T00:38:28.949117Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "seed 1 epochs 1: e-means 0.45583 (0.9s) PISCES 0.45208 (1.5s) change -0.82%\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "seed 1 epochs 3: e-means 0.45079 (3.1s) PISCES 0.44921 (3.0s) change -0.35%\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "seed 2 epochs 1: e-means 0.45764 (1.0s) PISCES 0.45328 (1.3s) change -0.95%\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "seed 2 epochs 3: e-means 0.45022 (2.8s) PISCES 0.44922 (2.4s) change -0.22%\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "seed 3 epochs 1: e-means 0.45633 (1.0s) PISCES 0.45430 (1.1s) change -0.44%\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "seed 3 epochs 3: e-means 0.45083 (3.1s) PISCES 0.44903 (2.9s) change -0.40%\n" ] } ], "source": [ "import time, faiss\n", "from faiss.contrib.e_means import NumpyEmeans\n", "from aries import pisces\n", "\n", "def mse(c):\n", " D, _ = faiss.knn(val, np.ascontiguousarray(c, dtype='float32'), 1)\n", " return float(D.astype('float64').mean())\n", "\n", "k = 2048\n", "rows = []\n", "for seed in [1, 2, 3]:\n", " init = x[np.random.RandomState(seed).permutation(len(x))[:k]].copy()\n", " for epochs in [1, 3]:\n", " t = time.time(); c, _ = NumpyEmeans(x.shape[1], k, nepochs=epochs, seed=seed, batch_law='throughput').train(x, init_centroids=init.copy()); te = time.time() - t\n", " t = time.time(); cp, _, _ = pisces(x, k, nepochs=epochs, seed=seed, init=init.copy()); tp = time.time() - t\n", " rows.append((seed, epochs, mse(c), te, mse(cp), tp))\n", " print(f'seed {seed} epochs {epochs}: e-means {rows[-1][2]:.5f} ({te:.1f}s) PISCES {rows[-1][4]:.5f} ({tp:.1f}s) change {100*(rows[-1][4]/rows[-1][2]-1):+.2f}%')" ] }, { "cell_type": "markdown", "id": "dc7a3e01", "metadata": {}, "source": [ "Negative change = PISCES has lower error. Expect roughly −0.4% to −1% at 1 epoch and −0.2% to −0.4% at 3 epochs.\n", "At this small k (2,048) PISCES assigns rows exactly (the sparse index only pays from k = 4,096 on), so it takes\n", "about the same time as e-means here. The speed-up appears at larger k: on the paper's benchmark (k = 16,384,\n", "1M rows) PISCES takes 0.27-0.42× e-means' time." ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.18" } }, "nbformat": 4, "nbformat_minor": 5 }