PISCES / benchmarks /run_analysis.sh
Nabidnur's picture
PISCES (ARIES #1): code, paper, notebooks, results and figures
c09a3f4 verified
Raw History Blame Contribute Delete
2.25 kB
#!/usr/bin/env bash
# Deep analysis of the frozen PISCES (after run_final_pisces.sh). Usage: bash benchmarks/run_analysis.sh DATA_ROOT [THREADS]
# 1. leave-one-out ablation at full scale (DEEP* 1M, k = 16,384, validation rows, seeds 1-3, 3 epochs)
# 2. scaling in k (1,024 / 4,096 / 65,536 at n = 1M; validation rows, seeds 1-3, 3 epochs)
# 3. downstream IVF index quality (10K DEEP1B queries, exact ground truth on the 1M rows)
# 4. mechanism: revival timeline, cluster sizes, Lemma 2 on real clusters
set -euo pipefail
DATA=$1; T=${2:-4}; cd "$(dirname "$0")/.."
P="python -I benchmarks/paper_protocol.py"
R=${RESULTS:-results}; mkdir -p "$R"
BASE='"split":"principal","gain":"greedy","split_pass":true,"tau":0.2,"every_until":0.3,"approx":"auto","approx_last":"auto","fused":true,"codec":"sq8r"'
$P "$DATA/deep1b" "$R"/ablation_full_val.jsonl --eval val --epochs 3 --seeds 1 2 3 --threads $T --methods official pisces \
--variant no_principal_split "{${BASE/\"split\":\"principal\"/\"split\":\"faiss\"},\"alpha_scale\":1.4}" \
--variant no_gain_ranking "{${BASE/\"gain\":\"greedy\"/\"gain\":\"usage\"},\"alpha_scale\":1.4}" \
--variant no_consistent_closing "{${BASE/\"split_pass\":true/\"split_pass\":false},\"alpha_scale\":1.4}" \
--variant tau_0.1 "{${BASE/\"tau\":0.2/\"tau\":0.1},\"alpha_scale\":1.4}" \
--variant step_x1.0 "{$BASE,\"alpha_scale\":1.0}" \
--variant no_early_revival "{${BASE/\"every_until\":0.3/\"every_until\":0.0},\"alpha_scale\":1.4}" \
--variant exact_last_epoch "{${BASE/,\"approx_last\":\"auto\"/},\"alpha_scale\":1.4}" \
--variant exact_assignment "{\"split\":\"principal\",\"gain\":\"greedy\",\"split_pass\":true,\"tau\":0.2,\"every_until\":0.3,\"fused\":true,\"alpha_scale\":1.4}" \
--variant no_engineering "{${BASE/,\"fused\":true,\"codec\":\"sq8r\"/},\"alpha_scale\":1.4}"
for k in 1024 4096 65536; do
$P "$DATA/deep1b" "$R"/scaling_k${k}_val.jsonl --k $k --eval val --epochs 3 --seeds 1 2 3 --threads $T --methods official pisces
done
python -I benchmarks/ivf_downstream.py "$DATA/deep1b" "$R"/ivf_downstream.jsonl --epochs 3 --seeds 10 11 12 --methods official pisces --threads $T
python -I benchmarks/mechanism_analysis.py "$DATA/deep1b" "$R"/mechanism_seed10.json --seed 10 --threads $T