PISCES / benchmarks /run_final.sh
Nabidnur's picture
PISCES (ARIES #1): code, paper, notebooks, results and figures
c09a3f4 verified
Raw History Blame Contribute Delete
1.52 kB
#!/usr/bin/env bash
# Final confirmation of PISCES (iterations 1-22) in one session, so every time ratio is same-machine.
# Usage: bash benchmarks/run_final.sh DATA_ROOT [THREADS] (results in results/final/)
set -euo pipefail
DATA=$1; T=${2:-4}; cd "$(dirname "$0")/.."
P="python -I benchmarks/paper_protocol.py"
R=results/final; mkdir -p $R
# 1. time to quality, validation rows, seeds 1-5
$P "$DATA/deep1b" $R/deep1m_ttq_val.jsonl --eval val --seeds 1 2 3 4 5 --epochs 3 5 --methods official --threads $T
$P "$DATA/deep1b" $R/deep1m_ttq_val.jsonl --eval val --seeds 1 2 3 4 5 --epochs 1 2 3 4 --methods pisces --threads $T
$P "$DATA/deep1b" $R/deep1m_ttq_val.jsonl --eval val --seeds 1 2 3 4 5 --epochs 3 --methods emeans_approx --threads $T
# 2. DEEP* 1M test rows, report seeds 10-19
$P "$DATA/deep1b" $R/deep1m_test.jsonl --epochs 1 3 5 --seeds 10 11 12 13 14 15 16 17 18 19 --methods official pisces --threads $T
$P "$DATA/deep1b" $R/deep1m_test.jsonl --epochs 3 --seeds 10 11 12 13 14 15 16 17 18 19 --methods emeans_approx pisces_v10 --threads $T
# 3. held-out datasets
$P "$DATA/spacev" $R/spacev1m_test.jsonl --dataset spacev-1M --epochs 3 --seeds 10 11 12 13 14 --methods official pisces emeans_approx --threads $T
$P "$DATA/ssnpp" $R/ssnpp1m_test.jsonl --dataset fb-ssnpp-1M --epochs 3 --seeds 10 11 12 13 14 --methods official pisces emeans_approx --threads $T
# 4. analysis: leave-one-out, scaling in k, downstream IVF, mechanism
RESULTS=$R bash benchmarks/run_analysis.sh "$DATA" $T
echo FINAL-DONE