PISCES / benchmarks /run_final_v3.sh
Nabidnur's picture
PISCES (ARIES #1): code, paper, notebooks, results and figures
c09a3f4 verified
Raw History Blame Contribute Delete
1.44 kB
#!/usr/bin/env bash
# Confirmation of the frozen PE-means v3. Usage: bash benchmarks/run_final_v2.sh DATA_ROOT [THREADS]
set -euo pipefail
DATA=$1; T=${2:-4}; cd "$(dirname "$0")/.."
P="python -I benchmarks/paper_protocol.py"
R=${RESULTS:-results} # set RESULTS to a scratch dir to keep the tree clean
mkdir -p "$R"; for f in deep1m_cpu ssnpp1m_cpu spacev1m_cpu; do
[ "$R" != results ] && [ -f results/$f.jsonl ] && [ ! -f "$R/$f.jsonl" ] && cp results/$f.jsonl "$R/"; done
sync_back() { [ "$R" != results ] && cp "$R"/*.jsonl results/ || true; }
# 1. time-to-quality on validation rows (dev seeds)
$P "$DATA/deep1b" "$R"/deep1m_v3_time_to_quality_val.jsonl --eval val --seeds 1 2 3 4 5 --epochs 3 5 --methods official --threads $T
$P "$DATA/deep1b" "$R"/deep1m_v3_time_to_quality_val.jsonl --eval val --seeds 1 2 3 4 5 --epochs 2 3 4 --methods pemeans_v3 --threads $T
sync_back
# 2. final DEEP* 1M test rows, report seeds 10-19
$P "$DATA/deep1b" "$R"/deep1m_cpu.jsonl --epochs 1 3 5 --seeds 10 11 12 13 14 15 16 17 18 19 --methods official pemeans_v3 --threads $T
sync_back
# 3. held-out datasets, never used in development
$P "$DATA/ssnpp" "$R"/ssnpp1m_cpu.jsonl --dataset fb-ssnpp-1M --epochs 3 --seeds 10 11 12 13 14 --methods official pemeans_v3 --threads $T
$P "$DATA/spacev" "$R"/spacev1m_cpu.jsonl --dataset spacev-1M --epochs 3 --seeds 10 11 12 13 14 --methods official pemeans_v3 --threads $T
sync_back