overnight_driver.sbatch

#!/usr/bin/env bash
# Autonomous on-cluster driver for the Comma-2T ToMBench n=72 experiment.
# Runs as its own SLURM job so the pipeline completes with NO local session /
# SSH involvement. Each tick: recover failed training cells, submit evals for
# complete adapters (both idempotent), report. When all 144 eval JSONs are in
# and nothing is live, run the paired analysis and exit.
#SBATCH --job-name=c2ttom_driver
#SBATCH --partition=ice-cpu
#SBATCH --qos=coe-ice
#SBATCH --cpus-per-task=2
#SBATCH --mem=8G
#SBATCH --time=12:00:00
#SBATCH --output=/storage/ice-shared/cs7634/staff/TDA/logs/comma_unlearn/%x_%j.out
#SBATCH --error=/storage/ice-shared/cs7634/staff/TDA/logs/comma_unlearn/%x_%j.err
set -uo pipefail
REPO_DIR="$HOME/dev/data-attribution-comma-attr"
TDA=/storage/ice-shared/cs7634/staff/TDA
TOM=$TDA/comma/tom_unlearning
EVAL=$TOM/eval
cd "$REPO_DIR"
export PATH="$REPO_DIR/.venv/bin:$PATH"
mkdir -p "$EVAL"
echo "driver start $(date) node=$(hostname)"

for i in $(seq 1 120); do
    bash "$TOM/submit_tom_train.sh" >> "$TOM/driver.log" 2>&1 || true
    bash "$TOM/submit_tom_evals.sh" >> "$TOM/driver.log" 2>&1 || true
    ntr=$(squeue -u "$USER" -h -o '%j' 2>/dev/null | grep -c c2ttomtr || true)
    nev=$(squeue -u "$USER" -h -o '%j' 2>/dev/null | grep -c c2ttom_eval || true)
    nj=$(ls "$EVAL"/*_tombench_eval.json 2>/dev/null | wc -l)
    echo "driver tick $i: train_left=$ntr eval_live=$nev json=$nj/144 $(date)"
    if [ "$ntr" = "0" ] && [ "$nev" = "0" ] && [ "$nj" -ge 144 ]; then
        echo "driver: all 144 evals present, running paired analysis"
        .venv/bin/python "$TOM/analyze_tom_unlearning.py" \
            --eval-dir "$EVAL" --out-csv "$TOM/tom_paired.csv" \
            > "$TOM/tom_analysis.txt" 2>&1
        echo "driver: DONE -> $TOM/tom_analysis.txt $(date)"
        exit 0
    fi
    sleep 300
done
echo "driver: TIMEOUT after 120 ticks $(date)"
.venv/bin/python "$TOM/analyze_tom_unlearning.py" \
    --eval-dir "$EVAL" --out-csv "$TOM/tom_paired.csv" \
    > "$TOM/tom_analysis_partial.txt" 2>&1 || true
exit 1