submit_tom_evals.sh

#!/usr/bin/env bash
# Idempotent ToMBench eval submitter for the Comma-2T ToM unlearning experiment.
# Submits eval for any complete-and-unevaluated adapter in both arms:
#   expA (influence): $TDA/comma/tom_unlearning/runs/c2ttom_expA_tombench_<topic>_s<seed>
#   exp1 (random):    $TDA/comma/unlearn_faithful/comma_2t/full/comma_2t_full_exp1_<topic>_s<seed>
# Re-run after training finishes to pick up expA adapters. Skips done/live cells.
set -uo pipefail
TDA=/storage/ice-shared/cs7634/staff/TDA
EVAL_DIR=$TDA/comma/tom_unlearning/eval
SBATCH=$TDA/comma/tom_unlearning/comma_tombench_eval.sbatch
BASELINE=0.5132867
MODEL=common-pile/comma-v0.1-2t
MIN_ADAPTER_BYTES=30000000
EXCLUDE="atl1-1-03-010-[10,15,20,25,30]-0,atl1-1-03-011-[3,8,13,18,23,28]-0,atl1-1-03-013-3-0,atl1-1-03-014-16-0"
CONSTRAINT="HX00|A100-80GB|L40S|A40"
TOPICS="adult_content art_and_design crime_and_law education_and_jobs electronics_and_hardware entertainment fashion_and_beauty finance_and_business food_and_dining games health history_and_geography home_and_hobbies industrial literature politics religion science_math_and_technology social_life software software_development sports_and_fitness transportation travel_and_tourism"
SEEDS="42 43 44"
mkdir -p "$EVAL_DIR"
live=$(squeue -u "$USER" -h -o '%j' 2>/dev/null || true)

submit_one() {  # label adapter_dir topic seed
    local label=$1 adapter=$2 topic=$3 seed=$4
    local out="$EVAL_DIR/${label}_tombench_eval.json"
    if [ -f "$out" ]; then echo "done     $label"; return; fi
    if echo "$live" | grep -q "^c2ttom_eval_${label}\$"; then echo "live     $label"; return; fi
    local f="$adapter/adapter_model.safetensors"
    if [ ! -f "$f" ] || [ "$(stat -c%s "$f" 2>/dev/null || echo 0)" -lt "$MIN_ADAPTER_BYTES" ]; then
        echo "no-adapt $label"; return
    fi
    local jid
    jid=$(sbatch --parsable --job-name="c2ttom_eval_${label}" \
        --exclude="$EXCLUDE" --constraint="$CONSTRAINT" \
        --export="ALL,MODEL_ID=${MODEL},ADAPTER_DIR=${adapter},OUTPUT_JSON=${out},TOPIC_BIN=${topic},SEED=${seed},BASELINE=${BASELINE}" \
        "$SBATCH")
    echo "SUBMIT   $label -> $jid"
}

for topic in $TOPICS; do
    for seed in $SEEDS; do
        submit_one "c2ttom_expA_tombench_${topic}_s${seed}" \
            "$TDA/comma/tom_unlearning/runs/c2ttom_expA_tombench_${topic}_s${seed}/adapter" "$topic" "$seed"
        submit_one "exp1_${topic}_s${seed}" \
            "$TDA/comma/unlearn_faithful/comma_2t/full/comma_2t_full_exp1_${topic}_s${seed}/adapter" "$topic" "$seed"
    done
done