submit_tom_train.sh

#!/usr/bin/env bash
# Idempotent training recovery for the 72 expA ToMBench cells. Resubmits any cell
# whose adapter is missing/undersized AND whose train job is not live. Safe to
# run in a loop (the overnight driver calls it each tick).
set -uo pipefail
REPO_DIR="${REPO_DIR:-$HOME/dev/data-attribution-comma-attr}"
cd "$REPO_DIR"
source scripts/comma/comma_2t_sweep_env.sh
export EXTRA_OVERRIDES="$SWEEP_PAPER_OVERRIDES"
export MAXWALL=340
TDA=/storage/ice-shared/cs7634/staff/TDA
RUNS=$TDA/comma/tom_unlearning/runs
MANIFEST=$TDA/comma/tom_unlearning/manifest_all.tsv
MIN=30000000
live=$(squeue -u "$USER" -h -o '%j' 2>/dev/null || true)
missing=""
while IFS=$'\t' read -r label topic seed forget; do
    [ "$label" = "label" ] && continue
    [ -z "$label" ] && continue
    f="$RUNS/$label/adapter/adapter_model.safetensors"
    if [ -f "$f" ] && [ "$(stat -c%s "$f" 2>/dev/null || echo 0)" -ge "$MIN" ]; then continue; fi
    if echo "$live" | grep -q "^c2ttomtr_${label}\$"; then continue; fi
    missing="${missing:+$missing,}$label"
done < "$MANIFEST"
if [ -z "$missing" ]; then echo "train-recover: none missing $(date)"; exit 0; fi
echo "train-recover: resubmitting $missing $(date)"
./.venv/bin/python scripts/unlearning/submit_faithful_unlearning.py \
    --manifest "$MANIFEST" --runs-root "$RUNS" \
    --dolma-cache "$TDA/comma/retain_cache" --model-config comma_2t_lora \
    --labels "$missing" --max-forget-docs 200 --batch-size 4 --grad-accum 4 \
    --log-dir "$SWEEP_LOG_DIR" --job-prefix c2ttomtr \
    --exclude "$SWEEP_EXCLUDE_NODES" --constraint "$SWEEP_GPU_CONSTRAINT" --submit