comma_tombench_eval.sbatch

#!/usr/bin/env bash
# Evaluate a Comma-2T unlearned LoRA adapter on ToMBench (tombench:mc::socialtda)
# and compute gamma vs the base Comma-2T ToMBench score. Standalone wrapper,
# isolated from the core-4 eval_harness. Reuses the comma-attr venv + the
# tombench-capable olmes clone (no worktree git mutation).
#
# Env: ADAPTER_DIR (empty for base baseline), OUTPUT_JSON (required);
#   MODEL_ID (comma checkpoint), TOPIC_BIN, SEED, BASELINE, WRAPPER.
#SBATCH --job-name=c2ttom_eval
#SBATCH --qos=coe-ice
#SBATCH --gres=gpu:1
#SBATCH --constraint=HX00
#SBATCH --cpus-per-task=8
#SBATCH --mem=128G
#SBATCH --time=03:00:00
#SBATCH --output=/storage/ice-shared/cs7634/staff/TDA/logs/comma_unlearn/%x_%j.out
#SBATCH --error=/storage/ice-shared/cs7634/staff/TDA/logs/comma_unlearn/%x_%j.err
set -uo pipefail
REPO_DIR="${REPO_DIR:-$HOME/dev/data-attribution-comma-attr}"
OLMES_DIR="${OLMES_DIR:-$HOME/scratch/olmes-social-deps-gmatlin3/olmes}"
WRAPPER="${WRAPPER:-/storage/ice-shared/cs7634/staff/TDA/comma/tom_unlearning/eval_tombench_unlearn.py}"
cd "$REPO_DIR"
export PATH="$REPO_DIR/.venv/bin:$PATH"
export PYTHONPATH="$REPO_DIR/src:$OLMES_DIR${PYTHONPATH:+:$PYTHONPATH}"
source scripts/slurm/pace_env.sh
export VLLM_WORKER_MULTIPROC_METHOD=spawn
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
export TMPDIR="${TMPDIR:-/tmp/c2ttom_eval_${SLURM_JOB_ID:-x}}"
mkdir -p "$TMPDIR" /storage/ice-shared/cs7634/staff/TDA/logs/comma_unlearn

MODEL_ID="${MODEL_ID:-common-pile/comma-v0.1-2t}"
BASELINE="${BASELINE:-0.5132867}"
: "${OUTPUT_JSON:?need OUTPUT_JSON}"
mkdir -p "$(dirname "$OUTPUT_JSON")"

ADAPTER_ARG=()
if [ -n "${ADAPTER_DIR:-}" ]; then
    ADAPTER_ARG+=(--adapter-dir "$ADAPTER_DIR")
fi

echo "c2ttom eval: model=$MODEL_ID adapter=${ADAPTER_DIR:-BASE} topic=${TOPIC_BIN:-none} seed=${SEED:-none} baseline=$BASELINE out=$OUTPUT_JSON node=$(hostname) $(date)"

python "$WRAPPER" \
    --model-id "$MODEL_ID" \
    "${ADAPTER_ARG[@]}" \
    --output-json "$OUTPUT_JSON" \
    --baseline "$BASELINE" \
    --topic-bin "${TOPIC_BIN:-none}" \
    --seed "${SEED:-none}"
rc=$?
echo "done rc=$rc $(date)"
exit "$rc"