social-data-attribution-w2/scripts/slurm/evaluation/olmes_instruct_base.sbatch

#!/usr/bin/env bash
# Re-run GSM8K-only instruct-base evaluation for Olmo-3-7B-Instruct.
# Only GSM8K needed re-running after the stop sequence fix (PR #79).
# MC tasks (socialiqa, mmlu) were unaffected by the bug.
# Requires: GPU node with vLLM support.
#
#SBATCH --job-name=olmes_gsm8k_instruct
#SBATCH --partition=coe-gpu
#SBATCH --qos=coe-ice
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=8
#SBATCH --mem=64G
#SBATCH --time=16:00:00
#SBATCH --output=logs/olmes_eval/%x_%j.out
#SBATCH --error=logs/olmes_eval/%x_%j.err

set -euo pipefail
cd "${SLURM_SUBMIT_DIR:-$(pwd)}"
mkdir -p logs/olmes_eval

export PATH=".venv/bin:$PATH"
export PYTHONPATH="src:olmes"
export HF_HOME="$HOME/scratch/hf_cache"
export HF_TOKEN=$(cat ~/.hf_token)
export VLLM_WORKER_MULTIPROC_METHOD=spawn
export TOKENIZERS_PARALLELISM=false

MODEL="allenai/Olmo-3-7B-Instruct"

echo "===== OLMES GSM8K instruct-base eval: ${MODEL} ====="
echo "Job ID: ${SLURM_JOB_ID}"
echo "Node: $(hostname)"
echo "GPU: $(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null || echo 'unknown')"
date

python -m oe_eval.launch \
  --model "${MODEL}" \
  --model-type vllm \
  --model-args '{"trust_remote_code": true, "max_length": 8192}' \
  --task "gsm8k::olmo3:n8:v2:instruct" \
  --output-dir "runs/evaluation_olmes/${MODEL//\//-}_gsm8k_$(date -u +%Y%m%d_%H%M%S)" \
  --gpus 1

echo "===== Done ====="
date