workspace/scripts/slurm/eval_bundle_consensus_dominance.sbatch

#!/bin/bash
#SBATCH --job-name=bundle_dom
#SBATCH --account=def-yalda
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=4
#SBATCH --mem=16G
#SBATCH --time=01:00:00
#SBATCH --output=outputs/hpc/logs/%x_%j.out
#SBATCH --error=outputs/hpc/logs/%x_%j.err

set -euo pipefail

PROJECT_DIR="${PROJECT_DIR:-$SLURM_SUBMIT_DIR}"
cd "$PROJECT_DIR"
mkdir -p outputs/hpc/logs

export OMP_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export MKL_NUM_THREADS=1
export DOVLA_TORCH_THREADS=1
export PYTHONDONTWRITEBYTECODE=1

PYTHON="${PYTHON:-$PROJECT_DIR/.venv/bin/python}"
CAL_INPUT="${CAL_INPUT:-runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json}"
CAL_INDEX="${CAL_INDEX:-data/cil_charts_rgb_refs/train/index.json}"
EVAL_INPUT="${EVAL_INPUT:-runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json}"
EVAL_INDEX="${EVAL_INDEX:-data/cil_charts_rgb_refs/test/index.json}"
SOURCE_INDEX="${SOURCE_INDEX:-data/cil_charts_rgb_refs/train/index.json}"
if [[ -z "${CHECKPOINT_TEMPLATE:-}" ]]; then
  CHECKPOINT_TEMPLATE='runs/ctt_residual_base_context_obs_seed{seed}/model.pt'
fi
K="${K:-16}"
BOOTSTRAP_SAMPLES="${BOOTSTRAP_SAMPLES:-1000}"

"$PYTHON" scripts/eval_learned_dominance_selector.py \
  --calibration-input "$CAL_INPUT" \
  --calibration-target-index "$CAL_INDEX" \
  --eval-input "$EVAL_INPUT" \
  --eval-target-index "$EVAL_INDEX" \
  --checkpoint-template "$CHECKPOINT_TEMPLATE" \
  --out-dir runs/ctt_base_context_obs_learned_dominance_bundle_envclip_k16_train_to_test \
  --feature-set bundle_consensus \
  --target utility_margin \
  --threshold-scope task \
  --fit-objective pointwise \
  --k "$K" \
  --bootstrap-samples "$BOOTSTRAP_SAMPLES" \
  --no-markdown-report

"$PYTHON" scripts/eval_learned_dominance_selector.py \
  --calibration-input "$CAL_INPUT" \
  --calibration-target-index "$CAL_INDEX" \
  --eval-input "$EVAL_INPUT" \
  --eval-target-index "$EVAL_INDEX" \
  --checkpoint-template "$CHECKPOINT_TEMPLATE" \
  --out-dir runs/ctt_base_context_obs_learned_dominance_chart_bundle_envclip_k16_train_to_test \
  --feature-set chart_bundle_consensus \
  --selector-chart-feature-mode base_context_obs \
  --target utility_margin \
  --threshold-scope task \
  --fit-objective pointwise \
  --k "$K" \
  --bootstrap-samples "$BOOTSTRAP_SAMPLES" \
  --no-markdown-report

"$PYTHON" scripts/eval_learned_dominance_selector.py \
  --calibration-input "$CAL_INPUT" \
  --calibration-target-index "$CAL_INDEX" \
  --eval-input "$EVAL_INPUT" \
  --eval-target-index "$EVAL_INDEX" \
  --checkpoint-template "$CHECKPOINT_TEMPLATE" \
  --out-dir runs/ctt_base_context_obs_learned_dominance_score_chart_bundle_envclip_k16_train_to_test \
  --feature-set score_chart_bundle_consensus \
  --selector-chart-feature-mode base_context_obs \
  --target utility_margin \
  --threshold-scope task \
  --fit-objective pointwise \
  --k "$K" \
  --bootstrap-samples "$BOOTSTRAP_SAMPLES" \
  --no-markdown-report

"$PYTHON" scripts/eval_learned_dominance_selector.py \
  --calibration-input "$CAL_INPUT" \
  --calibration-target-index "$CAL_INDEX" \
  --eval-input "$EVAL_INPUT" \
  --eval-target-index "$EVAL_INDEX" \
  --source-index "$SOURCE_INDEX" \
  --checkpoint-template "$CHECKPOINT_TEMPLATE" \
  --out-dir runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_envclip_k16_train_to_test \
  --feature-set chart_source_bundle_consensus \
  --selector-chart-feature-mode base_context_obs \
  --target success_weighted_margin \
  --success-bonus 2.0 \
  --threshold-scope task \
  --fit-objective pointwise \
  --k "$K" \
  --bootstrap-samples "$BOOTSTRAP_SAMPLES" \
  --no-markdown-report

"$PYTHON" scripts/build_selector_diagnostic_sweep.py \
  --out-dir runs/ctt_selector_diagnostic_sweep