gpu/single-gpu/home_benjamin_hdbasin/code/scripts/run_commercial_benchmark.sh

#!/usr/bin/env bash
set -euo pipefail

OUT_ROOT="${1:-runs/commercial_benchmark_a100}"
SEEDS="${SEEDS:-10}"
SEED_OFFSET="${SEED_OFFSET:-300}"
IMAGE_BUDGET="${IMAGE_BUDGET:-10}"
TEXT_BUDGET="${TEXT_BUDGET:-6}"
TABULAR_BUDGET="${TABULAR_BUDGET:-8}"

mkdir -p "${OUT_ROOT}"

python3 run_experiments.py \
  --fashion-mnist --cifar10 \
  --seeds "${SEEDS}" --seed-offset "${SEED_OFFSET}" \
  --budget "${IMAGE_BUDGET}" --ablations full \
  --fashion-train-size 2048 --fashion-val-size 1024 --fashion-epochs 2 \
  --cifar-train-size 2048 --cifar-val-size 1024 --cifar-epochs 2 \
  --outdir "${OUT_ROOT}/image_${SEEDS}x${IMAGE_BUDGET}"

python3 run_experiments.py \
  --sst2 --imdb \
  --seeds "${SEEDS}" --seed-offset "${SEED_OFFSET}" \
  --budget "${TEXT_BUDGET}" --ablations full \
  --sst2-train-size 256 --sst2-val-size 256 \
  --imdb-train-size 256 --imdb-val-size 256 \
  --outdir "${OUT_ROOT}/transformer_${SEEDS}x${TEXT_BUDGET}"

python3 run_experiments.py \
  --ag-news --tabular-credit \
  --seeds "${SEEDS}" --seed-offset "${SEED_OFFSET}" \
  --budget "${TABULAR_BUDGET}" --ablations full \
  --ag-train-size 1200 --ag-val-size 500 \
  --tabular-train-size 1200 --tabular-val-size 400 \
  --outdir "${OUT_ROOT}/classic_${SEEDS}x${TABULAR_BUDGET}"

python3 -m hdbasin.aggregate \
  --include "$(basename "${OUT_ROOT}")/image_${SEEDS}x${IMAGE_BUDGET},$(basename "${OUT_ROOT}")/transformer_${SEEDS}x${TEXT_BUDGET},$(basename "${OUT_ROOT}")/classic_${SEEDS}x${TABULAR_BUDGET}" \
  --outdir "${OUT_ROOT}/combined"

PYTHONPATH=. python3 scripts/build_target_savings_report.py \
  --input "${OUT_ROOT}/combined/raw/evaluations.csv" \
  --outdir "${OUT_ROOT}/target_savings"