gpu/single-gpu/home_benjamin_hdbasin/code/scripts/run_full.sh

#!/usr/bin/env bash
set -uo pipefail

OUT_ROOT="${1:-runs/isobasinflow_full_$(date -u +%Y%m%d_%H%M%S)}"
mkdir -p "${OUT_ROOT}"
GPU_LOG="${OUT_ROOT}/gpu_telemetry.csv"
RUN_LOG="${OUT_ROOT}/run.log"
echo "timestamp,name,utilization_gpu,memory_used_mib,power_draw_w" > "${GPU_LOG}"

monitor_gpu() {
  while true; do
    if command -v nvidia-smi >/dev/null 2>&1; then
      nvidia-smi --query-gpu=timestamp,name,utilization.gpu,memory.used,power.draw --format=csv,noheader,nounits >> "${GPU_LOG}" 2>/dev/null || true
    else
      date -u "+%Y-%m-%d %H:%M:%S,no_gpu,0,0,0" >> "${GPU_LOG}"
    fi
    sleep 30
  done
}

monitor_gpu &
MONITOR_PID=$!
trap 'kill "${MONITOR_PID}" 2>/dev/null || true' EXIT

{
  echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  echo "out_root=${OUT_ROOT}"
  echo "gpu=$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null || echo none)"
  export MEDIUM_SYNTH_SEEDS="${FULL_SYNTH_SEEDS:-20}"
  export MEDIUM_SYNTH_BUDGET="${FULL_SYNTH_BUDGET:-80}"
  export MEDIUM_MARKET_SEEDS="${FULL_MARKET_SEEDS:-5}"
  export MEDIUM_MARKET_SEED_OFFSET="${FULL_MARKET_SEED_OFFSET:-9000}"
  export MEDIUM_TEXT_DATASETS="${FULL_TEXT_DATASETS:-sst2,ag_news,tweet_sentiment,emotion,imdb}"
  export MEDIUM_IMAGE_DATASETS="${FULL_IMAGE_DATASETS:-cifar10,fashion_mnist}"
  export MEDIUM_TEXT_BUDGET="${FULL_TEXT_BUDGET:-8}"
  export MEDIUM_IMAGE_BUDGET="${FULL_IMAGE_BUDGET:-10}"
  export MEDIUM_TEXT_TRAIN_SIZE="${FULL_TEXT_TRAIN_SIZE:-256}"
  export MEDIUM_TEXT_VAL_SIZE="${FULL_TEXT_VAL_SIZE:-256}"
  export MEDIUM_IMAGE_TRAIN_SIZE="${FULL_IMAGE_TRAIN_SIZE:-2048}"
  export MEDIUM_IMAGE_VAL_SIZE="${FULL_IMAGE_VAL_SIZE:-1024}"
  export MEDIUM_IMAGE_EPOCHS="${FULL_IMAGE_EPOCHS:-2}"
  RESET_RUN_LOG=0 bash scripts/run_medium.sh "${OUT_ROOT}"
  echo "finished_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
} 2>&1 | tee -a "${RUN_LOG}"

kill "${MONITOR_PID}" 2>/dev/null || true
trap - EXIT