gpu/single-gpu/home_benjamin_hdbasin/code/scripts/launch_remote_full.sh

#!/usr/bin/env bash
set -euo pipefail

REMOTE="${REMOTE:-benjamin@136.115.202.117}"
REMOTE_DIR="${REMOTE_DIR:-/home/benjamin/hdbasin}"
SESSION="${SESSION:-isobasinflow_full}"
MIN_FREE_GB="${MIN_FREE_GB:-20}"

ssh "${REMOTE}" "set -euo pipefail
  command -v tmux >/dev/null
  command -v nvidia-smi >/dev/null
  echo remote_host=\$(hostname)
  nvidia-smi --query-gpu=name,memory.total,memory.used,utilization.gpu --format=csv,noheader
"

rsync -az --exclude '.git/' --exclude 'runs/' --exclude 'results/' --exclude 'figures/' --exclude 'reports/' \
  ./ "${REMOTE}:${REMOTE_DIR}/"

ssh "${REMOTE}" "set -euo pipefail
  free_gb=\$(df -BG /home | awk 'NR==2 {gsub(\"G\", \"\", \$4); print \$4}')
  if [ \"\${free_gb}\" -lt \"${MIN_FREE_GB}\" ]; then
    find '${REMOTE_DIR}/runs' -mindepth 1 -maxdepth 1 -type d -mtime +1 -print -exec rm -rf {} + 2>/dev/null || true
    rm -rf /home/benjamin/.cache/pip/* 2>/dev/null || true
  fi
  free_gb=\$(df -BG /home | awk 'NR==2 {gsub(\"G\", \"\", \$4); print \$4}')
  if [ \"\${free_gb}\" -lt \"${MIN_FREE_GB}\" ]; then
    echo \"free_disk_gb=\${free_gb}; below MIN_FREE_GB=${MIN_FREE_GB}\" >&2
    exit 2
  fi
  cd '${REMOTE_DIR}'
  bash scripts/run_smoke.sh runs/remote_isobasinflow_smoke_\$(date -u +%Y%m%d_%H%M%S)
  tmux kill-session -t '${SESSION}' 2>/dev/null || true
  tmux new-session -d -s '${SESSION}' 'cd ${REMOTE_DIR} && bash scripts/run_full.sh runs/remote_isobasinflow_full_\$(date -u +%Y%m%d_%H%M%S)'
  tmux ls | grep '${SESSION}'
"