scripts/disk_guard.sh

#!/usr/bin/env bash
# disk_guard.sh
# Watches mount usage and (a) logs WARN when usage >= WARN_PCT, (b) kills the
# tmux session and pipeline workers when usage >= STOP_PCT to prevent the
# filesystem from filling up entirely (which can wedge other users on the box).
#
# Defaults: warn at 92%, hard stop at 96%, poll every 60s.
# Env vars: WARN_PCT, STOP_PCT, INTERVAL, SESSION, MOUNT, LOG

set -u

WARN_PCT="${WARN_PCT:-92}"
STOP_PCT="${STOP_PCT:-96}"
INTERVAL="${INTERVAL:-60}"
SESSION="${SESSION:-chart_24h}"
MOUNT="${MOUNT:-/data}"

cd "$(dirname "$0")/.."
LOG="${LOG:-logs/disk_watch.log}"
mkdir -p "$(dirname "$LOG")"

echo "[$(date -Iseconds)] disk_guard start: mount=$MOUNT warn=${WARN_PCT}% stop=${STOP_PCT}% interval=${INTERVAL}s session=$SESSION" | tee -a "$LOG"

already_warned=0
while :; do
  pct=$(df --output=pcent "$MOUNT" 2>/dev/null | tail -1 | tr -dc 0-9)
  avail=$(df -h "$MOUNT" 2>/dev/null | tail -1 | awk '{print $4}')
  ts=$(date -Iseconds)

  if [ -z "$pct" ]; then
    echo "[$ts] disk_guard ERROR: failed to read df for $MOUNT" | tee -a "$LOG"
    sleep "$INTERVAL"
    continue
  fi

  if [ "$pct" -ge "$STOP_PCT" ]; then
    echo "[$ts] CRITICAL pct=${pct}% avail=$avail  -> killing tmux session '$SESSION' and pipeline workers" | tee -a "$LOG"
    tmux kill-session -t "$SESSION" 2>/dev/null || true
    pkill -f "pipeline.py" 2>/dev/null || true
    pkill -f "chromedriver" 2>/dev/null || true
    pkill -f "chrome --type=" 2>/dev/null || true
    echo "[$ts] disk_guard exiting (post-kill); manual cleanup may be required" | tee -a "$LOG"
    exit 0
  elif [ "$pct" -ge "$WARN_PCT" ]; then
    if [ "$already_warned" -eq 0 ]; then
      echo "[$ts] WARN pct=${pct}% avail=$avail (>= ${WARN_PCT}%)" | tee -a "$LOG"
      already_warned=1
    fi
  else
    if [ "$already_warned" -eq 1 ]; then
      echo "[$ts] OK    pct=${pct}% avail=$avail (cleared)" | tee -a "$LOG"
      already_warned=0
    fi
  fi

  sleep "$INTERVAL"
done