Spaces:
Sleeping
Sleeping
File size: 1,962 Bytes
58e6885 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | #!/usr/bin/env bash
# disk_guard.sh
# Watches mount usage and (a) logs WARN when usage >= WARN_PCT, (b) kills the
# tmux session and pipeline workers when usage >= STOP_PCT to prevent the
# filesystem from filling up entirely (which can wedge other users on the box).
#
# Defaults: warn at 92%, hard stop at 96%, poll every 60s.
# Env vars: WARN_PCT, STOP_PCT, INTERVAL, SESSION, MOUNT, LOG
set -u
WARN_PCT="${WARN_PCT:-92}"
STOP_PCT="${STOP_PCT:-96}"
INTERVAL="${INTERVAL:-60}"
SESSION="${SESSION:-chart_24h}"
MOUNT="${MOUNT:-/data}"
cd "$(dirname "$0")/.."
LOG="${LOG:-logs/disk_watch.log}"
mkdir -p "$(dirname "$LOG")"
echo "[$(date -Iseconds)] disk_guard start: mount=$MOUNT warn=${WARN_PCT}% stop=${STOP_PCT}% interval=${INTERVAL}s session=$SESSION" | tee -a "$LOG"
already_warned=0
while :; do
pct=$(df --output=pcent "$MOUNT" 2>/dev/null | tail -1 | tr -dc 0-9)
avail=$(df -h "$MOUNT" 2>/dev/null | tail -1 | awk '{print $4}')
ts=$(date -Iseconds)
if [ -z "$pct" ]; then
echo "[$ts] disk_guard ERROR: failed to read df for $MOUNT" | tee -a "$LOG"
sleep "$INTERVAL"
continue
fi
if [ "$pct" -ge "$STOP_PCT" ]; then
echo "[$ts] CRITICAL pct=${pct}% avail=$avail -> killing tmux session '$SESSION' and pipeline workers" | tee -a "$LOG"
tmux kill-session -t "$SESSION" 2>/dev/null || true
pkill -f "pipeline.py" 2>/dev/null || true
pkill -f "chromedriver" 2>/dev/null || true
pkill -f "chrome --type=" 2>/dev/null || true
echo "[$ts] disk_guard exiting (post-kill); manual cleanup may be required" | tee -a "$LOG"
exit 0
elif [ "$pct" -ge "$WARN_PCT" ]; then
if [ "$already_warned" -eq 0 ]; then
echo "[$ts] WARN pct=${pct}% avail=$avail (>= ${WARN_PCT}%)" | tee -a "$LOG"
already_warned=1
fi
else
if [ "$already_warned" -eq 1 ]; then
echo "[$ts] OK pct=${pct}% avail=$avail (cleared)" | tee -a "$LOG"
already_warned=0
fi
fi
sleep "$INTERVAL"
done
|