code/watchdog.sh

#!/usr/bin/env bash
# Overnight guardian (runs as systemd service skills-watchdog, independent of any Claude session):
#  - every 5 min: relaunch any job from code/jobs.conf that is not running and has not exited 0
#    (max 6 relaunches per job per 24 h, then it is left for a human / Claude and flagged in STATUS.md)
#  - keeps the oomguard running (Claude Code processes get oom_score_adj -900)
#  - every 30 min: regenerate STATUS.md and upload it to the bucket root
cd /root/skills-db
declare -A launches
last_status=0
while true; do
  now=$(date +%s)
  while IFS='|' read -r name mem cmd; do
    name=$(echo "$name" | xargs); mem=$(echo "$mem" | xargs); cmd=$(echo "$cmd" | xargs)
    [[ -z "$name" || "$name" == \#* ]] && continue
    systemctl is-active --quiet "skills-$name" && continue
    [[ -f "logs/$name.exit" && "$(cat logs/$name.exit)" == "0" ]] && continue
    stamps=$(echo "${launches[$name]} $now" | tr ' ' '\n' | awk -v n="$now" '$1 > n - 86400' | tr '\n' ' ')
    if [ "$(echo $stamps | wc -w)" -gt 6 ]; then
      echo "$(date -u +%FT%TZ) GIVING UP on $name (6 relaunches in 24h) — needs attention" >> logs/watchdog.log; continue
    fi
    launches[$name]="$stamps"
    echo "$(date -u +%FT%TZ) relaunching $name (exit=$(cat logs/$name.exit 2>/dev/null || echo none))" >> logs/watchdog.log
    code/run_job.sh "$name" "$mem" $cmd >> logs/watchdog.log 2>&1
  done < code/jobs.conf
  systemctl is-active --quiet skills-oomguard || systemd-run --unit=skills-oomguard --collect --quiet -p MemoryMax=20M \
    /bin/bash -c 'while true; do for p in $(pgrep -f claude); do echo -900 > /proc/$p/oom_score_adj 2>/dev/null; done; sleep 30; done'
  if [ $((now - last_status)) -ge 1800 ]; then
    .venv/bin/python code/status.py > /dev/null 2>> logs/watchdog.log && last_status=$now
  fi
  sleep 300
done