code/memguard.sh

#!/usr/bin/env bash
# Machine-wide memory guard (systemd service skills-memguard), on top of the per-job MemoryMax caps:
# every 10 s, if MemAvailable < LOW: freeze (cgroup-freeze) the lowest-priority running job — frozen pages can be
# swapped out and the job cannot allocate more; once MemAvailable > HIGH, thaw frozen jobs (highest priority first).
# Priority (first = frozen first): analysis jobs, converters, shippers, fill, API crawler, GitSkills converter, crawlers.
LOW=350; HIGH=900
ORDER="dedup-count minhash clawhub-api-build ship-backfill ship-pass2 fill-repo-cap clawhub-api gitskills canon finish-chain crawl-chain"
cd /root/skills-db
while true; do
  avail=$(( $(awk '/MemAvailable/{print $2}' /proc/meminfo) / 1024 ))
  if [ "$avail" -lt "$LOW" ]; then
    for u in $ORDER; do
      st=$(systemctl show -p FreezerState --value skills-$u 2>/dev/null)
      if systemctl is-active --quiet skills-$u && [ "$st" = "running" ]; then
        systemctl freeze skills-$u && echo "$(date -u +%FT%TZ) MemAvailable=${avail}MB < ${LOW}: froze $u" >> logs/memguard.log
        break
      fi
    done
  elif [ "$avail" -gt "$HIGH" ]; then
    for u in $(echo $ORDER | tr ' ' '\n' | tac); do
      if [ "$(systemctl show -p FreezerState --value skills-$u 2>/dev/null)" = "frozen" ]; then
        systemctl thaw skills-$u && echo "$(date -u +%FT%TZ) MemAvailable=${avail}MB > ${HIGH}: thawed $u" >> logs/memguard.log
        break
      fi
    done
  fi
  sleep 10
done