code/run_job.sh

#!/usr/bin/env bash
# Run a long job as its own systemd service, OUTSIDE the Claude session's cgroup:
#  - survives Claude Code session restarts (session scopes kill their children, setsid doesn't escape)
#  - hard RAM cap (MemoryMax): exceeding it kills only this job, never Claude or other jobs
#  - OOMScoreAdjust=500: under global pressure the kernel kills jobs before anything else
#  - Restart=on-failure: jobs are resumable, so a kill/crash just restarts it (max 5 times/hour)
#  - EnvironmentFile=.env: services start with a clean env; this passes HF_TOKEN without exposing it on the command line
# Usage: code/run_job.sh NAME MEMMAX CMD...     e.g. code/run_job.sh crawl 800M .venv/bin/python code/x.py
# Inspect: systemctl status skills-NAME ; logs in logs/NAME.log ; stop: systemctl stop skills-NAME
set -euo pipefail
name=$1; mem=$2; shift 2
high="$(( ${mem%?} * 85 / 100 ))${mem: -1}"   # MemoryHigh = 85% of MemoryMax: reclaim cache / throttle before the hard cap
cd /root/skills-db
systemctl reset-failed "skills-$name" 2>/dev/null || true
systemd-run --unit="skills-$name" --collect --quiet \
  --working-directory=/root/skills-db \
  -p EnvironmentFile=/root/skills-db/.env -p Environment=HOME=/root -p MemoryMax="$mem" -p MemoryHigh="$high" -p MemorySwapMax=1G -p OOMScoreAdjust=500 -p Nice=5 \
  -p Restart=on-failure -p RestartSec=30 -p StartLimitIntervalSec=3600 -p StartLimitBurst=5 \
  -p StandardOutput=append:/root/skills-db/logs/"$name".log -p StandardError=append:/root/skills-db/logs/"$name".log \
  /root/skills-db/code/job_wrapper.sh "$name" "$@"
echo "started skills-$name (MemoryMax=$mem)"