backup_tom_hf.sbatch

#!/usr/bin/env bash
# Back up the Comma-2T ToMBench unlearning keepers to an HF bucket. Stages a clean
# tree (final adapters + provenance + eval JSONs + results + forget sets, EXCLUDING
# reproducible checkpoint-*/ and olmes_runs/ bulk), then syncs. Runs as a SLURM job
# so it survives any local session/SSH drop. HF discipline: Xet disabled + local HF_HOME.
#SBATCH --job-name=c2ttom_hfbackup
#SBATCH --partition=ice-cpu
#SBATCH --qos=coe-ice
#SBATCH --cpus-per-task=4
#SBATCH --mem=16G
#SBATCH --time=04:00:00
#SBATCH --output=/storage/ice-shared/cs7634/staff/TDA/logs/comma_unlearn/%x_%j.out
#SBATCH --error=/storage/ice-shared/cs7634/staff/TDA/logs/comma_unlearn/%x_%j.err
set -uo pipefail
HF=~/.local/bin/hf
export HF_TOKEN="$(cat ~/.hf_token)"
export HF_HUB_DISABLE_XET=1
export HF_HOME="${TMPDIR:-/tmp}/hf_tom_${SLURM_JOB_ID:-$$}"
mkdir -p "$HF_HOME"
TDA=/storage/ice-shared/cs7634/staff/TDA
TOM=$TDA/comma/tom_unlearning
FORGET=$TDA/comma/forget_sets/comma_2t/tombench
STAGE="${TMPDIR:-$HOME/scratch}/tom_hf_stage_${SLURM_JOB_ID:-$$}"
BUCKET=hf://buckets/HCAI-Lab/comma-2t-tom-unlearning

echo "=== staging -> $STAGE $(date) ==="
rm -rf "$STAGE"; mkdir -p "$STAGE/forget_sets"
rsync -a --exclude='checkpoint-*' "$TOM/runs" "$STAGE/"
rsync -a --exclude='olmes_runs' "$TOM/eval" "$STAGE/"
cp "$TOM"/tom_analysis.txt "$TOM"/tom_paired.csv "$TOM"/manifest_all.tsv "$TOM"/manifest.tsv "$TOM"/STATE.md "$STAGE/" 2>/dev/null || true
cp "$TOM"/*.sh "$TOM"/*.sbatch "$TOM"/*.py "$STAGE/" 2>/dev/null || true
cp "$FORGET"/*.parquet "$STAGE/forget_sets/" 2>/dev/null || true
echo "staged size:"; du -sh "$STAGE"
echo -n "adapters staged: "; find "$STAGE/runs" -name adapter_model.safetensors 2>/dev/null | wc -l
echo -n "eval JSONs staged: "; ls "$STAGE"/eval/*_tombench_eval.json 2>/dev/null | wc -l
echo -n "forget parquets staged: "; ls "$STAGE"/forget_sets/*.parquet 2>/dev/null | wc -l

echo "=== create bucket $(date) ==="
$HF buckets create HCAI-Lab/comma-2t-tom-unlearning 2>&1 | tail -3 || true
echo "=== sync $(date) ==="
$HF buckets sync "$STAGE" "$BUCKET" 2>&1 | tail -20
rc=$?
echo "=== verify: bucket listing ==="
$HF buckets ls "$BUCKET" 2>&1 | head -10
rm -rf "$STAGE" "$HF_HOME"
echo "=== DONE rc=$rc $(date) ==="
exit "$rc"