scripts/run_cell.sh

#!/usr/bin/env bash
# Run ONE ablation cell (build + eval) of the W4A8 SVDQuant grid, with its own logs.
# Invoked one-at-a-time by the operator (see saved feedback: no batched bg loops).
#
# Usage:  bash scripts/run_cell.sh <RANK> <variant> <WHITEN> <REFINE> [SMOOTH] [ALPHA]
#   e.g.  bash scripts/run_cell.sh 16 plain          0 0
#         bash scripts/run_cell.sh 16 whiten         1 0
#         bash scripts/run_cell.sh 16 plain_refine   0 3
#         bash scripts/run_cell.sh 16 whiten_refine  1 3
#         bash scripts/run_cell.sh 64 plain_refine_nosmooth 0 3 0      # SMOOTH=0 (s=1)
#         bash scripts/run_cell.sh 64 plain_refine_a25      0 3 1 0.25 # alpha sweep
set -uo pipefail
cd /workspace
[ -f .venv/bin/activate ] && source .venv/bin/activate  # system python if no venv (2026-06-10 box)
export PYTHONPATH=.
export HF_HOME=/workspace/.cache/huggingface
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True  # 32 GB Blackwell box: avoid fragmentation OOM

R="$1"; name="$2"; wh="$3"; rf="$4"; sm="${5:-1}"; al="${6:-0.5}"
OUT="outputs/abl_c300_r${R}_${name}"
blog="tmp/abl_r${R}_${name}.build.log"
elog="tmp/abl_r${R}_${name}.eval.log"

echo ">>> [$(date +%H:%M:%S)] BUILD r=$R variant=$name (WHITEN=$wh REFINE=$rf SMOOTH=$sm ALPHA=$al) -> $OUT"
RANK=$R ALPHA=$al WGROUP=64 N_CALIB=300 WHITEN=$wh REFINE=$rf SMOOTH=$sm \
  CALIB_DIR=data/monet_cache MB=4 OUT="$OUT" \
  python3 -u scripts/12_build_svdquant.py 2>&1 | tee "$blog"
if ! grep -q "DONE ->" "$blog"; then
  echo "!!! BUILD FAILED r=$R $name — see $blog"; tail -6 "$blog"; exit 1
fi
echo ">>> [$(date +%H:%M:%S)] EVAL  r=$R variant=$name"
python3 -u scripts/13_eval_svdquant.py "$OUT" 2>&1 | tee "$elog"
if ! grep -q "saved .* montages" "$elog"; then
  echo "!!! EVAL FAILED r=$R $name — see $elog"; tail -6 "$elog"; exit 1
fi
echo "=== CELL DONE r=$R $name :: $(grep 'eval_vel_loss=' "$elog" | tail -1) ::"
grep -E "rel-err: mean|output-recon rel-err|refinement: best-iter|x smaller" "$blog" | sed 's/^/    /'