scripts/_run_cache.sh

#!/usr/bin/env bash
exec >> /workspace/_polish_cache.log 2>&1
echo "[$(date)] CACHE WRAPPER START"
source /opt/miniconda3/etc/profile.d/conda.sh
conda activate dp
cd /workspace/diffusion-pipe
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
export NCCL_P2P_DISABLE=1
export NCCL_IB_DISABLE=1
exec setsid deepspeed --num_gpus=8 train.py --deepspeed --config /workspace/configs/38_anima_polish.toml --cache_only