#!/usr/bin/env bash
exec >> /workspace/_polish_train.log 2>&1
echo "[$(date)] TRAIN WRAPPER START"
source /opt/miniconda3/etc/profile.d/conda.sh
conda activate dp
cd /workspace/diffusion-pipe
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
export NCCL_P2P_DISABLE=1
export NCCL_IB_DISABLE=1
export NCCL_DEBUG=WARN
export NCCL_TIMEOUT=600
export NCCL_BLOCKING_WAIT=1
export NCCL_ASYNC_ERROR_HANDLING=1
exec setsid deepspeed --num_gpus=8 train.py --deepspeed --config /workspace/configs/38_anima_polish.toml