script_1/submit.slurm

#!/bin/bash





#SBATCH --job-name=script_1
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1          # crucial - only 1 task per dist per node!
#SBATCH --gres=gpu:8
#SBATCH --cpus-per-task=16
#SBATCH --time=8000
#SBATCH --partition=hopper-prod
#SBATCH --mem=0
#SBATCH --qos=normal

#SBATCH --output=/fsx/craffel/lingua_logs/script_1/logs/%j.stdout
#SBATCH --error=/fsx/craffel/lingua_logs/script_1/logs/%j.stderr

#SBATCH --begin=now+0minutes
#SBATCH --mail-type=ALL
#SBATCH --mail-user=craffel@huggingface.co
#SBATCH --requeue

#SBATCH --open-mode=append

# Mimic the effect of "conda init", which doesn't work for scripts
eval "$(/fsx/craffel/miniconda3/bin/conda shell.bash hook)"
source activate /fsx/craffel/miniconda3/envs/lingua_250401

cd /fsx/craffel/lingua_logs/script_1/code/

export OMP_NUM_THREADS=1
export LAUNCH_WITH="SBATCH"
export DUMP_DIR=/fsx/craffel/lingua_logs/script_1
export TMPDIR=/scratch

srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//fw_edu && rsync -arm /fsx/craffel/lingua/data/flexitok//fw_edu /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//dan_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//dan_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//swe_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//swe_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//vie_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//vie_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//hun_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//hun_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//fas_Arab && rsync -arm /fsx/craffel/lingua/data/flexitok//fas_Arab /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//tur_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//tur_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//ces_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//ces_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//arb_Arab && rsync -arm /fsx/craffel/lingua/data/flexitok//arb_Arab /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//ell_Grek && rsync -arm /fsx/craffel/lingua/data/flexitok//ell_Grek /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//ind_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//ind_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//nld_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//nld_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//pol_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//pol_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//por_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//por_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//ita_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//ita_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//jpn_Jpan && rsync -arm /fsx/craffel/lingua/data/flexitok//jpn_Jpan /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//fra_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//fra_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//spa_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//spa_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//deu_Latn && rsync -arm /fsx/craffel/lingua/data/flexitok//deu_Latn /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//cmn_Hani && rsync -arm /fsx/craffel/lingua/data/flexitok//cmn_Hani /scratch/craffel/lingua/data/flexitok//'
srun --ntasks-per-node=1 bash -c 'mkdir -p /scratch/craffel/lingua/data/flexitok//rus_Cyrl && rsync -arm /fsx/craffel/lingua/data/flexitok//rus_Cyrl /scratch/craffel/lingua/data/flexitok//'


srun -o $DUMP_DIR/logs/%j_%t.out -e $DUMP_DIR/logs/%j_%t.err -n 8 -N 1 python -u -m apps.main.train config=$DUMP_DIR/base_config.yaml