File size: 711 Bytes
fed1832 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 |
#!/bin/bash
#SBATCH --job-name=load-data
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=8
#SBATCH --partition=physical-gpu
#SBATCH --mem=32G
#SBATCH --output=logs/%j-load-data.out
ENV_FILE=$1
if [ -z "$ENV_FILE" ]; then
echo "Error: No environment file specified"
exit 1
fi
if [ ! -f "$ENV_FILE" ]; then
echo "Error: Environment file $ENV_FILE not found"
exit 1
fi
# Load the specified environment file
source "$ENV_FILE"
# Create the artifact folder
mkdir -p "$OUTPUT_DIR"
export PYTHONUNBUFFERED=1
# Run the Python script
python ../load_data.py \
--languages $LANGUAGES \
--model-id $MODEL_ID \
--tokenizer $TOKENIZER \
--output-dir $OUTPUT_DIR
echo "SRE SINGLE JOB DONE" |