job_scripts/load_data.sh

#!/bin/bash
#SBATCH --job-name=load-data
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=8
#SBATCH --partition=physical-gpu
#SBATCH --mem=32G
#SBATCH --output=logs/%j-load-data.out

ENV_FILE=$1

if [ -z "$ENV_FILE" ]; then
    echo "Error: No environment file specified"
    exit 1
fi

if [ ! -f "$ENV_FILE" ]; then
    echo "Error: Environment file $ENV_FILE not found"
    exit 1
fi

# Load the specified environment file
source "$ENV_FILE"

# Create the artifact folder
mkdir -p "$OUTPUT_DIR"

export PYTHONUNBUFFERED=1

# Run the Python script
python ../load_data.py \
    --languages $LANGUAGES \
    --model-id $MODEL_ID \
    --tokenizer $TOKENIZER \
    --output-dir $OUTPUT_DIR

echo "SRE SINGLE JOB DONE"