tests/functional_tests/shell_test_utils/start_interactive_job.sh

#!/bin/bash

# Function to print usage
print_usage() {
    echo "Usage: $0 [OPTIONS]"
    echo
    echo "Required options:"
    echo "  --partition PARTITION    Slurm partition"
    echo "  --slurm-account ACCOUNT  Slurm account/PPP"
    echo "  --image IMAGE           Container image"
    echo "  --dataset-dir DIR       Dataset root directory"
    echo
    echo "Optional options:"
    echo "  --time TIME             Job time limit (default: 1:00:00)"
    echo
    echo "Example:"
    echo "  $0 --partition dgx --slurm-account nvidia --image nvcr.io/nvidia/pytorch:23.10-py3 --dataset-dir /datasets"
    echo "  $0 --partition dgx --slurm-account nvidia --image nvcr.io/nvidia/pytorch:23.10-py3 --dataset-dir /datasets --time 2:00:00"
}

# Initialize variables
PARTITION=""
SLURM_ACCOUNT=""
IMAGE=""
DATASET_DIR=""
TIME="1:00:00"
RECIPES_DIR="tests/test_utils/recipes"
CONTAINER_MOUNTS=""
NO_GPUS_PER_TASK="FALSE"

# Declare associative array for tracking unique mounts
declare -A seen_mounts

# Parse command line arguments
while [[ $# -gt 0 ]]; do
    case $1 in
    --partition)
        PARTITION="$2"
        shift 2
        ;;
    --slurm-account)
        SLURM_ACCOUNT="$2"
        shift 2
        ;;
    --image)
        IMAGE="$2"
        shift 2
        ;;
    --dataset-dir)
        DATASET_DIR="$2"
        shift 2
        ;;
    --time)
        TIME="$2"
        shift 2
        ;;
    --no-gpus-per-task)
        NO_GPUS_PER_TASK="TRUE"
        shift 1
        ;;
    --help)
        print_usage
        exit 0
        ;;
    *)
        echo "Error: Unknown option '$1'"
        print_usage
        exit 1
        ;;
    esac
done

# Validate required arguments
if [ -z "$PARTITION" ] || [ -z "$SLURM_ACCOUNT" ] || [ -z "$IMAGE" ] || [ -z "$DATASET_DIR" ]; then
    echo "Error: Missing required arguments"
    print_usage
    exit 1
fi

# Add current directory to container mounts
CONTAINER_MOUNTS="$DATASET_DIR:/mnt/artifacts,$(pwd):/opt/megatron-lm"

# Build the final srun command
SRUN_CMD="srun \
    --partition=$PARTITION \
    --account=$SLURM_ACCOUNT \
    --container-image=$IMAGE \
    --container-workdir=/opt/megatron-lm \
    --container-mounts=$CONTAINER_MOUNTS \
    --no-container-mount-home \
    --nodes=1 \
    $(if [ "$NO_GPUS_PER_TASK" = "FALSE" ]; then echo "--gpus-per-task=8"; fi) \
    --time=$TIME \
    --pty bash"

printf "Generated srun command with all container mounts:\n\n"
echo "$SRUN_CMD"
echo
read -p "Execute this command? (y/n): " response
if [[ "$response" =~ ^[Yy]$ ]]; then
    echo "Executing command..."
    eval "$SRUN_CMD"
else
    echo "Command not executed."
fi