run_audio_multi.sh

#!/usr/bin/env bash

# Cleanup function
cleanup() {
    echo "Cleaning up..."
    # Kill all background processes
    pkill -P $$  # Kill all child processes of the current script
    exit 0
}

# Set trap to catch SIGINT (Ctrl+C) and SIGTERM signals
trap cleanup SIGINT SIGTERM

# Help function
print_help() {
    echo "Usage: bash $0 [options] --model MODEL_NAME --data DATASET_NAME [DATASET_NAME ...]"
    echo
    echo "Required parameters:"
    echo "  --model MODEL_NAME       Model name"
    echo "  --data DATASET_NAME      One or more dataset names"
    echo "  --work-dir DIR           Working directory (default: eval_result)"
    echo "Optional parameters:"
    echo "  --eval-method METHOD     Evaluation method (default: default)"
    echo "  --eval-file FILE         Path to evaluation result file (default: auto)"
    echo
    echo "Control parameters:"
    echo "  --force-reinfer          Force re-inference"
    echo "  --reeval                 Re-evaluate"
    echo "  --skip-eval              Skip evaluation"
    echo "  --debug                  Debug mode"
}

EVAL_METHOD="default"
EVAL_FILE="auto"
MODEL=""
DEFAULT_WORK_DIR="eval_result"

DATASETS=()

# Parse command line arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --help|-h)
            print_help
            exit 0
            ;;
        --model)
            MODEL="$2"
            shift 2
            ;;
        --data)
            # 支持多个数据集：--data noizeus Librispeech xxx
            shift  # 丢掉 --data 本身
            while [[ $# -gt 0 && "$1" != --* ]]; do
                DATASETS+=("$1")
                shift
            done
            ;;
        --nproc_per_node)
            NPROC_PER_NODE="$2"
            shift 2
            ;;
        --nnodes)
            NNODES="$2"
            shift 2
            ;;
        --node-rank)
            NODE_RANK="$2"
            shift 2
            ;;
        --master-addr)
            MASTER_ADDR="$2"
            shift 2
            ;;
        --master-port)
            MASTER_PORT="$2"
            shift 2
            ;;
        --work-dir)
            WORK_DIR="$2"
            shift 2
            ;;
        --eval-method)
            EVAL_METHOD="$2"
            shift 2
            ;;
        --eval-file)
            EVAL_FILE="$2"
            shift 2
            ;;
        --force-reinfer)
            FORCE_REINFER="true"
            shift
            ;;
        --reeval)
            REEVAL="true"
            shift
            ;;
        --skip-eval)
            SKIP_EVAL="true"
            shift
            ;;
        --debug)
            DEBUG="true"
            shift
            ;;
        *)
            echo "Unknown parameter: $1"
            print_help
            exit 1
            ;;
    esac
done

# 必要参数检查
if [ -z "$MODEL" ] || [ ${#DATASETS[@]} -eq 0 ]; then
    echo "Error: --model 和 --data 都是必填的，且 --data 后至少要有一个数据集名。"
    print_help
    exit 1
fi

# 把数据集数组拼成一个字符串，方便后面拼命令
DATA_STR="${DATASETS[*]}"

# Set default work-dir
if [ -z "$WORK_DIR" ]; then
    WORK_DIR="$DEFAULT_WORK_DIR"
fi

# Ensure work-dir exists
if [ ! -d "$WORK_DIR" ]; then
    mkdir -p "$WORK_DIR"
    echo "Created work directory: $WORK_DIR"
fi

# GPU groups for different configurations

# Get available GPU count
get_gpu_count() {
    if command -v nvidia-smi &> /dev/null; then
        nvidia-smi --query-gpu=gpu_name --format=csv,noheader | wc -l
    else
        echo "0"
    fi
}

# Dynamically construct GPU groups
TOTAL_GPUS=$(get_gpu_count)
echo "Detected $TOTAL_GPUS GPUs"

# Construct single GPU groups
GPU_1_GROUPS=()
for ((i=0; i<TOTAL_GPUS; i++)); do
    GPU_1_GROUPS+=($i)
done

# Construct 4-GPU groups
declare -A GPU_4_GROUPS
group_idx=0
for ((i=0; i<TOTAL_GPUS; i+=4)); do
    if ((i+3 < TOTAL_GPUS)); then
        GPU_4_GROUPS[$group_idx]="${i} $((i+1)) $((i+2)) $((i+3))"
        ((group_idx++))
    fi
done

# If no GPU is detected, show warning
if [ "$TOTAL_GPUS" -eq 0 ]; then
    echo "No GPU devices detected, can only run evaluation"
fi

# Split MODEL string into array by space（支持多个 model）
IFS=' ' read -r -a models <<< "$MODEL"

# 如果 GPU 数 < 4 但模型里包含 StepAudio，则直接报错
if [ "$TOTAL_GPUS" -lt 4 ]; then
    for m in "${models[@]}"; do
        if [[ "$m" == "StepAudio" ]]; then
            echo "Warning: StepAudio requires at least 4x80G GPUs, but only $TOTAL_GPUS GPUs available"
            exit 1
        fi
    done
fi

# If reeval is specified, no GPU is needed, single process inference is sufficient
if [ -n "$REEVAL" ]; then
    for model in "${models[@]}"; do
        echo "Running reeval for model: $model on datasets: $DATA_STR"
        CMD="python run_audio.py \
        --model $model \
        --data $DATA_STR \
        --work-dir $WORK_DIR \
        --reeval"

        [ "$EVAL_FILE" != "auto" ] && CMD="$CMD --eval-file $EVAL_FILE"
        [ "$EVAL_METHOD" != "default" ] && CMD="$CMD --eval-method $EVAL_METHOD"
        echo "Executing command: $CMD"
        eval "$CMD"
    done
    exit 0
fi

# Loop through each model
for model in "${models[@]}"; do
    if [[ $model == "StepAudio" ]]; then
        NUM_GPUS=4
        GPU_GROUPS=("${GPU_4_GROUPS[@]}")
    else
        NUM_GPUS=1
        GPU_GROUPS=("${GPU_1_GROUPS[@]}")
    fi

    for i in "${!GPU_GROUPS[@]}"; do
        rank=$i
        # Convert space-separated GPU list to comma-separated string
        if [[ $NUM_GPUS == 4 ]]; then
            CUDA_DEVICES=$(echo ${GPU_GROUPS[$i]} | tr ' ' ',')
        else
            CUDA_DEVICES=${GPU_GROUPS[$i]}
        fi
        WORLD_SIZE=${#GPU_GROUPS[@]}

        echo "Running inference for model: $model on datasets: $DATA_STR on GPU group: $CUDA_DEVICES"
        CMD="CUDA_VISIBLE_DEVICES=$CUDA_DEVICES python run_audio.py \
        --model $model \
        --data $DATA_STR \
        --work-dir $WORK_DIR \
        --rank $rank \
        --world-size $WORLD_SIZE"

        # Add optional parameters
        [ "$EVAL_METHOD" != "default" ] && CMD="$CMD --eval-method $EVAL_METHOD"
        [ -n "$FORCE_REINFER" ] && CMD="$CMD --force-reinfer"
        [ -n "$SKIP_EVAL" ] && CMD="$CMD --skip-eval"
        [ -n "$DEBUG" ] && CMD="$CMD --debug"

        echo "Executing command: $CMD"
        eval "$CMD &"
    done
    wait
    echo "Inference for model: $model completed."
done