| input_folder=$1 | |
| output_folder=$2 | |
| mkdir -p $output_folder | |
| declare -A chunks | |
| for file in ${input_folder}/*.chunk.*_out_*.jsonl; do | |
| chunks["${file%%_out_*}"]=1 | |
| done | |
| for prefix in ${!chunks[@]}; do | |
| cat ${prefix}_out_*.jsonl > ${output_folder}/${prefix##*/}.jsonl | |
| echo "Merged files for ${prefix}_out_*.jsonl into ${output_folder}/${prefix##*/}.jsonl" | |
| done |