precompress_merge_jsonls.sh

#!/bin/bash

input_folder=$1
output_folder=$2

mkdir -p $output_folder

declare -A chunks

for file in ${input_folder}/*.chunk.*_out_*.jsonl; do
    chunks["${file%%_out_*}"]=1
done

for prefix in ${!chunks[@]}; do
    cat ${prefix}_out_*.jsonl > ${output_folder}/${prefix##*/}.jsonl
    echo "Merged files for ${prefix}_out_*.jsonl into ${output_folder}/${prefix##*/}.jsonl"
done