File size: 835 Bytes
03a77f1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 |
import os
import json
def merge_json_files(folder_path, output_file):
merged_data = []
# Iterate over all files in the folder
for filename in os.listdir(folder_path):
if filename.endswith(".jsonl"):
file_path = os.path.join(folder_path, filename)
print(file_path)
with open(file_path) as f:
data=[json.loads(line) for line in f]
merged_data.extend(data)
# Write the merged data to the output file
with open(output_file, "w", encoding="utf-8") as out_f:
json.dump(merged_data, out_f, indent=4, ensure_ascii=False)
print(f"Merged {len(merged_data)} entries into {output_file}")
# Example usage:
merge_json_files("/home/gs4288/visual_pruning_kl/data/playground", "/home/gs4288/visual_pruning_kl/data/playground/merged.json")
|