import shutil
from datasets import load_from_disk, concatenate_datasets

a = load_from_disk("/workspace/sdxs/datasets/mjnj_640")
b = load_from_disk("/workspace/sdxs/datasets/d23_640")
merged = concatenate_datasets([a, b])

merged.save_to_disk("/workspace/sdxs/datasets/mjnj_640_merged")
shutil.rmtree("/workspace/sdxs/datasets/mjnj_640")
shutil.move("/workspace/sdxs/datasets/mjnj_640_merged", "/workspace/sdxs/datasets/mjnj_640")