File size: 431 Bytes
126b4dc |
1 2 3 4 5 6 7 8 9 10 11 |
import shutil
from datasets import load_from_disk, concatenate_datasets
a = load_from_disk("/workspace/sdxs/datasets/mjnj_640")
b = load_from_disk("/workspace/sdxs/datasets/d23_640")
merged = concatenate_datasets([a, b])
merged.save_to_disk("/workspace/sdxs/datasets/mjnj_640_merged")
shutil.rmtree("/workspace/sdxs/datasets/mjnj_640")
shutil.move("/workspace/sdxs/datasets/mjnj_640_merged", "/workspace/sdxs/datasets/mjnj_640")
|