File size: 729 Bytes
a20151e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | from datasets import load_dataset
dataset = load_dataset("wikitext", "wikitext-103-v1")
train_file = '/mnt/data/vinhbk/wikitext103/train.txt'
valid_file = '/mnt/data/vinhbk/wikitext103/valid.txt'
test_file = '/mnt/data/vinhbk/wikitext103/test.txt'
# Save the train split
with open(train_file, 'w', encoding='utf-8') as f:
for item in dataset['train']:
f.write(item['text'] + '\n')
# Save the validation split
with open(valid_file, 'w', encoding='utf-8') as f:
for item in dataset['validation']:
f.write(item['text'] + '\n')
# Save the test split
with open(test_file, 'w', encoding='utf-8') as f:
for item in dataset['test']:
f.write(item['text'] + '\n')
print("Files saved successfully!") |