File size: 671 Bytes
31f25cb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | import json
from datasets import load_dataset
from tqdm import tqdm
dataset = load_dataset('dyyyyyyyy/ScaleQuest-Math', split='train')
print("Converting dataset to jsonl format")
output_file = "./ScaleQuest_Math_train_1m.jsonl"
ix = 0
with open(output_file, 'w', encoding='utf-8') as f:
for item in tqdm(dataset):
conv = {
'id': ix,
'conversations': [
{'from': 'human', 'value': item['query']},
{'from': 'gpt', 'value': item['response']}
]
}
ix += 1
f.write(json.dumps(conv, ensure_ascii=False) + '\n')
print(f"Conversion complete. Output saved as {output_file}") |