File size: 1,320 Bytes
0ea0810 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | import json
import random
# 设置随机种子以便可复现
random.seed(42)
# 读取原始 jsonl 文件
input_file = "alpaca-gpt4.jsonl" # 请替换为你的路径
with open(input_file, "r", encoding="utf-8") as f:
data = [json.loads(line) for line in f]
# 打乱数据
random.shuffle(data)
# 重新格式化为 {"text": instruction + input, "result": output}
processed = []
for item in data:
instruction = item.get("instruction", "")
input_text = item.get("input", "")
output_text = item.get("output", "")
text = instruction + "\n" + input_text if input_text else instruction
processed.append({
"text": text.strip(),
"result": output_text.strip()
})
# 划分数据
n = len(processed)
n_train = int(n * 0.8)
n_valid = int(n * 0.1)
n_test = n - n_train - n_valid # 保证总数一致
train_data = processed[:n_train]
valid_data = processed[n_train:n_train + n_valid]
test_data = processed[n_train + n_valid:]
# 写入三个文件
def write_jsonl(filename, dataset):
with open(filename, "w", encoding="utf-8") as f:
for item in dataset:
f.write(json.dumps(item, ensure_ascii=False) + "\n")
write_jsonl("alpaca-gpt4-train.jsonl", train_data)
write_jsonl("alpaca-gpt4-valid.jsonl", valid_data)
write_jsonl("alpaca-gpt4-test.jsonl", test_data)
|