split_train_eval.py

#!/usr/bin/python3
import json
from sklearn.model_selection import train_test_split

with open('inverted.json', 'r') as f:
    data = json.load(f)
    
train_data, eval_data = train_test_split(data, test_size=0.2, random_state=42)

with open('train.json', 'w') as f:
    json.dump(train_data, f, indent=4)

with open('eval.json', 'w') as f:
    json.dump(eval_data, f, indent=4)