SDLM/utils/data_process/data_token_statistic.json

{
    "ScaleQuest_Math_train_1m.jsonl": 466580477,
    "smoltalk_1100k.jsonl": 867431700,
    "opc-sft-stage2-436k.jsonl": 255362659,
    "sciRIFF_4096_70k.jsonl": 84761820,
    "Table-GPT_train_13k.jsonl": 11589982,
    "tulu-3-sft-mixture_train_939k.jsonl": 637217331
}