| { |
| "config": { |
| "val_examples": 2000, |
| "max_train_examples": 200000, |
| "min_supervised_tokens": 16, |
| "shuffle": true, |
| "format": "messages", |
| "messages_field": "messages", |
| "sources": [ |
| { |
| "source_name": "smol_magpie_ultra", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "smol-magpie-ultra", |
| "split": "train", |
| "weight": 0.4, |
| "row_filters": { |
| "quality": "good" |
| } |
| }, |
| { |
| "source_name": "openhermes", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "openhermes-100k", |
| "split": "train", |
| "weight": 0.15 |
| }, |
| { |
| "source_name": "self_oss_instruct", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "self-oss-instruct", |
| "split": "train", |
| "weight": 0.15 |
| }, |
| { |
| "source_name": "everyday_conversations", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "everyday-conversations", |
| "split": "train", |
| "weight": 0.01 |
| }, |
| { |
| "source_name": "numina_cot", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "numina-cot-100k", |
| "split": "train", |
| "weight": 0.1 |
| }, |
| { |
| "source_name": "metamathqa", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "metamathqa-50k", |
| "split": "train", |
| "weight": 0.05 |
| }, |
| { |
| "source_name": "longalign", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "longalign", |
| "split": "train", |
| "weight": 0.015 |
| }, |
| { |
| "source_name": "ultrachat_200k", |
| "path": "HuggingFaceH4/ultrachat_200k", |
| "config_name": null, |
| "split": "train_sft", |
| "weight": 0.125 |
| } |
| ] |
| }, |
| "sources": [ |
| { |
| "name": "smol_magpie_ultra", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "smol-magpie-ultra", |
| "weight": 0.4, |
| "train_target": 80000, |
| "val_target": 800, |
| "train_examples": 80000, |
| "val_examples": 800, |
| "rows_seen": 117281, |
| "skipped_rows": 36481 |
| }, |
| { |
| "name": "openhermes", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "openhermes-100k", |
| "weight": 0.15, |
| "train_target": 30000, |
| "val_target": 300, |
| "train_examples": 30000, |
| "val_examples": 300, |
| "rows_seen": 31945, |
| "skipped_rows": 1645 |
| }, |
| { |
| "name": "self_oss_instruct", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "self-oss-instruct", |
| "weight": 0.15, |
| "train_target": 30000, |
| "val_target": 300, |
| "train_examples": 30000, |
| "val_examples": 300, |
| "rows_seen": 30300, |
| "skipped_rows": 0 |
| }, |
| { |
| "name": "everyday_conversations", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "everyday-conversations", |
| "weight": 0.01, |
| "train_target": 2000, |
| "val_target": 20, |
| "train_examples": 2000, |
| "val_examples": 20, |
| "rows_seen": 2020, |
| "skipped_rows": 0 |
| }, |
| { |
| "name": "numina_cot", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "numina-cot-100k", |
| "weight": 0.1, |
| "train_target": 20000, |
| "val_target": 200, |
| "train_examples": 20000, |
| "val_examples": 200, |
| "rows_seen": 20200, |
| "skipped_rows": 0 |
| }, |
| { |
| "name": "metamathqa", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "metamathqa-50k", |
| "weight": 0.05, |
| "train_target": 10000, |
| "val_target": 100, |
| "train_examples": 10000, |
| "val_examples": 100, |
| "rows_seen": 10104, |
| "skipped_rows": 4 |
| }, |
| { |
| "name": "longalign", |
| "path": "HuggingFaceTB/smoltalk", |
| "config_name": "longalign", |
| "weight": 0.015, |
| "train_target": 3000, |
| "val_target": 30, |
| "train_examples": 3000, |
| "val_examples": 30, |
| "rows_seen": 3030, |
| "skipped_rows": 0 |
| }, |
| { |
| "name": "ultrachat_200k", |
| "path": "HuggingFaceH4/ultrachat_200k", |
| "config_name": null, |
| "weight": 0.125, |
| "train_target": 25000, |
| "val_target": 250, |
| "train_examples": 25000, |
| "val_examples": 250, |
| "rows_seen": 25250, |
| "skipped_rows": 0 |
| } |
| ], |
| "tokenizer_meta": { |
| "vocab_size": 49152, |
| "special_tokens": { |
| "pad_token": "<pad>", |
| "bos_token": "<bos>", |
| "eos_token": "<eos>", |
| "unk_token": "<unk>", |
| "pad_token_id": 0, |
| "bos_token_id": 1, |
| "eos_token_id": 2, |
| "unk_token_id": 3 |
| }, |
| "data_config": { |
| "sources": [ |
| { |
| "name": "fineweb_edu", |
| "path": "HuggingFaceFW/fineweb-edu", |
| "split": "train", |
| "weight": 0.6, |
| "text_field": "text", |
| "config_name": "sample-10BT", |
| "data_dir": null, |
| "revision": null, |
| "streaming": true, |
| "shuffle_buffer": 10000, |
| "sample_documents": null |
| }, |
| { |
| "name": "cosmopedia_v2", |
| "path": "HuggingFaceTB/smollm-corpus", |
| "split": "train", |
| "weight": 0.2, |
| "text_field": "text", |
| "config_name": "cosmopedia-v2", |
| "data_dir": null, |
| "revision": null, |
| "streaming": true, |
| "shuffle_buffer": 10000, |
| "sample_documents": null |
| }, |
| { |
| "name": "the_stack_python", |
| "path": "bigcode/the-stack-dedup", |
| "split": "train", |
| "weight": 0.1, |
| "text_field": "content", |
| "config_name": null, |
| "data_dir": "data/python", |
| "revision": null, |
| "streaming": true, |
| "shuffle_buffer": 2000, |
| "sample_documents": null |
| }, |
| { |
| "name": "finemath", |
| "path": "HuggingFaceTB/finemath", |
| "split": "train", |
| "weight": 0.1, |
| "text_field": "text", |
| "config_name": "finemath-4plus", |
| "data_dir": null, |
| "revision": null, |
| "streaming": true, |
| "shuffle_buffer": 5000, |
| "sample_documents": null |
| } |
| ], |
| "tokenizer_sample_documents": 2000000, |
| "tokenizer_min_frequency": 2, |
| "tokenizer_special_tokens": [ |
| "<pad>", |
| "<bos>", |
| "<eos>", |
| "<unk>" |
| ], |
| "train_tokens": 10000000000, |
| "val_tokens": 20000000, |
| "shard_size_tokens": 100000000 |
| } |
| }, |
| "train": { |
| "num_examples": 200000, |
| "seq_len": 2048, |
| "input_ids_path": "train_input_ids.bin", |
| "labels_path": "train_labels.bin" |
| }, |
| "val": { |
| "num_examples": 2000, |
| "seq_len": 2048, |
| "input_ids_path": "val_input_ids.bin", |
| "labels_path": "val_labels.bin" |
| } |
| } |
