| defaults: | |
| - _base | |
| - _self_ | |
| # Dataset Identity | |
| dataset_name: slimpajama_5m | |
| init_fn: lmr.data.slimpajama.initialize_dataset | |
| # Token Limits | |
| max_tokens_train: 5m | |
| max_tokens_validation: 1m | |
| max_tokens_test: 1m | |
| tokens_buffer: 10k | |
| # Proportional Sampling | |
| sampling_type: proportional | |
| proportions: | |
| RedPajamaCommonCrawl: 0.38 | |
| RedPajamaC4: 0.20 | |
| RedPajamaGithub: 0.06 | |
| RedPajamaBook: 0.09 | |
| RedPajamaArXiv: 0.09 | |
| RedPajamaWikipedia: 0.09 | |
| RedPajamaStackExchange: 0.09 |