defaults:
  - _base
  - _self_
  
# Dataset Identity
dataset_name: slimpajama_60b
init_fn: lmr.data.slimpajama.initialize_dataset

# Token Limits
max_tokens_train: 60b       # Determines the number of tokens SAVED, not necessarily the total training tokens.
max_tokens_validation: 60m
max_tokens_test: 60m

tokens_buffer: 100k          # Saves additional tokens (per component) to prevent batch/sequence level inconsistencies.

# Proportional Sampling (Keeping default proportions)
sampling_type: proportional
proportions:
  RedPajamaCommonCrawl: 0.522
  RedPajamaC4: 0.267
  RedPajamaGithub: 0.052
  RedPajamaBook: 0.042
  RedPajamaArXiv: 0.046
  RedPajamaWikipedia: 0.038
  RedPajamaStackExchange: 0.033