| epoch: 1.25T | |
| dataset: bigdata | |
| num_params: 1B | |
| max_seq_len: 2048 | |
| seed: 1 | |
| precision: fp32 | |
| # Tokenizer | |
| tokenizer: | |
| # name: [Add name from memory] | |
| pretrained_model_name_or_path: | |
| kwargs: | |
| model_max_length: 2048 | |
| model: | |
| name: open_lm | |
| # pretrained_model_name_or_path: [add name from memory] | |
| init_device: cpu | |
| pretrained: true | |
| load_path: # Add your (optional) Composer checkpoint path here! | |
| device_eval_batch_size: 8 | |
| # FSDP config for model sharding | |
| fsdp_config: | |
| sharding_strategy: FULL_SHARD | |
| mixed_precision: FULL | |
| icl_tasks: | |
| - | |
| label: mmlu | |
| dataset_uri: local_data/mmlu.jsonl # ADD YOUR OWN DATASET URI | |
| num_fewshot: [0] | |
| icl_task_type: multiple_choice | |
| continuation_delimiter: 'Answer: ' # this separates questions from answers | |
| has_categories: true | |