Add files using upload-large-folder tool

latest-unsharded/config.yaml

@@ -0,0 +1,1050 @@
+run_name: 1B_multi_benchmark_all_25B_rerun
+seed: 42
+epoch: null
+dry_run: false
+model:
+  d_model: 2048
+  n_heads: 16
+  n_kv_heads: null
+  clip_qkv: null
+  n_layers: 16
+  mlp_ratio: 8
+  mlp_hidden_size: null
+  activation_type: swiglu
+  block_type: sequential
+  block_group_size: 1
+  alibi: false
+  alibi_bias_max: 8.0
+  rope: true
+  rope_full_precision: true
+  rope_theta: 500000
+  flash_attention: true
+  attention_dropout: 0.0
+  multi_query_attention: null
+  attention_layer_norm: true
+  residual_dropout: 0.0
+  embedding_dropout: 0.0
+  embedding_layer_norm: false
+  layer_norm_type: rms
+  layer_norm_with_affine: true
+  layer_norm_eps: 1.0e-06
+  attention_layer_norm_with_affine: true
+  max_sequence_length: 4096
+  include_bias: false
+  bias_for_layer_norm: false
+  scale_logits: false
+  vocab_size: 100278
+  embedding_size: 100352
+  weight_tying: false
+  eos_token_id: 100257
+  pad_token_id: 100277
+  init_device: cuda
+  init_fn: normal
+  init_std: 0.02
+  init_cutoff_factor: 3.0
+  precision: amp_bf16
+  scale_emb_init: false
+  emb_init_std: null
+  norm_after: true
+optimizer:
+  name: adamw
+  learning_rate: 7.4487e-05
+  weight_decay: 0.1
+  betas:
+  - 0.9
+  - 0.95
+  eps: 1.0e-08
+  no_decay_norm_and_bias: null
+  selective_updates: false
+  decay_norm_and_bias: true
+  decay_embeddings: false
+  metrics_log_interval: 1
+  record_update_metrics: false
+scheduler:
+  name: linear_with_warmup
+  units: steps
+  t_warmup: 0
+  t_max: null
+  alpha_f: 0.0
+  grad_clip_warmup_steps: null
+  grad_clip_warmup_factor: null
+  warmup_min_lr: null
+data_prefix: null
+filter_config:
+  filter_percentage: 1.0
+  reference_sa_load_path: null
+  reference_grad_load_path: reference_grad/mmlu_stem_canonical/ref_grad.pt
+  cosine_sim: false
+  no_training: false
+  save_sa: false
+  aggregate_sa: false
+  reference_set_label: mmlu_stem_canonical
+  reference_set_as_train: false
+  global_filter: false
+  per_token_filter: false
+  update_ref_sa_interval: -1
+  zero_threshold: false
+  compute_and_save_offline_score: false
+  offline_score_load_path: null
+  online_filter: false
+debug: true
+data:
+  paths:
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-00-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-01-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-02-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-03-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-04-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-05-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-06-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-07-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-08-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-09-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-10-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-11-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-12-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-13-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-14-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-15-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-16-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-17-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-18-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-19-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-20-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-21-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-22-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-23-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-24-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-25-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-26-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-27-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-28-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-29-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-30-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-31-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-32-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-33-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-34-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-35-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-36-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-37-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-38-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-39-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-40-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-41-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-42-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-43-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-44-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-45-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-46-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-47-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-48-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-49-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-50-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-51-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-52-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-53-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-54-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-55-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-56-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-57-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-58-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-59-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-60-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-61-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-62-00000.npy
+  - /p/vast1/pretrain/pretrain-filtering/datasets/1B_coverage-multi_benchmark_all-token_count_25B/part-63-00000.npy
+  memmap_dtype: uint32
+  datasets: null
+  label_mask_paths: null
+  pad_direction: right
+  generate_attention_mask: false
+  generate_doc_lengths: false
+  num_workers: 1
+  drop_last: true
+  pin_memory: true
+  prefetch_factor: 8
+  persistent_workers: true
+  timeout: 0
+  seed: null
+  instance_filter:
+    repetition_max_period: 13
+    repetition_min_period: 1
+    repetition_max_count: 32
+  custom_dataset: null
+restore_dataloader: true
+fast_forward_batches: null
+evaluators:
+- label: mmlu_stem_canonical
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_stem_canonical_len_norm
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_stem_canonical_subject_whitelist
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_stem_canonical_len_norm_subject_whitelist
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_stem_canonical_delta_score_whitelist
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_stem_canonical_len_norm_delta_score_whitelist
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: coconut_safety
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_stem_canonical_len_norm_high_school_whitelist
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: hellaswag_ce_loss
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_stem_mc_5shot_ce_loss
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_stem_canonical_high_school_whitelist
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: piqa
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: hellaswag
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: winogrande
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: openbook_qa
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: boolq
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: sciq
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: arc_easy
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: arc_challenge
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: copa
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: commonsense_qa
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: social_iqa
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_stem_var
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_humanities_var
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_social_sciences_var
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_other_var
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_stem_mc_5shot
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_humanities_mc_5shot
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_social_sciences_mc_5shot
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_other_mc_5shot
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_stem_mc_5shot_test
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_humanities_mc_5shot_test
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_social_sciences_mc_5shot_test
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: mmlu_other_mc_5shot_test
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: basic_arithmetic
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: trivia_qa_wiki_ppl
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: natural_qs_open_ppl
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+- label: arc_easy_ppl
+  type: downstream
+  data:
+    paths: null
+    memmap_dtype: uint16
+    datasets: null
+    label_mask_paths: null
+    pad_direction: right
+    generate_attention_mask: false
+    generate_doc_lengths: false
+    num_workers: 0
+    drop_last: false
+    pin_memory: false
+    prefetch_factor: null
+    persistent_workers: false
+    timeout: 0
+    seed: null
+    instance_filter: null
+    custom_dataset: null
+  device_eval_batch_size: null
+  subset_num_batches: null
+eval_interval: 500
+tokenizer:
+  identifier: tokenizers/allenai_dolma2.json
+  truncate_direction: right
+save_folder: outputs/1B_multi_benchmark_all_25B_tuo_nodes16_mbzs2
+remote_save_folder: null
+canceled_check_interval: 50
+save_interval: 500
+save_interval_unsharded: 500
+save_interval_ephemeral: 500
+save_num_checkpoints_to_keep: -1
+save_num_unsharded_checkpoints_to_keep: -1
+save_overwrite: true
+force_save_unsharded: false
+no_pre_train_checkpoint: true
+load_path: outputs/1B_multi_benchmark_all_25B_tuo_nodes16_mbzs2/step11933-unsharded
+load_path_sharded_checkpointer: null
+try_load_latest_save: true
+reset_optimizer_state: false
+reset_trainer_state: false
+sharded_checkpointer: olmo_core
+new_style_checkpoints: null
+max_duration: 25e9T
+global_train_batch_size: 512
+device_train_batch_size: 8
+device_train_microbatch_size: 2
+device_eval_batch_size: 4
+eval_subset_num_batches: -1
+eval_on_load: false
+device_train_grad_accum: 4
+max_grad_norm: 1.0
+max_grad_norm_ratio: null
+precision: amp_bf16
+wandb:
+  project: olmo-small
+  entity: tomg-group-umd
+  group: null
+  name: 1B_multi_benchmark_all_25B_rerun
+  tags:
+  - watching
+  log_artifacts: false
+  rank_zero_only: true
+  log_interval: 1
+speed_monitor:
+  window_size: 1
+  gpu_flops_available: null
+console_log_interval: 1
+gen1_gc_interval: 10
+compile: null
+distributed_strategy: ddp
+fsdp:
+  use_orig_params: true
+  sharding_strategy: FULL_SHARD
+  wrapping_strategy: null
+  precision: pure
+  hybrid_sharding_num_model_replicas: null
+ddp:
+  grad_sync_mode: batch
+  find_unused_params: false
+single:
+  device: auto
+softmax_auxiliary_loss: true
+auxiliary_loss_multiplier: 1.0e-05
+time_limit: null
+extra_steps_after_cancel: 10
+early_stopping_factor: null
+save_data_indices: true
+python_profiling: false
+torch_profiling: false
+stop_at: 11931
+stop_after: null
+activation_checkpointing: null
+fused_loss: true
+hf_datasets_cache_dir: null
+module_outputs_save_steps: null

latest-unsharded/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:378444f01ad5da8d4333e6a0722b86c9f38c9d6b5ae7da7b38208b349fcf1b04
+size 5939713931

latest-unsharded/optim.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3874b5a30d96defb250227ef0b299a9da01c52671cfe2709bcabc1e2720ba428
+size 11879446447

latest-unsharded/train.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d64d2d9da4f94ebb5370ecd9426c1d5966659a8332af6c0c5599cf99c1f5c0c3
+size 15893