Add files using upload-large-folder tool

pretrain/olmo2-600M-10xC-k1/step100000-hf/chat_template.jinja

@@ -0,0 +1,4 @@
+{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

pretrain/olmo2-600M-10xC-k1/step100000-hf/config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 1344,
+  "initializer_range": 0.02,
+  "intermediate_size": 5376,
+  "max_position_embeddings": 2048,
+  "model_type": "olmo2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 16,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 500000,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.15.1",
+  "use_cache": true,
+  "vocab_size": 100278
+}

pretrain/olmo2-600M-10xC-k1/step100000-hf/generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "do_sample": true,
+  "eos_token_id": 100257,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 100277,
+  "transformers_version": "5.15.1",
+  "use_cache": true
+}

pretrain/olmo2-600M-10xC-k1/step100000-hf/tokenizer-export-audit.json

@@ -0,0 +1,9 @@
+{
+  "passed": true,
+  "source": "allenai/dolma2-tokenizer",
+  "revision": null,
+  "backend_sha256": "30ddf87d4cc948a8bd56980b84605ee465064d0b5871c7bf2fa6d00f04c99b9a",
+  "tokenizer_sha256": "18e309ad7f9c60037eaf26aca401a96128187cad2bdfa01a508dca7526bf6300",
+  "tokenizer_class": "TokenizersBackend",
+  "probe_count": 8
+}

pretrain/olmo2-600M-10xC-k1/step100000-hf/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 2048,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>"
+}

pretrain/olmo2-600M-10xC-k1/step100000/config.json

@@ -0,0 +1 @@
+{"model": {"d_model": 1344, "vocab_size": 100352, "n_layers": 16, "block": {"sequence_mixer": {"name": "default", "n_heads": 16, "bias": false, "rope": {"name": "default", "theta": 500000, "full_precision": true, "no_global_rope": false, "partial_rotary_factor": 1.0, "_CLASS_": "olmo_core.nn.rope.RoPEConfig"}, "qk_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "backend": "torch", "dtype": "float32", "use_head_qk_norm": false, "scalable_softmax": false, "attention_sinks": false, "_CLASS_": "olmo_core.nn.attention.AttentionConfig", "type": "attention"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "feed_forward": {"hidden_size": 5376, "name": "default", "bias": false, "dtype": "float32", "activation": "silu", "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"}, "name": "reordered_norm", "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "float32", "loss_implementation": "default", "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"}, "name": "default", "dtype": "float32", "init_method": "normal", "init_seed": 2352, "init_std": 0.02, "tie_word_embeddings": false, "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "paths": ["/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.prefix6308545764.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.prefix9671333560.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.prefix3431295592.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/allenai/dolma2-tokenizer/part-09-00000.prefix2012701596.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.prefix1952181972.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-009-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-065-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-038-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-087-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-011-00000.prefix1990182664.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.prefix604873468.npy"], "expand_glob": false, "metadata": [{"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "pes2o"}, {"label": "proofpile-2-arxiv"}, {"label": "proofpile-2-open-web-math"}, {"label": "proofpile-2-stack"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "wikipedia"}], "include_instance_metadata": true, "instance_filter_config": {"repetition_max_period": 13, "repetition_min_period": 1, "repetition_max_count": 32, "_CLASS_": "olmo_core.data.numpy_dataset.InstanceFilterConfig"}, "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "generate_doc_lengths": false, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyFSLDatasetConfig"}, "data_loader": {"global_batch_size": 1048576, "seed": 34521, "num_workers": 4, "ignore_fingerprint_mismatch": false, "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig", "type": "numpy"}, "train_module": {"rank_microbatch_size": 65536, "max_sequence_length": 2048, "optim": {"group_overrides": [{"params": ["embeddings.weight"], "opts": {"weight_decay": 0.0}, "_CLASS_": "olmo_core.optim.config.OptimGroupOverride"}], "compile": false, "fixed_fields": ["initial_lr"], "lr": 0.002657591869667391, "betas": [0.9, 0.95], "eps": 1e-08, "weight_decay": 0.1, "fused": true, "_CLASS_": "olmo_core.optim.adamw.AdamWConfig", "type": "adamw"}, "max_grad_norm": 1.0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "units": "steps", "warmup": 570, "alpha_f": 0.1, "warmup_min_lr": 0.0, "_CLASS_": "olmo_core.optim.scheduler.CosWithWarmup", "type": "cos_with_warmup"}, "compile_model": true, "dp_config": {"name": "hsdp", "param_dtype": "bfloat16", "reduce_dtype": "float32", "wrapping_strategy": "full", "prefetch_factor": 0, "only_allreduce_last_microbatch": true, "reduce_grads_in_fp32": true, "accumulate_grads_in_fp32": true, "use_reduce_scatter": false, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig"}, "passk_k": 1, "passk_log_top": 65536, "label_ignore_index": -100, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerTrainModuleConfig"}, "trainer": {"save_folder": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/checkpoints", "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "load_strategy": "if_available", "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "olmo_core.train.checkpoint.CheckpointerConfig"}, "save_overwrite": false, "max_duration": {"value": 119476492800, "unit": "tokens", "_CLASS_": "olmo_core.train.common.Duration"}, "cancel_check_interval": 10, "metrics_collect_interval": 10, "callbacks": {"gpu_monitor": {"_CLASS_": "olmo_core.train.callbacks.gpu_memory_monitor.GPUMemoryMonitorCallback"}, "config_saver": {"fname": "config.json", "_CLASS_": "olmo_core.train.callbacks.config_saver.ConfigSaverCallback"}, "garbage_collector": {"gc_interval": 1000, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.garbage_collector.GarbageCollectorCallback"}, "lm_evaluator": {"eval_dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "mix": "v3-small-ppl-validation", "mix_base_dir": "/data/home/guanning/olmo_data/olmo-data.org", "expand_glob": false, "include_instance_metadata": true, "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyPaddedFSLDatasetConfig"}, "eval_interval": 2000, "eval_on_startup": false, "eval_on_finish": true, "cancel_after_first_eval": false, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "deterministic": true, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.LMEvaluatorCallbackConfig"}, "downstream_evaluator": {"tasks": ["arc_challenge_test_bpb_5shot", "arc_challenge_test_mc_5shot_fast", "arc_easy_test_bpb_5shot", "arc_easy_test_mc_5shot_fast", "basic_skills_arithmetic_rc_5shot", "basic_skills_coding_rc_5shot", "basic_skills_common_knowledge_rc_5shot", "basic_skills_logical_reasoning_rc_5shot", "basic_skills_pattern_rc_5shot", "basic_skills_string_operations_rc_5shot", "codex_humaneval_gold_bpb_3shot", "codex_mbpp_gold_bpb_3shot", "copycolors_10way_fast", "hellaswag_bpb_5shot", "minerva_math_500_gold_bpb_0shot", "mmlu_humanities_test_bpb_5shot", "mmlu_humanities_test_mc_5shot_fast", "mmlu_other_test_bpb_5shot", "mmlu_other_test_mc_5shot_fast", "mmlu_social_sciences_test_bpb_5shot", "mmlu_social_sciences_test_mc_5shot_fast", "mmlu_stem_test_bpb_5shot", "mmlu_stem_test_mc_5shot_fast", "mt_mbpp_cpp_gold_bpb_3shot", "mt_mbpp_java_gold_bpb_3shot", "mt_mbpp_rust_gold_bpb_3shot"], "tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "eval_interval": 10000, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "eval_on_startup": false, "eval_on_finish": true, "cancel_after_first_eval": false, "log_interval": 5, "lazy": false, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.DownstreamEvaluatorCallbackConfig"}, "checkpointer": {"save_interval": 5000, "ephemeral_save_interval": 1000, "save_async": true, "remove": "ephemeral_only", "max_checkpoints": 3, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.checkpointer.CheckpointerCallback"}, "wandb": {"enabled": true, "name": "olmo2-600M-10xC-k1", "project": "olmo-pretrain-1018", "entity": "train-llm", "group": "olmo2-600M-10xC", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "_CLASS_": "olmo_core.train.callbacks.wandb.WandBCallback"}}, "bookkeeping_soft_timeout": 30, "no_checkpoints": false, "no_evals": false, "_CLASS_": "olmo_core.train.config.TrainerConfig"}, "init_seed": 2352, "_CLASS_": "__main__.ExperimentConfig"}

pretrain/olmo2-600M-10xC-k1/step105000-hf/chat_template.jinja

@@ -0,0 +1,4 @@
+{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

pretrain/olmo2-600M-10xC-k1/step105000-hf/config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 1344,
+  "initializer_range": 0.02,
+  "intermediate_size": 5376,
+  "max_position_embeddings": 2048,
+  "model_type": "olmo2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 16,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 500000,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.15.1",
+  "use_cache": true,
+  "vocab_size": 100278
+}

pretrain/olmo2-600M-10xC-k1/step105000-hf/generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "do_sample": true,
+  "eos_token_id": 100257,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 100277,
+  "transformers_version": "5.15.1",
+  "use_cache": true
+}

pretrain/olmo2-600M-10xC-k1/step105000-hf/tokenizer-export-audit.json

@@ -0,0 +1,9 @@
+{
+  "passed": true,
+  "source": "allenai/dolma2-tokenizer",
+  "revision": null,
+  "backend_sha256": "30ddf87d4cc948a8bd56980b84605ee465064d0b5871c7bf2fa6d00f04c99b9a",
+  "tokenizer_sha256": "18e309ad7f9c60037eaf26aca401a96128187cad2bdfa01a508dca7526bf6300",
+  "tokenizer_class": "TokenizersBackend",
+  "probe_count": 8
+}

pretrain/olmo2-600M-10xC-k1/step105000-hf/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 2048,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>"
+}

pretrain/olmo2-600M-10xC-k1/step105000/.metadata.json

@@ -0,0 +1 @@
+{"ephemeral": false, "version": "3.0.0"}

pretrain/olmo2-600M-10xC-k1/step105000/config.json

@@ -0,0 +1 @@
+{"model": {"d_model": 1344, "vocab_size": 100352, "n_layers": 16, "block": {"sequence_mixer": {"name": "default", "n_heads": 16, "bias": false, "rope": {"name": "default", "theta": 500000, "full_precision": true, "no_global_rope": false, "partial_rotary_factor": 1.0, "_CLASS_": "olmo_core.nn.rope.RoPEConfig"}, "qk_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "backend": "torch", "dtype": "float32", "use_head_qk_norm": false, "scalable_softmax": false, "attention_sinks": false, "_CLASS_": "olmo_core.nn.attention.AttentionConfig", "type": "attention"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "feed_forward": {"hidden_size": 5376, "name": "default", "bias": false, "dtype": "float32", "activation": "silu", "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"}, "name": "reordered_norm", "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "float32", "loss_implementation": "default", "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"}, "name": "default", "dtype": "float32", "init_method": "normal", "init_seed": 2352, "init_std": 0.02, "tie_word_embeddings": false, "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "paths": ["/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.prefix6308545764.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.prefix9671333560.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.prefix3431295592.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/allenai/dolma2-tokenizer/part-09-00000.prefix2012701596.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.prefix1952181972.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-009-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-065-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-038-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-087-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-011-00000.prefix1990182664.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.prefix604873468.npy"], "expand_glob": false, "metadata": [{"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "pes2o"}, {"label": "proofpile-2-arxiv"}, {"label": "proofpile-2-open-web-math"}, {"label": "proofpile-2-stack"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "wikipedia"}], "include_instance_metadata": true, "instance_filter_config": {"repetition_max_period": 13, "repetition_min_period": 1, "repetition_max_count": 32, "_CLASS_": "olmo_core.data.numpy_dataset.InstanceFilterConfig"}, "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "generate_doc_lengths": false, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyFSLDatasetConfig"}, "data_loader": {"global_batch_size": 1048576, "seed": 34521, "num_workers": 4, "ignore_fingerprint_mismatch": false, "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig", "type": "numpy"}, "train_module": {"rank_microbatch_size": 65536, "max_sequence_length": 2048, "optim": {"group_overrides": [{"params": ["embeddings.weight"], "opts": {"weight_decay": 0.0}, "_CLASS_": "olmo_core.optim.config.OptimGroupOverride"}], "compile": false, "fixed_fields": ["initial_lr"], "lr": 0.002657591869667391, "betas": [0.9, 0.95], "eps": 1e-08, "weight_decay": 0.1, "fused": true, "_CLASS_": "olmo_core.optim.adamw.AdamWConfig", "type": "adamw"}, "max_grad_norm": 1.0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "units": "steps", "warmup": 570, "alpha_f": 0.1, "warmup_min_lr": 0.0, "_CLASS_": "olmo_core.optim.scheduler.CosWithWarmup", "type": "cos_with_warmup"}, "compile_model": true, "dp_config": {"name": "hsdp", "param_dtype": "bfloat16", "reduce_dtype": "float32", "wrapping_strategy": "full", "prefetch_factor": 0, "only_allreduce_last_microbatch": true, "reduce_grads_in_fp32": true, "accumulate_grads_in_fp32": true, "use_reduce_scatter": false, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig"}, "passk_k": 1, "passk_log_top": 65536, "label_ignore_index": -100, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerTrainModuleConfig"}, "trainer": {"save_folder": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/checkpoints", "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "load_strategy": "if_available", "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "olmo_core.train.checkpoint.CheckpointerConfig"}, "save_overwrite": false, "max_duration": {"value": 119476492800, "unit": "tokens", "_CLASS_": "olmo_core.train.common.Duration"}, "cancel_check_interval": 10, "metrics_collect_interval": 10, "callbacks": {"gpu_monitor": {"_CLASS_": "olmo_core.train.callbacks.gpu_memory_monitor.GPUMemoryMonitorCallback"}, "config_saver": {"fname": "config.json", "_CLASS_": "olmo_core.train.callbacks.config_saver.ConfigSaverCallback"}, "garbage_collector": {"gc_interval": 1000, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.garbage_collector.GarbageCollectorCallback"}, "lm_evaluator": {"eval_dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "mix": "v3-small-ppl-validation", "mix_base_dir": "/data/home/guanning/olmo_data/olmo-data.org", "expand_glob": false, "include_instance_metadata": true, "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyPaddedFSLDatasetConfig"}, "eval_interval": 2000, "eval_on_startup": false, "eval_on_finish": true, "cancel_after_first_eval": false, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "deterministic": true, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.LMEvaluatorCallbackConfig"}, "downstream_evaluator": {"tasks": ["arc_challenge_test_bpb_5shot", "arc_challenge_test_mc_5shot_fast", "arc_easy_test_bpb_5shot", "arc_easy_test_mc_5shot_fast", "basic_skills_arithmetic_rc_5shot", "basic_skills_coding_rc_5shot", "basic_skills_common_knowledge_rc_5shot", "basic_skills_logical_reasoning_rc_5shot", "basic_skills_pattern_rc_5shot", "basic_skills_string_operations_rc_5shot", "codex_humaneval_gold_bpb_3shot", "codex_mbpp_gold_bpb_3shot", "copycolors_10way_fast", "hellaswag_bpb_5shot", "minerva_math_500_gold_bpb_0shot", "mmlu_humanities_test_bpb_5shot", "mmlu_humanities_test_mc_5shot_fast", "mmlu_other_test_bpb_5shot", "mmlu_other_test_mc_5shot_fast", "mmlu_social_sciences_test_bpb_5shot", "mmlu_social_sciences_test_mc_5shot_fast", "mmlu_stem_test_bpb_5shot", "mmlu_stem_test_mc_5shot_fast", "mt_mbpp_cpp_gold_bpb_3shot", "mt_mbpp_java_gold_bpb_3shot", "mt_mbpp_rust_gold_bpb_3shot"], "tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "eval_interval": 10000, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "eval_on_startup": false, "eval_on_finish": true, "cancel_after_first_eval": false, "log_interval": 5, "lazy": false, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.DownstreamEvaluatorCallbackConfig"}, "checkpointer": {"save_interval": 5000, "ephemeral_save_interval": 1000, "save_async": true, "remove": "ephemeral_only", "max_checkpoints": 3, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.checkpointer.CheckpointerCallback"}, "wandb": {"enabled": true, "name": "olmo2-600M-10xC-k1", "project": "olmo-pretrain-1018", "entity": "train-llm", "group": "olmo2-600M-10xC", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "_CLASS_": "olmo_core.train.callbacks.wandb.WandBCallback"}}, "bookkeeping_soft_timeout": 30, "no_checkpoints": false, "no_evals": false, "_CLASS_": "olmo_core.train.config.TrainerConfig"}, "init_seed": 2352, "_CLASS_": "__main__.ExperimentConfig"}

pretrain/olmo2-600M-10xC-k1/step105000/data_paths.txt

@@ -0,0 +1,50 @@
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.prefix6308545764.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.prefix9671333560.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.prefix3431295592.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/allenai/dolma2-tokenizer/part-09-00000.prefix2012701596.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.prefix1952181972.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-009-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-065-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-038-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-087-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-011-00000.prefix1990182664.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.prefix604873468.npy

pretrain/olmo2-600M-10xC-k1/step110000-hf/chat_template.jinja

@@ -0,0 +1,4 @@
+{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

pretrain/olmo2-600M-10xC-k1/step110000-hf/config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 1344,
+  "initializer_range": 0.02,
+  "intermediate_size": 5376,
+  "max_position_embeddings": 2048,
+  "model_type": "olmo2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 16,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 500000,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.15.1",
+  "use_cache": true,
+  "vocab_size": 100278
+}

pretrain/olmo2-600M-10xC-k1/step110000-hf/generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "do_sample": true,
+  "eos_token_id": 100257,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 100277,
+  "transformers_version": "5.15.1",
+  "use_cache": true
+}

pretrain/olmo2-600M-10xC-k1/step110000-hf/tokenizer-export-audit.json

@@ -0,0 +1,9 @@
+{
+  "passed": true,
+  "source": "allenai/dolma2-tokenizer",
+  "revision": null,
+  "backend_sha256": "30ddf87d4cc948a8bd56980b84605ee465064d0b5871c7bf2fa6d00f04c99b9a",
+  "tokenizer_sha256": "18e309ad7f9c60037eaf26aca401a96128187cad2bdfa01a508dca7526bf6300",
+  "tokenizer_class": "TokenizersBackend",
+  "probe_count": 8
+}

pretrain/olmo2-600M-10xC-k1/step110000-hf/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 2048,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>"
+}

pretrain/olmo2-600M-10xC-k1/step110000/.metadata.json

@@ -0,0 +1 @@
+{"ephemeral": false, "version": "3.0.0"}

pretrain/olmo2-600M-10xC-k1/step110000/config.json

@@ -0,0 +1 @@
+{"model": {"d_model": 1344, "vocab_size": 100352, "n_layers": 16, "block": {"sequence_mixer": {"name": "default", "n_heads": 16, "bias": false, "rope": {"name": "default", "theta": 500000, "full_precision": true, "no_global_rope": false, "partial_rotary_factor": 1.0, "_CLASS_": "olmo_core.nn.rope.RoPEConfig"}, "qk_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "backend": "torch", "dtype": "float32", "use_head_qk_norm": false, "scalable_softmax": false, "attention_sinks": false, "_CLASS_": "olmo_core.nn.attention.AttentionConfig", "type": "attention"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "feed_forward": {"hidden_size": 5376, "name": "default", "bias": false, "dtype": "float32", "activation": "silu", "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"}, "name": "reordered_norm", "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "float32", "loss_implementation": "default", "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"}, "name": "default", "dtype": "float32", "init_method": "normal", "init_seed": 2352, "init_std": 0.02, "tie_word_embeddings": false, "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "paths": ["/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.prefix6308545764.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.prefix9671333560.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.prefix3431295592.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/allenai/dolma2-tokenizer/part-09-00000.prefix2012701596.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.prefix1952181972.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-009-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-065-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-038-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-087-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-011-00000.prefix1990182664.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.prefix604873468.npy"], "expand_glob": false, "metadata": [{"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "pes2o"}, {"label": "proofpile-2-arxiv"}, {"label": "proofpile-2-open-web-math"}, {"label": "proofpile-2-stack"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "wikipedia"}], "include_instance_metadata": true, "instance_filter_config": {"repetition_max_period": 13, "repetition_min_period": 1, "repetition_max_count": 32, "_CLASS_": "olmo_core.data.numpy_dataset.InstanceFilterConfig"}, "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "generate_doc_lengths": false, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyFSLDatasetConfig"}, "data_loader": {"global_batch_size": 1048576, "seed": 34521, "num_workers": 4, "ignore_fingerprint_mismatch": false, "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig", "type": "numpy"}, "train_module": {"rank_microbatch_size": 65536, "max_sequence_length": 2048, "optim": {"group_overrides": [{"params": ["embeddings.weight"], "opts": {"weight_decay": 0.0}, "_CLASS_": "olmo_core.optim.config.OptimGroupOverride"}], "compile": false, "fixed_fields": ["initial_lr"], "lr": 0.002657591869667391, "betas": [0.9, 0.95], "eps": 1e-08, "weight_decay": 0.1, "fused": true, "_CLASS_": "olmo_core.optim.adamw.AdamWConfig", "type": "adamw"}, "max_grad_norm": 1.0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "units": "steps", "warmup": 570, "alpha_f": 0.1, "warmup_min_lr": 0.0, "_CLASS_": "olmo_core.optim.scheduler.CosWithWarmup", "type": "cos_with_warmup"}, "compile_model": true, "dp_config": {"name": "hsdp", "param_dtype": "bfloat16", "reduce_dtype": "float32", "wrapping_strategy": "full", "prefetch_factor": 0, "only_allreduce_last_microbatch": true, "reduce_grads_in_fp32": true, "accumulate_grads_in_fp32": true, "use_reduce_scatter": false, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig"}, "passk_k": 1, "passk_log_top": 65536, "label_ignore_index": -100, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerTrainModuleConfig"}, "trainer": {"save_folder": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/checkpoints", "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "load_strategy": "if_available", "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "olmo_core.train.checkpoint.CheckpointerConfig"}, "save_overwrite": false, "max_duration": {"value": 119476492800, "unit": "tokens", "_CLASS_": "olmo_core.train.common.Duration"}, "cancel_check_interval": 10, "metrics_collect_interval": 10, "callbacks": {"gpu_monitor": {"_CLASS_": "olmo_core.train.callbacks.gpu_memory_monitor.GPUMemoryMonitorCallback"}, "config_saver": {"fname": "config.json", "_CLASS_": "olmo_core.train.callbacks.config_saver.ConfigSaverCallback"}, "garbage_collector": {"gc_interval": 1000, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.garbage_collector.GarbageCollectorCallback"}, "lm_evaluator": {"eval_dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "mix": "v3-small-ppl-validation", "mix_base_dir": "/data/home/guanning/olmo_data/olmo-data.org", "expand_glob": false, "include_instance_metadata": true, "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyPaddedFSLDatasetConfig"}, "eval_interval": 2000, "eval_on_startup": false, "eval_on_finish": true, "cancel_after_first_eval": false, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "deterministic": true, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.LMEvaluatorCallbackConfig"}, "downstream_evaluator": {"tasks": ["arc_challenge_test_bpb_5shot", "arc_challenge_test_mc_5shot_fast", "arc_easy_test_bpb_5shot", "arc_easy_test_mc_5shot_fast", "basic_skills_arithmetic_rc_5shot", "basic_skills_coding_rc_5shot", "basic_skills_common_knowledge_rc_5shot", "basic_skills_logical_reasoning_rc_5shot", "basic_skills_pattern_rc_5shot", "basic_skills_string_operations_rc_5shot", "codex_humaneval_gold_bpb_3shot", "codex_mbpp_gold_bpb_3shot", "copycolors_10way_fast", "hellaswag_bpb_5shot", "minerva_math_500_gold_bpb_0shot", "mmlu_humanities_test_bpb_5shot", "mmlu_humanities_test_mc_5shot_fast", "mmlu_other_test_bpb_5shot", "mmlu_other_test_mc_5shot_fast", "mmlu_social_sciences_test_bpb_5shot", "mmlu_social_sciences_test_mc_5shot_fast", "mmlu_stem_test_bpb_5shot", "mmlu_stem_test_mc_5shot_fast", "mt_mbpp_cpp_gold_bpb_3shot", "mt_mbpp_java_gold_bpb_3shot", "mt_mbpp_rust_gold_bpb_3shot"], "tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "eval_interval": 10000, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "eval_on_startup": false, "eval_on_finish": true, "cancel_after_first_eval": false, "log_interval": 5, "lazy": false, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.DownstreamEvaluatorCallbackConfig"}, "checkpointer": {"save_interval": 5000, "ephemeral_save_interval": 1000, "save_async": true, "remove": "ephemeral_only", "max_checkpoints": 3, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.checkpointer.CheckpointerCallback"}, "wandb": {"enabled": true, "name": "olmo2-600M-10xC-k1", "project": "olmo-pretrain-1018", "entity": "train-llm", "group": "olmo2-600M-10xC", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "_CLASS_": "olmo_core.train.callbacks.wandb.WandBCallback"}}, "bookkeeping_soft_timeout": 30, "no_checkpoints": false, "no_evals": false, "_CLASS_": "olmo_core.train.config.TrainerConfig"}, "init_seed": 2352, "_CLASS_": "__main__.ExperimentConfig"}

pretrain/olmo2-600M-10xC-k1/step110000/data_paths.txt

@@ -0,0 +1,50 @@
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.prefix6308545764.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.prefix9671333560.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.prefix3431295592.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/allenai/dolma2-tokenizer/part-09-00000.prefix2012701596.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.prefix1952181972.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-009-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-065-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-038-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-087-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-011-00000.prefix1990182664.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.prefix604873468.npy

pretrain/olmo2-600M-10xC-k1/step113000-hf/chat_template.jinja

@@ -0,0 +1,4 @@
+{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

pretrain/olmo2-600M-10xC-k1/step113000-hf/config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 1344,
+  "initializer_range": 0.02,
+  "intermediate_size": 5376,
+  "max_position_embeddings": 2048,
+  "model_type": "olmo2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 16,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 500000,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.15.1",
+  "use_cache": true,
+  "vocab_size": 100278
+}

pretrain/olmo2-600M-10xC-k1/step113000-hf/tokenizer-export-audit.json

@@ -0,0 +1,9 @@
+{
+  "passed": true,
+  "source": "allenai/dolma2-tokenizer",
+  "revision": null,
+  "backend_sha256": "30ddf87d4cc948a8bd56980b84605ee465064d0b5871c7bf2fa6d00f04c99b9a",
+  "tokenizer_sha256": "18e309ad7f9c60037eaf26aca401a96128187cad2bdfa01a508dca7526bf6300",
+  "tokenizer_class": "TokenizersBackend",
+  "probe_count": 8
+}

pretrain/olmo2-600M-10xC-k1/step113000/.metadata.json

@@ -0,0 +1 @@
+{"ephemeral": true, "version": "3.0.0"}

pretrain/olmo2-600M-10xC-k1/step113000/config.json

@@ -0,0 +1 @@
+{"model": {"d_model": 1344, "vocab_size": 100352, "n_layers": 16, "block": {"sequence_mixer": {"name": "default", "n_heads": 16, "bias": false, "rope": {"name": "default", "theta": 500000, "full_precision": true, "no_global_rope": false, "partial_rotary_factor": 1.0, "_CLASS_": "olmo_core.nn.rope.RoPEConfig"}, "qk_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "backend": "torch", "dtype": "float32", "use_head_qk_norm": false, "scalable_softmax": false, "attention_sinks": false, "_CLASS_": "olmo_core.nn.attention.AttentionConfig", "type": "attention"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "feed_forward": {"hidden_size": 5376, "name": "default", "bias": false, "dtype": "float32", "activation": "silu", "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"}, "name": "reordered_norm", "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "float32", "loss_implementation": "default", "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"}, "name": "default", "dtype": "float32", "init_method": "normal", "init_seed": 2352, "init_std": 0.02, "tie_word_embeddings": false, "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "paths": ["/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.prefix6308545764.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.prefix9671333560.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.prefix3431295592.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/allenai/dolma2-tokenizer/part-09-00000.prefix2012701596.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.prefix1952181972.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-009-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-065-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-038-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-087-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-011-00000.prefix1990182664.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.prefix604873468.npy"], "expand_glob": false, "metadata": [{"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "pes2o"}, {"label": "proofpile-2-arxiv"}, {"label": "proofpile-2-open-web-math"}, {"label": "proofpile-2-stack"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "wikipedia"}], "include_instance_metadata": true, "instance_filter_config": {"repetition_max_period": 13, "repetition_min_period": 1, "repetition_max_count": 32, "_CLASS_": "olmo_core.data.numpy_dataset.InstanceFilterConfig"}, "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "generate_doc_lengths": false, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyFSLDatasetConfig"}, "data_loader": {"global_batch_size": 1048576, "seed": 34521, "num_workers": 4, "ignore_fingerprint_mismatch": false, "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig", "type": "numpy"}, "train_module": {"rank_microbatch_size": 65536, "max_sequence_length": 2048, "optim": {"group_overrides": [{"params": ["embeddings.weight"], "opts": {"weight_decay": 0.0}, "_CLASS_": "olmo_core.optim.config.OptimGroupOverride"}], "compile": false, "fixed_fields": ["initial_lr"], "lr": 0.002657591869667391, "betas": [0.9, 0.95], "eps": 1e-08, "weight_decay": 0.1, "fused": true, "_CLASS_": "olmo_core.optim.adamw.AdamWConfig", "type": "adamw"}, "max_grad_norm": 1.0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "units": "steps", "warmup": 570, "alpha_f": 0.1, "warmup_min_lr": 0.0, "_CLASS_": "olmo_core.optim.scheduler.CosWithWarmup", "type": "cos_with_warmup"}, "compile_model": true, "dp_config": {"name": "hsdp", "param_dtype": "bfloat16", "reduce_dtype": "float32", "wrapping_strategy": "full", "prefetch_factor": 0, "only_allreduce_last_microbatch": true, "reduce_grads_in_fp32": true, "accumulate_grads_in_fp32": true, "use_reduce_scatter": false, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig"}, "passk_k": 1, "passk_log_top": 65536, "label_ignore_index": -100, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerTrainModuleConfig"}, "trainer": {"save_folder": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/checkpoints", "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "load_strategy": "if_available", "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "olmo_core.train.checkpoint.CheckpointerConfig"}, "save_overwrite": false, "max_duration": {"value": 119476492800, "unit": "tokens", "_CLASS_": "olmo_core.train.common.Duration"}, "cancel_check_interval": 10, "metrics_collect_interval": 10, "callbacks": {"gpu_monitor": {"_CLASS_": "olmo_core.train.callbacks.gpu_memory_monitor.GPUMemoryMonitorCallback"}, "config_saver": {"fname": "config.json", "_CLASS_": "olmo_core.train.callbacks.config_saver.ConfigSaverCallback"}, "garbage_collector": {"gc_interval": 1000, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.garbage_collector.GarbageCollectorCallback"}, "lm_evaluator": {"eval_dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "mix": "v3-small-ppl-validation", "mix_base_dir": "/data/home/guanning/olmo_data/olmo-data.org", "expand_glob": false, "include_instance_metadata": true, "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyPaddedFSLDatasetConfig"}, "eval_interval": 2000, "eval_on_startup": false, "eval_on_finish": true, "cancel_after_first_eval": false, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "deterministic": true, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.LMEvaluatorCallbackConfig"}, "downstream_evaluator": {"tasks": ["arc_challenge_test_bpb_5shot", "arc_challenge_test_mc_5shot_fast", "arc_easy_test_bpb_5shot", "arc_easy_test_mc_5shot_fast", "basic_skills_arithmetic_rc_5shot", "basic_skills_coding_rc_5shot", "basic_skills_common_knowledge_rc_5shot", "basic_skills_logical_reasoning_rc_5shot", "basic_skills_pattern_rc_5shot", "basic_skills_string_operations_rc_5shot", "codex_humaneval_gold_bpb_3shot", "codex_mbpp_gold_bpb_3shot", "copycolors_10way_fast", "hellaswag_bpb_5shot", "minerva_math_500_gold_bpb_0shot", "mmlu_humanities_test_bpb_5shot", "mmlu_humanities_test_mc_5shot_fast", "mmlu_other_test_bpb_5shot", "mmlu_other_test_mc_5shot_fast", "mmlu_social_sciences_test_bpb_5shot", "mmlu_social_sciences_test_mc_5shot_fast", "mmlu_stem_test_bpb_5shot", "mmlu_stem_test_mc_5shot_fast", "mt_mbpp_cpp_gold_bpb_3shot", "mt_mbpp_java_gold_bpb_3shot", "mt_mbpp_rust_gold_bpb_3shot"], "tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "eval_interval": 10000, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "eval_on_startup": false, "eval_on_finish": true, "cancel_after_first_eval": false, "log_interval": 5, "lazy": false, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.DownstreamEvaluatorCallbackConfig"}, "checkpointer": {"save_interval": 5000, "ephemeral_save_interval": 1000, "save_async": true, "remove": "ephemeral_only", "max_checkpoints": 3, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.checkpointer.CheckpointerCallback"}, "wandb": {"enabled": true, "name": "olmo2-600M-10xC-k1", "project": "olmo-pretrain-1018", "entity": "train-llm", "group": "olmo2-600M-10xC", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "_CLASS_": "olmo_core.train.callbacks.wandb.WandBCallback"}}, "bookkeeping_soft_timeout": 30, "no_checkpoints": false, "no_evals": false, "_CLASS_": "olmo_core.train.config.TrainerConfig"}, "init_seed": 2352, "_CLASS_": "__main__.ExperimentConfig"}

pretrain/olmo2-600M-10xC-k1/step113000/data_paths.txt

@@ -0,0 +1,50 @@
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.prefix6308545764.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.prefix9671333560.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.prefix3431295592.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/allenai/dolma2-tokenizer/part-09-00000.prefix2012701596.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.prefix1952181972.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-009-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-065-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-038-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-087-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-011-00000.prefix1990182664.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.prefix604873468.npy

pretrain/olmo2-600M-10xC-k1/step113942-hf/chat_template.jinja

@@ -0,0 +1,4 @@
+{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

pretrain/olmo2-600M-10xC-k1/step113942-hf/config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 1344,
+  "initializer_range": 0.02,
+  "intermediate_size": 5376,
+  "max_position_embeddings": 2048,
+  "model_type": "olmo2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 16,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 500000,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.15.1",
+  "use_cache": true,
+  "vocab_size": 100278
+}

pretrain/olmo2-600M-10xC-k1/step113942-hf/generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "do_sample": true,
+  "eos_token_id": 100257,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 100277,
+  "transformers_version": "5.15.1",
+  "use_cache": true
+}

pretrain/olmo2-600M-10xC-k1/step113942-hf/tokenizer-export-audit.json

@@ -0,0 +1,9 @@
+{
+  "passed": true,
+  "source": "allenai/dolma2-tokenizer",
+  "revision": null,
+  "backend_sha256": "30ddf87d4cc948a8bd56980b84605ee465064d0b5871c7bf2fa6d00f04c99b9a",
+  "tokenizer_sha256": "18e309ad7f9c60037eaf26aca401a96128187cad2bdfa01a508dca7526bf6300",
+  "tokenizer_class": "TokenizersBackend",
+  "probe_count": 8
+}

pretrain/olmo2-600M-10xC-k1/step113942-hf/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 2048,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>"
+}

pretrain/olmo2-600M-10xC-k1/step113942/.metadata.json

@@ -0,0 +1 @@
+{"ephemeral": false, "version": "3.0.0"}

pretrain/olmo2-600M-10xC-k1/step113942/config.json

@@ -0,0 +1 @@
+{"model": {"d_model": 1344, "vocab_size": 100352, "n_layers": 16, "block": {"sequence_mixer": {"name": "default", "n_heads": 16, "bias": false, "rope": {"name": "default", "theta": 500000, "full_precision": true, "no_global_rope": false, "partial_rotary_factor": 1.0, "_CLASS_": "olmo_core.nn.rope.RoPEConfig"}, "qk_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "backend": "torch", "dtype": "float32", "use_head_qk_norm": false, "scalable_softmax": false, "attention_sinks": false, "_CLASS_": "olmo_core.nn.attention.AttentionConfig", "type": "attention"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "feed_forward": {"hidden_size": 5376, "name": "default", "bias": false, "dtype": "float32", "activation": "silu", "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"}, "name": "reordered_norm", "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "float32", "loss_implementation": "default", "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"}, "name": "default", "dtype": "float32", "init_method": "normal", "init_seed": 2352, "init_std": 0.02, "tie_word_embeddings": false, "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "paths": ["/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.prefix6308545764.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.prefix9671333560.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.prefix3431295592.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/allenai/dolma2-tokenizer/part-09-00000.prefix2012701596.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.prefix1952181972.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-009-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-065-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-038-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-087-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-011-00000.prefix1990182664.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.prefix604873468.npy"], "expand_glob": false, "metadata": [{"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "pes2o"}, {"label": "proofpile-2-arxiv"}, {"label": "proofpile-2-open-web-math"}, {"label": "proofpile-2-stack"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "wikipedia"}], "include_instance_metadata": true, "instance_filter_config": {"repetition_max_period": 13, "repetition_min_period": 1, "repetition_max_count": 32, "_CLASS_": "olmo_core.data.numpy_dataset.InstanceFilterConfig"}, "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "generate_doc_lengths": false, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyFSLDatasetConfig"}, "data_loader": {"global_batch_size": 1048576, "seed": 34521, "num_workers": 4, "ignore_fingerprint_mismatch": false, "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig", "type": "numpy"}, "train_module": {"rank_microbatch_size": 65536, "max_sequence_length": 2048, "optim": {"group_overrides": [{"params": ["embeddings.weight"], "opts": {"weight_decay": 0.0}, "_CLASS_": "olmo_core.optim.config.OptimGroupOverride"}], "compile": false, "fixed_fields": ["initial_lr"], "lr": 0.002657591869667391, "betas": [0.9, 0.95], "eps": 1e-08, "weight_decay": 0.1, "fused": true, "_CLASS_": "olmo_core.optim.adamw.AdamWConfig", "type": "adamw"}, "max_grad_norm": 1.0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "units": "steps", "warmup": 570, "alpha_f": 0.1, "warmup_min_lr": 0.0, "_CLASS_": "olmo_core.optim.scheduler.CosWithWarmup", "type": "cos_with_warmup"}, "compile_model": true, "dp_config": {"name": "hsdp", "param_dtype": "bfloat16", "reduce_dtype": "float32", "wrapping_strategy": "full", "prefetch_factor": 0, "only_allreduce_last_microbatch": true, "reduce_grads_in_fp32": true, "accumulate_grads_in_fp32": true, "use_reduce_scatter": false, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig"}, "passk_k": 1, "passk_log_top": 65536, "label_ignore_index": -100, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerTrainModuleConfig"}, "trainer": {"save_folder": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/checkpoints", "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "load_strategy": "if_available", "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "olmo_core.train.checkpoint.CheckpointerConfig"}, "save_overwrite": false, "max_duration": {"value": 119476492800, "unit": "tokens", "_CLASS_": "olmo_core.train.common.Duration"}, "cancel_check_interval": 10, "metrics_collect_interval": 10, "callbacks": {"gpu_monitor": {"_CLASS_": "olmo_core.train.callbacks.gpu_memory_monitor.GPUMemoryMonitorCallback"}, "config_saver": {"fname": "config.json", "_CLASS_": "olmo_core.train.callbacks.config_saver.ConfigSaverCallback"}, "garbage_collector": {"gc_interval": 1000, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.garbage_collector.GarbageCollectorCallback"}, "lm_evaluator": {"eval_dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "mix": "v3-small-ppl-validation", "mix_base_dir": "/data/home/guanning/olmo_data/olmo-data.org", "expand_glob": false, "include_instance_metadata": true, "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k1/work", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyPaddedFSLDatasetConfig"}, "eval_interval": 2000, "eval_on_startup": false, "eval_on_finish": true, "cancel_after_first_eval": false, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "deterministic": true, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.LMEvaluatorCallbackConfig"}, "downstream_evaluator": {"tasks": ["arc_challenge_test_bpb_5shot", "arc_challenge_test_mc_5shot_fast", "arc_easy_test_bpb_5shot", "arc_easy_test_mc_5shot_fast", "basic_skills_arithmetic_rc_5shot", "basic_skills_coding_rc_5shot", "basic_skills_common_knowledge_rc_5shot", "basic_skills_logical_reasoning_rc_5shot", "basic_skills_pattern_rc_5shot", "basic_skills_string_operations_rc_5shot", "codex_humaneval_gold_bpb_3shot", "codex_mbpp_gold_bpb_3shot", "copycolors_10way_fast", "hellaswag_bpb_5shot", "minerva_math_500_gold_bpb_0shot", "mmlu_humanities_test_bpb_5shot", "mmlu_humanities_test_mc_5shot_fast", "mmlu_other_test_bpb_5shot", "mmlu_other_test_mc_5shot_fast", "mmlu_social_sciences_test_bpb_5shot", "mmlu_social_sciences_test_mc_5shot_fast", "mmlu_stem_test_bpb_5shot", "mmlu_stem_test_mc_5shot_fast", "mt_mbpp_cpp_gold_bpb_3shot", "mt_mbpp_java_gold_bpb_3shot", "mt_mbpp_rust_gold_bpb_3shot"], "tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "eval_interval": 10000, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "eval_on_startup": false, "eval_on_finish": true, "cancel_after_first_eval": false, "log_interval": 5, "lazy": false, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.DownstreamEvaluatorCallbackConfig"}, "checkpointer": {"save_interval": 5000, "ephemeral_save_interval": 1000, "save_async": true, "remove": "ephemeral_only", "max_checkpoints": 3, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.checkpointer.CheckpointerCallback"}, "wandb": {"enabled": true, "name": "olmo2-600M-10xC-k1", "project": "olmo-pretrain-1018", "entity": "train-llm", "group": "olmo2-600M-10xC", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "_CLASS_": "olmo_core.train.callbacks.wandb.WandBCallback"}}, "bookkeeping_soft_timeout": 30, "no_checkpoints": false, "no_evals": false, "_CLASS_": "olmo_core.train.config.TrainerConfig"}, "init_seed": 2352, "_CLASS_": "__main__.ExperimentConfig"}

pretrain/olmo2-600M-10xC-k1/step113942/data_paths.txt

@@ -0,0 +1,50 @@
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.prefix6308545764.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.prefix9671333560.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.prefix3431295592.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/allenai/dolma2-tokenizer/part-09-00000.prefix2012701596.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.prefix1952181972.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-009-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-065-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-038-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-087-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-011-00000.prefix1990182664.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.prefix604873468.npy

pretrain/olmo2-600M-10xC-k16/step100000-hf/chat_template.jinja

@@ -0,0 +1,4 @@
+{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

pretrain/olmo2-600M-10xC-k16/step100000-hf/config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 1344,
+  "initializer_range": 0.02,
+  "intermediate_size": 5376,
+  "max_position_embeddings": 2048,
+  "model_type": "olmo2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 16,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 500000,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.15.1",
+  "use_cache": true,
+  "vocab_size": 100278
+}

pretrain/olmo2-600M-10xC-k16/step100000-hf/generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "do_sample": true,
+  "eos_token_id": 100257,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 100277,
+  "transformers_version": "5.15.1",
+  "use_cache": true
+}

pretrain/olmo2-600M-10xC-k16/step100000-hf/tokenizer-export-audit.json

@@ -0,0 +1,9 @@
+{
+  "passed": true,
+  "source": "allenai/dolma2-tokenizer",
+  "revision": null,
+  "backend_sha256": "30ddf87d4cc948a8bd56980b84605ee465064d0b5871c7bf2fa6d00f04c99b9a",
+  "tokenizer_sha256": "18e309ad7f9c60037eaf26aca401a96128187cad2bdfa01a508dca7526bf6300",
+  "tokenizer_class": "TokenizersBackend",
+  "probe_count": 8
+}

pretrain/olmo2-600M-10xC-k16/step100000-hf/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 2048,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|endoftext|>"
+}

pretrain/olmo2-600M-10xC-k16/step100000/.metadata.json

@@ -0,0 +1 @@
+{"ephemeral": false, "version": "3.0.0"}

pretrain/olmo2-600M-10xC-k16/step100000/config.json

@@ -0,0 +1 @@
+{"model": {"d_model": 1344, "vocab_size": 100352, "n_layers": 16, "block": {"sequence_mixer": {"name": "default", "n_heads": 16, "bias": false, "rope": {"name": "default", "theta": 500000, "full_precision": true, "no_global_rope": false, "partial_rotary_factor": 1.0, "_CLASS_": "olmo_core.nn.rope.RoPEConfig"}, "qk_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "backend": "torch", "dtype": "float32", "use_head_qk_norm": false, "scalable_softmax": false, "attention_sinks": false, "_CLASS_": "olmo_core.nn.attention.AttentionConfig", "type": "attention"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "feed_forward": {"hidden_size": 5376, "name": "default", "bias": false, "dtype": "float32", "activation": "silu", "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"}, "name": "reordered_norm", "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "float32", "loss_implementation": "default", "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"}, "name": "default", "dtype": "float32", "init_method": "normal", "init_seed": 2352, "init_std": 0.02, "tie_word_embeddings": false, "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "paths": ["/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.prefix6308545764.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.prefix9671333560.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.prefix3431295592.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/allenai/dolma2-tokenizer/part-09-00000.prefix2012701596.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.prefix1952181972.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-009-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-065-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-038-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-087-00000.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-011-00000.prefix1990182664.npy", "/data/home/guanning/olmo_data/olmo-data.org/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.prefix604873468.npy"], "expand_glob": false, "metadata": [{"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "dclm"}, {"label": "pes2o"}, {"label": "proofpile-2-arxiv"}, {"label": "proofpile-2-open-web-math"}, {"label": "proofpile-2-stack"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "starcoder"}, {"label": "wikipedia"}], "include_instance_metadata": true, "instance_filter_config": {"repetition_max_period": 13, "repetition_min_period": 1, "repetition_max_count": 32, "_CLASS_": "olmo_core.data.numpy_dataset.InstanceFilterConfig"}, "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k16/work", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "generate_doc_lengths": false, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyFSLDatasetConfig"}, "data_loader": {"global_batch_size": 1048576, "seed": 34521, "num_workers": 4, "ignore_fingerprint_mismatch": false, "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig", "type": "numpy"}, "train_module": {"rank_microbatch_size": 65536, "max_sequence_length": 2048, "optim": {"group_overrides": [{"params": ["embeddings.weight"], "opts": {"weight_decay": 0.0}, "_CLASS_": "olmo_core.optim.config.OptimGroupOverride"}], "compile": false, "fixed_fields": ["initial_lr"], "lr": 0.002657591869667391, "betas": [0.9, 0.95], "eps": 1e-08, "weight_decay": 0.1, "fused": true, "_CLASS_": "olmo_core.optim.adamw.AdamWConfig", "type": "adamw"}, "max_grad_norm": 1.0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "units": "steps", "warmup": 570, "alpha_f": 0.1, "warmup_min_lr": 0.0, "_CLASS_": "olmo_core.optim.scheduler.CosWithWarmup", "type": "cos_with_warmup"}, "compile_model": true, "dp_config": {"name": "hsdp", "param_dtype": "bfloat16", "reduce_dtype": "float32", "wrapping_strategy": "full", "prefetch_factor": 0, "only_allreduce_last_microbatch": true, "reduce_grads_in_fp32": true, "accumulate_grads_in_fp32": true, "use_reduce_scatter": false, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig"}, "passk_k": 16, "passk_log_top": 65536, "label_ignore_index": -100, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerTrainModuleConfig"}, "trainer": {"save_folder": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k16/checkpoints", "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k16/work", "load_strategy": "if_available", "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "olmo_core.train.checkpoint.CheckpointerConfig"}, "save_overwrite": false, "max_duration": {"value": 119476492800, "unit": "tokens", "_CLASS_": "olmo_core.train.common.Duration"}, "cancel_check_interval": 10, "metrics_collect_interval": 10, "callbacks": {"gpu_monitor": {"_CLASS_": "olmo_core.train.callbacks.gpu_memory_monitor.GPUMemoryMonitorCallback"}, "config_saver": {"fname": "config.json", "_CLASS_": "olmo_core.train.callbacks.config_saver.ConfigSaverCallback"}, "garbage_collector": {"gc_interval": 1000, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.garbage_collector.GarbageCollectorCallback"}, "lm_evaluator": {"eval_dataset": {"tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "mix": "v3-small-ppl-validation", "mix_base_dir": "/data/home/guanning/olmo_data/olmo-data.org", "expand_glob": false, "include_instance_metadata": true, "work_dir": "/data/home/guanning/olmo_runs/olmo2-600M-10xC-k16/work", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyPaddedFSLDatasetConfig"}, "eval_interval": 2000, "eval_on_startup": false, "eval_on_finish": true, "cancel_after_first_eval": false, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "deterministic": true, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.LMEvaluatorCallbackConfig"}, "downstream_evaluator": {"tasks": ["arc_challenge_test_bpb_5shot", "arc_challenge_test_mc_5shot_fast", "arc_easy_test_bpb_5shot", "arc_easy_test_mc_5shot_fast", "basic_skills_arithmetic_rc_5shot", "basic_skills_coding_rc_5shot", "basic_skills_common_knowledge_rc_5shot", "basic_skills_logical_reasoning_rc_5shot", "basic_skills_pattern_rc_5shot", "basic_skills_string_operations_rc_5shot", "codex_humaneval_gold_bpb_3shot", "codex_mbpp_gold_bpb_3shot", "copycolors_10way_fast", "hellaswag_bpb_5shot", "minerva_math_500_gold_bpb_0shot", "mmlu_humanities_test_bpb_5shot", "mmlu_humanities_test_mc_5shot_fast", "mmlu_other_test_bpb_5shot", "mmlu_other_test_mc_5shot_fast", "mmlu_social_sciences_test_bpb_5shot", "mmlu_social_sciences_test_mc_5shot_fast", "mmlu_stem_test_bpb_5shot", "mmlu_stem_test_mc_5shot_fast", "mt_mbpp_cpp_gold_bpb_3shot", "mt_mbpp_java_gold_bpb_3shot", "mt_mbpp_rust_gold_bpb_3shot"], "tokenizer": {"vocab_size": 100278, "eos_token_id": 100257, "pad_token_id": 100277, "identifier": "allenai/dolma2-tokenizer", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "eval_interval": 10000, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "eval_on_startup": false, "eval_on_finish": true, "cancel_after_first_eval": false, "log_interval": 5, "lazy": false, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.DownstreamEvaluatorCallbackConfig"}, "checkpointer": {"save_interval": 5000, "ephemeral_save_interval": 1000, "save_async": true, "remove": "ephemeral_only", "max_checkpoints": 3, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.checkpointer.CheckpointerCallback"}, "wandb": {"enabled": true, "name": "olmo2-600M-10xC-k16", "project": "olmo-pretrain-1018", "entity": "train-llm", "group": "olmo2-600M-10xC", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "_CLASS_": "olmo_core.train.callbacks.wandb.WandBCallback"}}, "bookkeeping_soft_timeout": 30, "no_checkpoints": false, "no_evals": false, "_CLASS_": "olmo_core.train.config.TrainerConfig"}, "init_seed": 2352, "_CLASS_": "__main__.ExperimentConfig"}

pretrain/olmo2-600M-10xC-k16/step100000/data_paths.txt

@@ -0,0 +1,50 @@
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.prefix6308545764.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.prefix9671333560.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.prefix3431295592.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/allenai/dolma2-tokenizer/part-09-00000.prefix2012701596.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.prefix1952181972.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-009-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-065-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-038-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-087-00000.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-011-00000.prefix1990182664.npy
+/data/home/guanning/olmo_data/olmo-data.org/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.prefix604873468.npy