Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,132 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_1.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_10.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_11.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_12.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_13.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_14.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_15.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_2.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_3.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_4.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_5.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_6.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_7.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_8.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__0_9.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_1.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_10.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_11.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_12.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_13.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_14.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_15.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_2.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_3.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_4.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_5.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_6.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_7.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_8.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__1_9.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_1.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_10.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_11.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_12.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_13.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_14.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_15.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_2.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_3.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_4.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_5.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_6.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_7.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_8.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__2_9.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_1.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_10.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_11.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_12.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_13.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_14.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_15.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_2.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_3.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_4.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_5.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_6.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_7.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_8.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__3_9.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_0.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_1.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_10.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_11.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_12.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_13.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_14.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_15.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_2.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_3.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_4.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_5.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_6.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_7.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_8.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__4_9.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_0.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_1.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_10.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_11.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_12.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_13.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_14.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_15.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_2.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_3.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_4.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_5.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_6.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_7.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_8.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__5_9.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_0.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_1.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_10.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_11.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_12.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_13.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_14.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_15.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_2.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_3.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_4.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_5.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_6.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_7.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_8.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__6_9.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_0.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_1.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_10.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_11.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_12.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_13.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_14.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_15.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_2.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_3.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_4.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_5.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_6.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_7.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_8.distcp filter=lfs diff=lfs merge=lfs -text
+olmo_core/model_and_optim/__7_9.distcp filter=lfs diff=lfs merge=lfs -text

olmo_core/.metadata.json

@@ -0,0 +1 @@
+{"version": "2.3.0"}

olmo_core/config.json

@@ -0,0 +1,349 @@
+{
+    "model": {
+        "d_model": 2048,
+        "vocab_size": 640,
+        "n_layers": 16,
+        "block": {
+            "attention": {
+                "name": "default",
+                "n_heads": 16,
+                "bias": false,
+                "rope": {
+                    "name": "default",
+                    "theta": 500000,
+                    "full_precision": true,
+                    "_CLASS_": "olmo_core.nn.rope.RoPEConfig"
+                },
+                "qk_norm": {
+                    "name": "rms",
+                    "eps": 1e-06,
+                    "bias": false,
+                    "dtype": "float32",
+                    "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"
+                },
+                "use_flash": true,
+                "dtype": "float32",
+                "_CLASS_": "olmo_core.nn.attention.AttentionConfig"
+            },
+            "layer_norm": {
+                "name": "rms",
+                "eps": 1e-06,
+                "bias": false,
+                "dtype": "float32",
+                "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"
+            },
+            "feed_forward": {
+                "hidden_size": 8192,
+                "name": "default",
+                "bias": false,
+                "dtype": "float32",
+                "act_name": "silu",
+                "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"
+            },
+            "name": "reordered_norm",
+            "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"
+        },
+        "lm_head": {
+            "name": "default",
+            "layer_norm": {
+                "name": "rms",
+                "eps": 1e-06,
+                "bias": false,
+                "dtype": "float32",
+                "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"
+            },
+            "bias": false,
+            "dtype": "float32",
+            "loss_implementation": "default",
+            "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"
+        },
+        "name": "bolmo_distill",
+        "dtype": "float32",
+        "init_method": "normal",
+        "init_seed": 0,
+        "init_std": 0.02,
+        "freeze_params": [
+            "boundary_predictor.*",
+            "teacher_embeddings.*"
+        ],
+        "local_encoder": {
+            "sliding_window_size": 0,
+            "d_model": 2048,
+            "n_layers": 1,
+            "block_config": {
+                "attention": {
+                    "name": "default",
+                    "n_heads": 16,
+                    "dtype": "float32",
+                    "_CLASS_": "olmo_core.nn.attention.AttentionConfig"
+                },
+                "layer_norm": {
+                    "name": "rms",
+                    "eps": 1e-06,
+                    "bias": false,
+                    "dtype": "float32",
+                    "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"
+                },
+                "feed_forward": {
+                    "hidden_size": 2816,
+                    "name": "default",
+                    "bias": false,
+                    "dtype": "float32",
+                    "act_name": "silu",
+                    "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"
+                },
+                "xlstm": {
+                    "num_heads": 16,
+                    "dtype": "float32",
+                    "_CLASS_": "olmo_core.nn.xlstm.XLSTMConfig"
+                },
+                "name": "xlstm",
+                "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"
+            },
+            "cross_attn_n_heads": 0,
+            "cross_attn_do_project": true,
+            "cross_attn_init_pooling": "amax",
+            "pooling": "hnet",
+            "add_hash_embeddings": false,
+            "add_expanded_embeddings": true,
+            "hash_byte_group_size": [
+                3,
+                4,
+                5,
+                6,
+                7,
+                8
+            ],
+            "hash_byte_group_vocab": [
+                1536,
+                3072,
+                6144,
+                12288,
+                24576,
+                49152
+            ],
+            "hash_byte_group_nb_functions": 1,
+            "add_norm_after_last_block": true,
+            "add_norm_after_pool": false,
+            "add_out_projection": true,
+            "boundary_predictor": "hnet",
+            "boundary_predictor_lookahead": 1,
+            "represent_bytes_with_embeddings": false,
+            "represent_bytes_with_last_mixed_out": false,
+            "blt_compat": false,
+            "dtype": "float32",
+            "_CLASS_": "olmo_core.nn.bolmo.config.LocalEncoderConfig"
+        },
+        "local_decoder": {
+            "sliding_window_size": 0,
+            "d_model": 2048,
+            "n_layers": 4,
+            "cross_attn_n_heads": 0,
+            "block_config": {
+                "attention": {
+                    "name": "default",
+                    "n_heads": 16,
+                    "dtype": "float32",
+                    "_CLASS_": "olmo_core.nn.attention.AttentionConfig"
+                },
+                "layer_norm": {
+                    "name": "rms",
+                    "eps": 1e-06,
+                    "bias": false,
+                    "dtype": "float32",
+                    "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"
+                },
+                "feed_forward": {
+                    "hidden_size": 2816,
+                    "name": "default",
+                    "bias": false,
+                    "dtype": "float32",
+                    "act_name": "silu",
+                    "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"
+                },
+                "xlstm": {
+                    "num_heads": 16,
+                    "dtype": "float32",
+                    "_CLASS_": "olmo_core.nn.xlstm.XLSTMConfig"
+                },
+                "name": "xlstm",
+                "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"
+            },
+            "depooling": "hnet",
+            "add_norm_before_first_block": true,
+            "add_norm_onto_residual": false,
+            "add_in_projection": true,
+            "add_projected_patch_residuals": false,
+            "hnet_smooth": false,
+            "hnet_smooth_ste": false,
+            "hnet_modulate": false,
+            "blt_compat": false,
+            "fuse_boundaries": true,
+            "no_boundaries": false,
+            "dtype": "float32",
+            "_CLASS_": "olmo_core.nn.bolmo.config.LocalDecoderConfig"
+        },
+        "share_blocks_between_teacher_and_student": false,
+        "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"
+    },
+    "dataset": {
+        "tokenizer": {
+            "vocab_size": 520,
+            "eos_token_id": 1,
+            "pad_token_id": 0,
+            "bos_token_id": 1,
+            "special_tokens": [
+                "<pad>",
+                "<bos>",
+                "<eos>",
+                "<bpe_token_end>"
+            ],
+            "special_tokens_first": true,
+            "original_identifier": "allenai/dolma2-tokenizer",
+            "bpe_token_end_id": 3,
+            "_CLASS_": "olmo_core.data.tokenizer.ByteTokenizerConfig"
+        },
+        "paths": [],
+        "expand_glob": false,
+        "include_instance_metadata": true,
+        "work_dir": "",
+        "ignore_fingerprint_mismatch": false,
+        "sequence_length": 4096,
+        "generate_doc_lengths": false,
+        "byte_sequence_length": 24576,
+        "_CLASS_": "olmo_core.data.numpy_dataset.NumpyByteFSLDatasetConfig"
+    },
+    "data_loader": {
+        "global_batch_size": 1572864,
+        "seed": 1234,
+        "num_workers": 24,
+        "ignore_fingerprint_mismatch": false,
+        "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig"
+    },
+    "train_module": {
+        "rank_microbatch_size": 98304,
+        "max_sequence_length": 24576,
+        "optim": {
+            "group_overrides": [
+                {
+                    "params": [
+                        "local_encoder.embedding.weight",
+                        "local_encoder.expanded_embeddings.weight"
+                    ],
+                    "opts": {
+                        "weight_decay": 0.0
+                    },
+                    "_CLASS_": "olmo_core.optim.config.OptimGroupOverride"
+                },
+                {
+                    "params": [
+                        "blocks.*"
+                    ],
+                    "opts": {
+                        "lr": 2.6e-05
+                    },
+                    "_CLASS_": "olmo_core.optim.config.OptimGroupOverride"
+                }
+            ],
+            "compile": false,
+            "fixed_fields": [
+                "initial_lr"
+            ],
+            "lr": 5.2e-05,
+            "betas": [
+                0.9,
+                0.95
+            ],
+            "eps": 1e-08,
+            "weight_decay": 0.1,
+            "_CLASS_": "olmo_core.optim.adamw.AdamWConfig"
+        },
+        "max_grad_norm": 0.5,
+        "scheduler": {
+            "lr_field": "lr",
+            "initial_lr_field": "initial_lr",
+            "units": "steps",
+            "alpha_f": 0.0,
+            "warmup_fraction": 0.1,
+            "warmup_min_lr": 0.0,
+            "_CLASS_": "olmo_core.optim.scheduler.LinearWithWarmup"
+        },
+        "compile_model": true,
+        "float8_config": {
+            "enabled": false,
+            "_CLASS_": "olmo_core.float8.Float8Config"
+        },
+        "dp_config": {
+            "name": "fsdp",
+            "param_dtype": "bfloat16",
+            "reduce_dtype": "float32",
+            "wrapping_strategy": "full",
+            "prefetch_factor": 0,
+            "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig"
+        },
+        "bolmo_config": {
+            "tokenizer": {
+                "vocab_size": 520,
+                "eos_token_id": 1,
+                "pad_token_id": 0,
+                "bos_token_id": 1,
+                "special_tokens": [
+                    "<pad>",
+                    "<bos>",
+                    "<eos>",
+                    "<bpe_token_end>"
+                ],
+                "special_tokens_first": true,
+                "original_identifier": "allenai/dolma2-tokenizer",
+                "bpe_token_end_id": 3,
+                "_CLASS_": "olmo_core.data.tokenizer.ByteTokenizerConfig"
+            },
+            "losses": [
+                "ce",
+                "boundary"
+            ],
+            "loss_weights": [
+                1.0,
+                4.0
+            ],
+            "binarization_temp": 1.0,
+            "temperature": 1.0,
+            "div_fn": "tvd_temp_limit",
+            "boundary_mode": "end",
+            "merge_boundary_loss": false,
+            "use_output_boundary_jsd": false,
+            "eval_add_boundary_logp": false,
+            "do_alm_debiasing": false,
+            "rep_compare_fn": "l2",
+            "start_ratio": 4.3,
+            "target_ratio": 4.3,
+            "gradual_boundary_compression_steps": 150000,
+            "encoder_loss_lookahead": 0,
+            "encoder_loss_no_lookahead_weight": 1.0,
+            "encoder_loss_lookahead_weights": [],
+            "patching": "dolma2",
+            "epsilon": 1e-06,
+            "skip_blocks": false,
+            "skip_teacher_blocks": false,
+            "skip_teacher": true,
+            "compute_teacher_ce": false,
+            "use_student_patch_reps_for_teacher": false,
+            "use_oracle_patch_reps": false,
+            "teacher_blocks_no_grad": true,
+            "student_blocks_no_grad": false,
+            "decoder_backprop_through_encoder": true,
+            "decoder_backprop_through_boundary_predictor": true,
+            "boundary_predictor_backprop_through_encoder": true,
+            "teacher_force_boundaries": false,
+            "boundary_threshold": "sample:0",
+            "xlstm_igate_bias_init": -10.0,
+            "skip_boundary_before_eos": true,
+            "_CLASS_": "olmo_core.nn.bolmo.config.BolmoConfig"
+        },
+        "label_ignore_index": -100,
+        "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerTrainModuleConfig"
+    },
+    "trainer": {},
+    "init_seed": 12536,
+    "_CLASS_": "__main__.ExperimentConfig"
+}

olmo_core/model_and_optim/.metadata

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea8ac9dd55708f4a36911565f33e7e223a9423ef21c79b868869c14bbd0b6cfa
+size 2464795

olmo_core/model_and_optim/__0_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9241563abb2e63f5e76e9047b32d1b46adf82bc18bba2fc33dcff6c1caf4e856
+size 138224729

olmo_core/model_and_optim/__0_1.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6ced12b46582166b9e74e636518a332aff6fa25ca3295baa81a9fb61d7844ac
+size 138224729

olmo_core/model_and_optim/__0_10.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7afb9d5ccf88ad2e9c0f3a4db73aab41f04065731bef1cb9e3e632a2d32a3bd
+size 138054699

olmo_core/model_and_optim/__0_11.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0285c42fd4de086b601dacbcc1b6042c30156a2ad0569c4c49e3864f99fdbb75
+size 138053122

olmo_core/model_and_optim/__0_12.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d727be4cca2724990cfb0f8e45ee9d71538a48b93a812f26246a3112391d55a0
+size 137924481

olmo_core/model_and_optim/__0_13.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c10f064d3c4a5a8f3b2020a8e630bc0270bb6dc50e7de8ea32a6c23e0f917ba5
+size 137924481

olmo_core/model_and_optim/__0_14.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f66920be4bb54652f8ebb8db3b01b448ad6ff467e2caf300dd74df04e780e50f
+size 137926058

olmo_core/model_and_optim/__0_15.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89eececcc6343abcd58709df669e959793180a439bedd5a9269028997c3b95f4
+size 137926058

olmo_core/model_and_optim/__0_2.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:452c356d9c7dfca4d617d3c41e2b758c516515e4319149770c9d4cee22751a98
+size 138224729

olmo_core/model_and_optim/__0_3.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93056e9e58af6bf3efce9901b41424a7bea1f3937c4fa69567d7ff904ec9cc3b
+size 138448453

olmo_core/model_and_optim/__0_4.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:165e64ab61aac760c9ac06bc259d989f7ba6caf9dc80a320040e2094ac4031c7
+size 138319274

olmo_core/model_and_optim/__0_5.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06bb7b56204a7491b3471bf4ce34eebe72a483fea29dd2ca6d950a1b69c5a4b2
+size 138319274

olmo_core/model_and_optim/__0_6.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8febe1110dd2aa1c63e8f4112934e0432e7cb120ad379705b6bd6f69c900fff4
+size 138319274

olmo_core/model_and_optim/__0_7.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29f5377b1855852505822287811f35dca870bd9a89711c6dd91e053917cfc798
+size 137902754

olmo_core/model_and_optim/__0_8.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6943898c5bb21c2b2249dad0168defbfbdcb63a7ad0973d751b468ec07d3f40
+size 137902754

olmo_core/model_and_optim/__0_9.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d46b7bc5207027ea7fd5d98242a338a2bc0c70dd3350d2e39851ca7e6688dfc1
+size 138054699

olmo_core/model_and_optim/__1_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0e30b56e0f4cdea3fcad9e636a0131caf679e2a5d66ad69fca0d60608001ce3
+size 137975051

olmo_core/model_and_optim/__1_1.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f19ba4c47b4093ee13f16632f9be4f04af4f068b7462db621139bbf1c2ba2678
+size 137975051

olmo_core/model_and_optim/__1_10.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd8a8524fca46aea6135faec75282ecae3ecc451b0f693614192f0b8a19a59c1
+size 137667506

olmo_core/model_and_optim/__1_11.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf27a46ff7598b81b3bd5f8663147542d7a16d6d31a44ed20968d06ff2b334c2
+size 137667506

olmo_core/model_and_optim/__1_12.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7de5fbb6ddd3855373b1311eb09827f7bce000b9cec11255f10ee17fcbf05eb
+size 137676064

olmo_core/model_and_optim/__1_13.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6ec75d600a68102f796d9d8fce2fdb0b9898dff43595cad570b60efcdb269b0
+size 137676064

olmo_core/model_and_optim/__1_14.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b66435c1ed78293f1af1f31bedbd97033cc797597391ab045bc584d58797f58
+size 137677641

olmo_core/model_and_optim/__1_15.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff36c0a40ac9b5137b10958759d3c3eb7fe1f64d0d9699ef27f44d1a7104ee43
+size 137677641

olmo_core/model_and_optim/__1_2.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efb835fa07e895daeef6b90d4528fe24de85f1d3950db6b123342594bcd41387
+size 137975051

olmo_core/model_and_optim/__1_3.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67679cdff1de9eb8b5574a2941ada5f5d98e7e3f7d8d8fac8cf6e934879db0aa
+size 138198775

olmo_core/model_and_optim/__1_4.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27723cefd17c891e7d891610acffeaf7ac2e8d31f109118b92bfee9c55016ba0
+size 138070857

olmo_core/model_and_optim/__1_5.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad447e78e2efadf7eab55bac518eb88714645b477a93f222048b5a856f7b15c0
+size 138070857

olmo_core/model_and_optim/__1_6.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8efabe23bc8209221a31ec2c2c0dd0ac4a24444cfb53eea1423d402406891cb7
+size 138070857

olmo_core/model_and_optim/__1_7.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:340d3f0e71d0e35e23a7a34c035339edfecb1268b9fdbcb6370e18be2754adc0
+size 137654337

olmo_core/model_and_optim/__1_8.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7b4d3f88074fd110a32ca232d5264a3848f8caa11f5334ff71c74df8e999d00
+size 137654337

olmo_core/model_and_optim/__1_9.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:118cdbe40e52906b81456b66df6d4a5743a9183f6dffc9afbba01a596a547faa
+size 137667506

olmo_core/model_and_optim/__2_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f6a41c5048cde6511c0b50bba5684c39f6922fafb5b11931e01a2a4160d8604
+size 137975051

olmo_core/model_and_optim/__2_1.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:328dd39d60e3663225c6c6fdb38e9aa600426dca217c0d962208e7af954d49f1
+size 137975051

olmo_core/model_and_optim/__2_10.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0578b542929035283b915bb5ec4883c42b9dd801f7988b3bef83033816d9ed35
+size 137667506

olmo_core/model_and_optim/__2_11.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da49896618a8f8303b681ec064c6355088b58fafd33b4c62b6cc740f1038851f
+size 137667506

olmo_core/model_and_optim/__2_12.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c79ff34c25d25fe399d9a289fcf6d1031142a88d62c50dc50b150aa02aa4956
+size 137676064

olmo_core/model_and_optim/__2_13.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb401a50823a9ed8038e1968d9a3e222d05509c33079c2b8b16e282cd67f5b41
+size 137676064

olmo_core/model_and_optim/__2_14.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb63c9c083f627e150c041a2d3de9282d6f643dae1e95e1a46691684fe406459
+size 137677641

olmo_core/model_and_optim/__2_15.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1205969c3caa7aaf7073f5afb49061fc7d646f86704cd4fab2caf44d28c00a6
+size 137677641

olmo_core/model_and_optim/__2_2.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51c4181a8c59c612f46ff9ce6520da3c74f841bc20a137ddac9b2d2e5f3f7690
+size 137975051

olmo_core/model_and_optim/__2_3.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94dd1a66905a5908f094413497af69d448d1d38159a2f97dfe3e78143f892c14
+size 138198775

olmo_core/model_and_optim/__2_4.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daf72d44ed90913fce12cfbe2626044c68edf87b14f5ebb3a48eb77591a80447
+size 138070857

olmo_core/model_and_optim/__2_5.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6fcb095aa0987410995ae325c87eb5e01bcdf0dfa35eae629e1ef2a11a76b3f
+size 138070857

olmo_core/model_and_optim/__2_6.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c8c5f22aabf49f0afc77bd3871ad5939f429d088814ede8bd602ae7f9e52055
+size 138070857

olmo_core/model_and_optim/__2_7.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc9faf4b47e1ea02e5a1b0fa8e8cddea7ac92bda6d230945722d594ece8b82bc
+size 137654337