Upload folder using huggingface_hub

DNAbert2_Pretrained/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

DNAbert2_Pretrained/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "/storage2/fs1/btc/Active/yeli/xiaoxiao.zhou/tokenize/tokenizers/DNAbert2_theirBPE/pretrain/models/model_2/checkpoint-150000",
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.29.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 4096
+}

DNAbert2_Pretrained/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "max_length": 512,
+  "pad_token_id": 0,
+  "transformers_version": "4.29.2"
+}

DNAbert2_Pretrained/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f955d3c0433c470a0bc280e8b61264489e5209007c9d6dd388b1ebe23f70f4b
+size 356844726

DNAbert2_Pretrained/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6d6c569bea0b6e042e097ec2a4b6cfec780f8e0a2a2181b3d4babd06fb07ebf
+size 14244

DNAbert2_Pretrained/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d7ab81f6f167a77d305fd2072c3f0653a829da163e38e23511eabde99bbdc11
+size 1064

DNAbert2_Pretrained/trainer_state.json

@@ -0,0 +1,316 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.824392126592049,
+  "global_step": 50000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.1,
+      "learning_rate": 7.142857142857143e-06,
+      "loss": 5.3189,
+      "step": 1000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": 5.3266,
+      "step": 2000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 2.1428571428571428e-05,
+      "loss": 5.3215,
+      "step": 3000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": 5.3156,
+      "step": 4000
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.571428571428572e-05,
+      "loss": 5.3094,
+      "step": 5000
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2857142857142856e-05,
+      "loss": 5.294,
+      "step": 6000
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 5e-05,
+      "loss": 5.2852,
+      "step": 7000
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 4.883720930232558e-05,
+      "loss": 5.2715,
+      "step": 8000
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 4.7674418604651164e-05,
+      "loss": 5.2477,
+      "step": 9000
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.651162790697675e-05,
+      "loss": 5.2361,
+      "step": 10000
+    },
+    {
+      "epoch": 1.06,
+      "learning_rate": 4.5348837209302326e-05,
+      "loss": 5.2234,
+      "step": 11000
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 4.418604651162791e-05,
+      "loss": 5.2037,
+      "step": 12000
+    },
+    {
+      "epoch": 1.25,
+      "learning_rate": 4.302325581395349e-05,
+      "loss": 5.1853,
+      "step": 13000
+    },
+    {
+      "epoch": 1.35,
+      "learning_rate": 4.186046511627907e-05,
+      "loss": 5.1751,
+      "step": 14000
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 4.0697674418604655e-05,
+      "loss": 5.1663,
+      "step": 15000
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 3.953488372093023e-05,
+      "loss": 5.1615,
+      "step": 16000
+    },
+    {
+      "epoch": 1.64,
+      "learning_rate": 3.837209302325582e-05,
+      "loss": 5.1547,
+      "step": 17000
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 3.7209302325581394e-05,
+      "loss": 5.1431,
+      "step": 18000
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 3.604651162790698e-05,
+      "loss": 5.1293,
+      "step": 19000
+    },
+    {
+      "epoch": 1.93,
+      "learning_rate": 3.488372093023256e-05,
+      "loss": 5.1278,
+      "step": 20000
+    },
+    {
+      "epoch": 2.03,
+      "learning_rate": 3.372093023255814e-05,
+      "loss": 5.1174,
+      "step": 21000
+    },
+    {
+      "epoch": 2.12,
+      "learning_rate": 3.2558139534883724e-05,
+      "loss": 5.1033,
+      "step": 22000
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 3.13953488372093e-05,
+      "loss": 5.1019,
+      "step": 23000
+    },
+    {
+      "epoch": 2.32,
+      "learning_rate": 3.0232558139534883e-05,
+      "loss": 5.096,
+      "step": 24000
+    },
+    {
+      "epoch": 2.41,
+      "learning_rate": 2.9069767441860467e-05,
+      "loss": 5.0878,
+      "step": 25000
+    },
+    {
+      "epoch": 2.51,
+      "learning_rate": 2.7906976744186048e-05,
+      "loss": 5.0847,
+      "step": 26000
+    },
+    {
+      "epoch": 2.61,
+      "learning_rate": 2.674418604651163e-05,
+      "loss": 5.0731,
+      "step": 27000
+    },
+    {
+      "epoch": 2.7,
+      "learning_rate": 2.5581395348837212e-05,
+      "loss": 5.0672,
+      "step": 28000
+    },
+    {
+      "epoch": 2.8,
+      "learning_rate": 2.441860465116279e-05,
+      "loss": 5.0717,
+      "step": 29000
+    },
+    {
+      "epoch": 2.89,
+      "learning_rate": 2.3255813953488374e-05,
+      "loss": 5.0631,
+      "step": 30000
+    },
+    {
+      "epoch": 2.99,
+      "learning_rate": 2.2093023255813955e-05,
+      "loss": 5.0626,
+      "step": 31000
+    },
+    {
+      "epoch": 3.09,
+      "learning_rate": 2.0930232558139536e-05,
+      "loss": 5.06,
+      "step": 32000
+    },
+    {
+      "epoch": 3.18,
+      "learning_rate": 1.9767441860465116e-05,
+      "loss": 5.0461,
+      "step": 33000
+    },
+    {
+      "epoch": 3.28,
+      "learning_rate": 1.8604651162790697e-05,
+      "loss": 5.039,
+      "step": 34000
+    },
+    {
+      "epoch": 3.38,
+      "learning_rate": 1.744186046511628e-05,
+      "loss": 5.0416,
+      "step": 35000
+    },
+    {
+      "epoch": 3.47,
+      "learning_rate": 1.6279069767441862e-05,
+      "loss": 5.0437,
+      "step": 36000
+    },
+    {
+      "epoch": 3.57,
+      "learning_rate": 1.5116279069767441e-05,
+      "loss": 5.03,
+      "step": 37000
+    },
+    {
+      "epoch": 3.67,
+      "learning_rate": 1.3953488372093024e-05,
+      "loss": 5.027,
+      "step": 38000
+    },
+    {
+      "epoch": 3.76,
+      "learning_rate": 1.2790697674418606e-05,
+      "loss": 5.0289,
+      "step": 39000
+    },
+    {
+      "epoch": 3.86,
+      "learning_rate": 1.1627906976744187e-05,
+      "loss": 5.0246,
+      "step": 40000
+    },
+    {
+      "epoch": 3.96,
+      "learning_rate": 1.0465116279069768e-05,
+      "loss": 5.0209,
+      "step": 41000
+    },
+    {
+      "epoch": 4.05,
+      "learning_rate": 9.302325581395349e-06,
+      "loss": 5.0167,
+      "step": 42000
+    },
+    {
+      "epoch": 4.15,
+      "learning_rate": 8.139534883720931e-06,
+      "loss": 5.0108,
+      "step": 43000
+    },
+    {
+      "epoch": 4.25,
+      "learning_rate": 6.976744186046512e-06,
+      "loss": 5.0121,
+      "step": 44000
+    },
+    {
+      "epoch": 4.34,
+      "learning_rate": 5.8139534883720935e-06,
+      "loss": 5.0122,
+      "step": 45000
+    },
+    {
+      "epoch": 4.44,
+      "learning_rate": 4.651162790697674e-06,
+      "loss": 5.0222,
+      "step": 46000
+    },
+    {
+      "epoch": 4.53,
+      "learning_rate": 3.488372093023256e-06,
+      "loss": 5.0162,
+      "step": 47000
+    },
+    {
+      "epoch": 4.63,
+      "learning_rate": 2.325581395348837e-06,
+      "loss": 5.0045,
+      "step": 48000
+    },
+    {
+      "epoch": 4.73,
+      "learning_rate": 1.1627906976744186e-06,
+      "loss": 5.0047,
+      "step": 49000
+    },
+    {
+      "epoch": 4.82,
+      "learning_rate": 0.0,
+      "loss": 5.005,
+      "step": 50000
+    }
+  ],
+  "max_steps": 50000,
+  "num_train_epochs": 5,
+  "total_flos": 1.2629229170809897e+18,
+  "trial_name": null,
+  "trial_params": null
+}

DNAbert2_Pretrained/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7a0b2d34455400cc6664215841506159df7ebf59680d22509badf6d4fae42c2
+size 4600

Grover_Pretrained/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Grover_Pretrained/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.29.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 610
+}

Grover_Pretrained/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "max_length": 512,
+  "pad_token_id": 0,
+  "transformers_version": "4.29.2"
+}

Grover_Pretrained/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b94eae11ebac271508993f2f27a4ad5e1f5028638f4e573fe3c5054987b4781
+size 692215499

Grover_Pretrained/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a448ff3dd52c8bfb983420e92e43e77892ace1424526d9c10b8580966404381a
+size 346122191

Grover_Pretrained/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:136a4f26618b8e44d873ce83286097bab76c8ac85c9f04861291f10d72aa2ac7
+size 14645

Grover_Pretrained/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b751cb1c63fcc5090801e168722b88043b19f294b16d123cbf19c81339681021
+size 1465

Grover_Pretrained/tokenizer.json

@@ -0,0 +1,1359 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "[CLS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "[CLS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "[CLS]": {
+        "id": "[CLS]",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "[CLS]"
+        ]
+      },
+      "[SEP]": {
+        "id": "[SEP]",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "[SEP]"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "vocab": {
+      "[PAD]": 0,
+      "[UNK]": 1,
+      "[CLS]": 2,
+      "[SEP]": 3,
+      "[MASK]": 4,
+      "A": 5,
+      "C": 6,
+      "G": 7,
+      "T": 8,
+      "TT": 9,
+      "AA": 10,
+      "TG": 11,
+      "AG": 12,
+      "CC": 13,
+      "TC": 14,
+      "AC": 15,
+      "GG": 16,
+      "ATT": 17,
+      "AT": 18,
+      "ATG": 19,
+      "GC": 20,
+      "TAA": 21,
+      "TCC": 22,
+      "ACC": 23,
+      "AAAA": 24,
+      "AGG": 25,
+      "ATC": 26,
+      "AGC": 27,
+      "TTC": 28,
+      "AAG": 29,
+      "TTTT": 30,
+      "TGC": 31,
+      "TGG": 32,
+      "AAC": 33,
+      "TTG": 34,
+      "TAG": 35,
+      "TAC": 36,
+      "CCC": 37,
+      "TATT": 38,
+      "TGGG": 39,
+      "TAT": 40,
+      "AGAA": 41,
+      "AGGG": 42,
+      "TTTC": 43,
+      "AGGC": 44,
+      "AGCC": 45,
+      "ATAA": 46,
+      "TGTG": 47,
+      "TTGG": 48,
+      "ATTC": 49,
+      "AAGG": 50,
+      "ACAC": 51,
+      "TCCC": 52,
+      "TCTC": 53,
+      "TATG": 54,
+      "TTTG": 55,
+      "TTCC": 56,
+      "AGTG": 57,
+      "ATGG": 58,
+      "AGAC": 59,
+      "AAAC": 60,
+      "ACCC": 61,
+      "TGCC": 62,
+      "ATTG": 63,
+      "ATCC": 64,
+      "AGAG": 65,
+      "ATGC": 66,
+      "ATAC": 67,
+      "TCTG": 68,
+      "TTAA": 69,
+      "TCAC": 70,
+      "TGAA": 71,
+      "TGGC": 72,
+      "TTGC": 73,
+      "TAAG": 74,
+      "TATC": 75,
+      "TAAC": 76,
+      "AAAG": 77,
+      "TTAC": 78,
+      "AAGC": 79,
+      "GGG": 80,
+      "TAGC": 81,
+      "GGC": 82,
+      "ATAT": 83,
+      "TACC": 84,
+      "AACC": 85,
+      "AATG": 86,
+      "TAGG": 87,
+      "GCC": 88,
+      "ATATT": 89,
+      "AGTC": 90,
+      "TTTTC": 91,
+      "AAAAC": 92,
+      "TGAC": 93,
+      "TTTAA": 94,
+      "AAAAG": 95,
+      "AATC": 96,
+      "TGTC": 97,
+      "TTATT": 98,
+      "ATAG": 99,
+      "TGAG": 100,
+      "TTTTG": 101,
+      "AAATT": 102,
+      "AATT": 103,
+      "AATAA": 104,
+      "TTTCC": 105,
+      "ACAG": 106,
+      "TCAG": 107,
+      "AAATG": 108,
+      "TGGGC": 109,
+      "ACTC": 110,
+      "AGGCC": 111,
+      "TTAG": 112,
+      "ACTG": 113,
+      "ACG": 114,
+      "ATATG": 115,
+      "TGGCC": 116,
+      "ATTTC": 117,
+      "ACAA": 118,
+      "ATCTC": 119,
+      "TATTC": 120,
+      "TGTAA": 121,
+      "ACTT": 122,
+      "ATGCC": 123,
+      "TAAAA": 124,
+      "AAAAAAAA": 125,
+      "ATTCC": 126,
+      "TTTAG": 127,
+      "TCCCC": 128,
+      "TTTGC": 129,
+      "TTCCC": 130,
+      "TGGGG": 131,
+      "TTCTC": 132,
+      "ATAAAA": 133,
+      "AGAAG": 134,
+      "TTTTTTTT": 135,
+      "ACCCC": 136,
+      "AGGGC": 137,
+      "ACCTC": 138,
+      "AGATG": 139,
+      "ATTAC": 140,
+      "AAGCC": 141,
+      "GGCC": 142,
+      "AGGAG": 143,
+      "TCAA": 144,
+      "ATTGC": 145,
+      "TATTG": 146,
+      "ATAAC": 147,
+      "ATATC": 148,
+      "TTTAC": 149,
+      "ATGGC": 150,
+      "AAGGC": 151,
+      "ACCAC": 152,
+      "GTG": 153,
+      "ATCCC": 154,
+      "AGAAC": 155,
+      "ATTTT": 156,
+      "TTGCC": 157,
+      "AAATC": 158,
+      "ATAAG": 159,
+      "TTGGC": 160,
+      "TGGAG": 161,
+      "ATGGG": 162,
+      "AAAGC": 163,
+      "AGGGG": 164,
+      "ATCAC": 165,
+      "ATTTG": 166,
+      "AATTC": 167,
+      "TGCAC": 168,
+      "TTTGG": 169,
+      "TCG": 170,
+      "AGAGC": 171,
+      "AAAGG": 172,
+      "GGGC": 173,
+      "TTGGG": 174,
+      "AGAAAA": 175,
+      "TATCC": 176,
+      "TCTCC": 177,
+      "ATAGC": 178,
+      "TGAGG": 179,
+      "TTTATT": 180,
+      "AGTAA": 181,
+      "AGAGG": 182,
+      "TCTTC": 183,
+      "ACATT": 184,
+      "TCCTG": 185,
+      "AGCCC": 186,
+      "TATGC": 187,
+      "TTAAAA": 188,
+      "AGATT": 189,
+      "TTAAC": 190,
+      "GGGG": 191,
+      "AAGAC": 192,
+      "TCATT": 193,
+      "TTCTG": 194,
+      "AGACC": 195,
+      "AAGGG": 196,
+      "ATACC": 197,
+      "TTTAT": 198,
+      "AAGTG": 199,
+      "TTATG": 200,
+      "AAGAA": 201,
+      "TAGCC": 202,
+      "TTCAC": 203,
+      "AGGTG": 204,
+      "TTGAA": 205,
+      "ATCTG": 206,
+      "AGCAC": 207,
+      "TGCTG": 208,
+      "AAACC": 209,
+      "ATGTG": 210,
+      "TTTTCC": 211,
+      "AGTTC": 212,
+      "TCCTC": 213,
+      "TATGG": 214,
+      "AATAC": 215,
+      "AGTGG": 216,
+      "TAGGC": 217,
+      "AGCTC": 218,
+      "ATAGG": 219,
+      "TTATC": 220,
+      "TTAAG": 221,
+      "TACCC": 222,
+      "TTTTTG": 223,
+      "AACAC": 224,
+      "TGCTC": 225,
+      "AGATC": 226,
+      "TCCCAGC": 227,
+      "AGCTG": 228,
+      "AATAG": 229,
+      "TCTTG": 230,
+      "AGTGGC": 231,
+      "ATTGG": 232,
+      "TACTC": 233,
+      "TAAAC": 234,
+      "AATGG": 235,
+      "AGGTC": 236,
+      "AGGAC": 237,
+      "TTGTG": 238,
+      "TATAC": 239,
+      "ATTTTC": 240,
+      "ATATAA": 241,
+      "AGGCTG": 242,
+      "ATTTAA": 243,
+      "AGTT": 244,
+      "AGTAG": 245,
+      "ATGAC": 246,
+      "AATGC": 247,
+      "TCCAC": 248,
+      "CCCC": 249,
+      "ATGTC": 250,
+      "AACTC": 251,
+      "TTTTTC": 252,
+      "TAAGC": 253,
+      "AAGTC": 254,
+      "TGGTG": 255,
+      "TATAA": 256,
+      "AGTGC": 257,
+      "TAAGG": 258,
+      "ACCTG": 259,
+      "TTAGC": 260,
+      "AAATAA": 261,
+      "TGCCTC": 262,
+      "AATCC": 263,
+      "TTGGCC": 264,
+      "TAGGG": 265,
+      "TGGAC": 266,
+      "TTGTC": 267,
+      "AACCC": 268,
+      "TTACC": 269,
+      "TAACC": 270,
+      "AATTTT": 271,
+      "AAAGAA": 272,
+      "ATTATT": 273,
+      "AGCG": 274,
+      "AAAAAC": 275,
+      "TAATG": 276,
+      "TTGAC": 277,
+      "AGTCC": 278,
+      "AACTG": 279,
+      "AGTTG": 280,
+      "AATTG": 281,
+      "TCTGC": 282,
+      "TTAGG": 283,
+      "TACAC": 284,
+      "AGAAGG": 285,
+      "ATATTC": 286,
+      "AAAACC": 287,
+      "AAAAGC": 288,
+      "TGCCC": 289,
+      "ACTGC": 290,
+      "AGAAGC": 291,
+      "TAATAA": 292,
+      "AATATT": 293,
+      "ACCATG": 294,
+      "TGGTC": 295,
+      "TTTTGC": 296,
+      "AACG": 297,
+      "TACTG": 298,
+      "ACACACAC": 299,
+      "ATTTTG": 300,
+      "TCCG": 301,
+      "TGCG": 302,
+      "AAAATG": 303,
+      "ACATG": 304,
+      "TCAGC": 305,
+      "ATCG": 306,
+      "AGTAC": 307,
+      "TTTTGG": 308,
+      "AATAT": 309,
+      "AGAGAA": 310,
+      "TTCG": 311,
+      "TCCAGCC": 312,
+      "ATATAC": 313,
+      "TCACC": 314,
+      "AAAAGG": 315,
+      "TGTGTGTG": 316,
+      "TCATC": 317,
+      "TGCTGGG": 318,
+      "TGAAG": 319,
+      "TGTAG": 320,
+      "TGTGG": 321,
+      "AAAAATT": 322,
+      "ACTTC": 323,
+      "TTCCCC": 324,
+      "ATAGAA": 325,
+      "TTGCCC": 326,
+      "AGGAGG": 327,
+      "TTTCCC": 328,
+      "TATATT": 329,
+      "ACCG": 330,
+      "ACTAC": 331,
+      "TCACTGC": 332,
+      "GCG": 333,
+      "TTTGTG": 334,
+      "ACAGC": 335,
+      "TCATG": 336,
+      "AGTTTT": 337,
+      "AGGAA": 338,
+      "TTTATG": 339,
+      "ATATTG": 340,
+      "TGATG": 341,
+      "TCTAA": 342,
+      "TGTGC": 343,
+      "AGGAAG": 344,
+      "TTTGGG": 345,
+      "TGTTC": 346,
+      "AGCCCC": 347,
+      "AGTTTC": 348,
+      "AGGCTGG": 349,
+      "TTTGCC": 350,
+      "ATTTCC": 351,
+      "ATACAC": 352,
+      "AAAATAA": 353,
+      "TAGAC": 354,
+      "AGGAGAA": 355,
+      "TGAGC": 356,
+      "TGGAA": 357,
+      "TTTTTAA": 358,
+      "AGCCTCCC": 359,
+      "ATGAA": 360,
+      "TTTAAG": 361,
+      "TCTGG": 362,
+      "TTTATC": 363,
+      "TTATAA": 364,
+      "TGATT": 365,
+      "AACAA": 366,
+      "TAGCTGGG": 367,
+      "TCAAG": 368,
+      "AAAAAA": 369,
+      "ACTTTGGG": 370,
+      "TATTCC": 371,
+      "TCAGG": 372,
+      "AACAG": 373,
+      "TTCTTC": 374,
+      "TGTGGC": 375,
+      "ATATGC": 376,
+      "ATTACAGGC": 377,
+      "AGGGGC": 378,
+      "AGGGCC": 379,
+      "TTATTC": 380,
+      "ATATCC": 381,
+      "TGTAATCCCAGC": 382,
+      "TACG": 383,
+      "AGAAAC": 384,
+      "TGTCC": 385,
+      "AGATGG": 386,
+      "TGTGCC": 387,
+      "TTTCTC": 388,
+      "TGAAC": 389,
+      "AGTCTC": 390,
+      "TGTTG": 391,
+      "ATTTTTT": 392,
+      "AAGAAG": 393,
+      "TGGGGC": 394,
+      "AGCAGC": 395,
+      "GCCC": 396,
+      "TTTGGC": 397,
+      "AGGCTGAGGC": 398,
+      "TGGGCC": 399,
+      "TTCTCC": 400,
+      "TAGAA": 401,
+      "TGGAGTGC": 402,
+      "ATTAA": 403,
+      "AGTGCC": 404,
+      "TGTCTC": 405,
+      "ATATGG": 406,
+      "ACATC": 407,
+      "TGGGGG": 408,
+      "TGACC": 409,
+      "ACTCC": 410,
+      "TAAAAC": 411,
+      "AGATAA": 412,
+      "TAATTTT": 413,
+      "TCAAC": 414,
+      "TCTAC": 415,
+      "TCTAG": 416,
+      "GAG": 417,
+      "TAAATG": 418,
+      "AGCAA": 419,
+      "TATATG": 420,
+      "ATATATAT": 421,
+      "ATTTGC": 422,
+      "TCCTCC": 423,
+      "CCCAC": 424,
+      "ATTTATT": 425,
+      "TCTGCC": 426,
+      "ATGGCC": 427,
+      "TCGC": 428,
+      "AGTATT": 429,
+      "AGAACC": 430,
+      "TTAAAC": 431,
+      "AAATTC": 432,
+      "AGAGAC": 433,
+      "ATTTAC": 434,
+      "ATTGCC": 435,
+      "AACAAC": 436,
+      "TTTAAC": 437,
+      "ACGG": 438,
+      "AAGAAAA": 439,
+      "TCTGGC": 440,
+      "ATTCTCC": 441,
+      "AGGTGG": 442,
+      "TGCTGC": 443,
+      "TTCAAG": 444,
+      "AGAGGG": 445,
+      "ACACC": 446,
+      "TCTTTT": 447,
+      "AGAGGC": 448,
+      "ATCACC": 449,
+      "TAAATT": 450,
+      "AAGGCC": 451,
+      "TTGCAGTG": 452,
+      "TGTAC": 453,
+      "AATTTC": 454,
+      "ATCCCC": 455,
+      "ACAAG": 456,
+      "ACAGG": 457,
+      "ACAAC": 458,
+      "TGCCCC": 459,
+      "AGATTC": 460,
+      "TTAGAA": 461,
+      "TTGGGG": 462,
+      "AGACAC": 463,
+      "TGGAAG": 464,
+      "ACCTCC": 465,
+      "ATGGGG": 466,
+      "AGCCTCC": 467,
+      "TTATTG": 468,
+      "TAAAAG": 469,
+      "ATCTTC": 470,
+      "ATCTCC": 471,
+      "TGAAGC": 472,
+      "TAATC": 473,
+      "AAATGC": 474,
+      "TTGTTG": 475,
+      "ATTCCC": 476,
+      "TACTAAAA": 477,
+      "ATAGTG": 478,
+      "AAATAC": 479,
+      "TTGGGC": 480,
+      "TAGAGAC": 481,
+      "TGTTTT": 482,
+      "TTCTGC": 483,
+      "TGGCCC": 484,
+      "TCTGTC": 485,
+      "AGCTCC": 486,
+      "AACTCC": 487,
+      "TTAGCC": 488,
+      "AAAGTGCTGGG": 489,
+      "ATAGAC": 490,
+      "TATTTTTAG": 491,
+      "ACTTG": 492,
+      "ACCACC": 493,
+      "AAACAC": 494,
+      "GTGG": 495,
+      "ATTTAG": 496,
+      "AGGAGC": 497,
+      "AGGCTGGAGTGC": 498,
+      "ATACCC": 499,
+      "ATGTAA": 500,
+      "ACGC": 501,
+      "AGTAT": 502,
+      "TTTACC": 503,
+      "ACTAA": 504,
+      "AGGCCC": 505,
+      "AAGGGG": 506,
+      "TCTCG": 507,
+      "ATGAAG": 508,
+      "AAAGAC": 509,
+      "TGAAAA": 510,
+      "AAGGGC": 511,
+      "ATAGGC": 512,
+      "AGAGTG": 513,
+      "AGCTGC": 514,
+      "ATGTTC": 515,
+      "TATTTC": 516,
+      "TGATC": 517,
+      "AGTTTG": 518,
+      "AGCTAA": 519,
+      "AGAGCC": 520,
+      "TGCTTC": 521,
+      "ATCATC": 522,
+      "AACATGG": 523,
+      "AGCTTC": 524,
+      "AAGAAC": 525,
+      "TTTTTTG": 526,
+      "AGGGGG": 527,
+      "ATAAGC": 528,
+      "TAAGCC": 529,
+      "ACTGG": 530,
+      "ACAAAA": 531,
+      "ATCATT": 532,
+      "TCTTTC": 533,
+      "ATGATG": 534,
+      "TGCAA": 535,
+      "AGGTTC": 536,
+      "AACATT": 537,
+      "ATGGGC": 538,
+      "ATAGAG": 539,
+      "AAATGG": 540,
+      "AGTTCC": 541,
+      "TTTAGC": 542,
+      "AACTTC": 543,
+      "AGCAAG": 544,
+      "ATAAAAC": 545,
+      "AAAATC": 546,
+      "AGCCAC": 547,
+      "AGGAAC": 548,
+      "TTAACC": 549,
+      "TATTTATT": 550,
+      "TTTCTG": 551,
+      "ATAAGG": 552,
+      "AGCCACC": 553,
+      "AGATGC": 554,
+      "TTAAGC": 555,
+      "TTGTAA": 556,
+      "AGTGTG": 557,
+      "AACCCC": 558,
+      "TTCATT": 559,
+      "ATCATG": 560,
+      "AATGAA": 561,
+      "AGGTGC": 562,
+      "AAAAAAAAAAAAAAAA": 563,
+      "AGGATG": 564,
+      "AGCCG": 565,
+      "TGGTGG": 566,
+      "AGTGGG": 567,
+      "TGCACTCCAGCC": 568,
+      "TATTGC": 569,
+      "TAGTC": 570,
+      "CCCG": 571,
+      "AAGTAA": 572,
+      "TAGTG": 573,
+      "TTTTTTTTTTTTTTTT": 574,
+      "AGCATT": 575,
+      "ATCTGC": 576,
+      "TCTCAC": 577,
+      "AAATTG": 578,
+      "TTTAGG": 579,
+      "AGACCC": 580,
+      "GGGCC": 581,
+      "TCCTTC": 582,
+      "ATAGGG": 583,
+      "AATATG": 584,
+      "TTATAC": 585,
+      "TAGAAG": 586,
+      "AAAGTG": 587,
+      "AAATCC": 588,
+      "TTCCTC": 589,
+      "TTTCAC": 590,
+      "AGTATG": 591,
+      "TACTAAAAATAC": 592,
+      "ATGTGC": 593,
+      "AGGAGGC": 594,
+      "TATATC": 595,
+      "TTCTAA": 596,
+      "TGAGGC": 597,
+      "ACACAC": 598,
+      "TCCCCC": 599,
+      "AACATC": 600,
+      "AAGCG": 601,
+      "AATGGC": 602,
+      "ACCCCC": 603,
+      "AGATAC": 604,
+      "ATAAAAG": 605,
+      "ATGATT": 606,
+      "TGGAGG": 607,
+      "AGTTAA": 608,
+      "": 609
+    },
+    "merges": [
+      "A ",
+      "C ",
+      "G ",
+      "T ",
+      "T T",
+      "A A",
+      "T G",
+      "A G",
+      "C C",
+      "T C",
+      "A C",
+      "G G",
+      "A TT",
+      "A T",
+      "A TG",
+      "G C",
+      "T AA",
+      "T CC",
+      "A CC",
+      "AA AA",
+      "AG G",
+      "A TC",
+      "AG C",
+      "TT C",
+      "AA G",
+      "TT TT",
+      "TG C",
+      "TG G",
+      "AA C",
+      "TT G",
+      "T AG",
+      "T AC",
+      "CC C",
+      "T ATT",
+      "TG GG",
+      "T AT",
+      "AG AA",
+      "AG GG",
+      "TT TC",
+      "AG GC",
+      "AG CC",
+      "AT AA",
+      "TG TG",
+      "TT GG",
+      "ATT C",
+      "AA GG",
+      "AC AC",
+      "TCC C",
+      "TC TC",
+      "T ATG",
+      "TT TG",
+      "TT CC",
+      "AG TG",
+      "ATG G",
+      "AG AC",
+      "AA AC",
+      "ACC C",
+      "TG CC",
+      "ATT G",
+      "AT CC",
+      "AG AG",
+      "ATG C",
+      "AT AC",
+      "TC TG",
+      "TT AA",
+      "TC AC",
+      "TG AA",
+      "TG GC",
+      "TT GC",
+      "TAA G",
+      "T ATC",
+      "TAA C",
+      "AA AG",
+      "TT AC",
+      "AA GC",
+      "GG G",
+      "T AGC",
+      "GG C",
+      "AT AT",
+      "T ACC",
+      "AA CC",
+      "AA TG",
+      "T AGG",
+      "G CC",
+      "AT ATT",
+      "AG TC",
+      "TT TTC",
+      "AAAA C",
+      "TG AC",
+      "TT TAA",
+      "AAAA G",
+      "AA TC",
+      "TG TC",
+      "TT ATT",
+      "AT AG",
+      "TG AG",
+      "TTTT G",
+      "AA ATT",
+      "AA TT",
+      "AA TAA",
+      "TT TCC",
+      "AC AG",
+      "TC AG",
+      "AA ATG",
+      "TGGG C",
+      "AC TC",
+      "AGG CC",
+      "TT AG",
+      "AC TG",
+      "AC G",
+      "AT ATG",
+      "TGG CC",
+      "ATT TC",
+      "AC AA",
+      "ATC TC",
+      "TATT C",
+      "TG TAA",
+      "AC TT",
+      "ATG CC",
+      "TAA AA",
+      "AAAA AAAA",
+      "ATT CC",
+      "TT TAG",
+      "TCC CC",
+      "TT TGC",
+      "TT CCC",
+      "TGGG G",
+      "TTC TC",
+      "AT AAAA",
+      "AG AAG",
+      "TTTT TTTT",
+      "ACC CC",
+      "AGGG C",
+      "ACC TC",
+      "AG ATG",
+      "ATT AC",
+      "AAG CC",
+      "GG CC",
+      "AGG AG",
+      "TC AA",
+      "ATT GC",
+      "TATT G",
+      "AT AAC",
+      "AT ATC",
+      "TT TAC",
+      "ATG GC",
+      "AAGG C",
+      "ACC AC",
+      "G TG",
+      "AT CCC",
+      "AG AAC",
+      "ATT TT",
+      "TTG CC",
+      "AA ATC",
+      "AT AAG",
+      "TTGG C",
+      "TGG AG",
+      "ATG GG",
+      "AA AGC",
+      "AGGG G",
+      "ATC AC",
+      "ATT TG",
+      "AA TTC",
+      "TGC AC",
+      "TT TGG",
+      "TC G",
+      "AG AGC",
+      "AA AGG",
+      "GG GC",
+      "TTGG G",
+      "AG AAAA",
+      "TAT CC",
+      "TC TCC",
+      "AT AGC",
+      "TG AGG",
+      "TT TATT",
+      "AG TAA",
+      "AG AGG",
+      "TC TTC",
+      "AC ATT",
+      "TCC TG",
+      "AG CCC",
+      "TATG C",
+      "TT AAAA",
+      "AG ATT",
+      "TT AAC",
+      "GG GG",
+      "AAG AC",
+      "TC ATT",
+      "TTC TG",
+      "AG ACC",
+      "AAGG G",
+      "AT ACC",
+      "TT TAT",
+      "AAG TG",
+      "TT ATG",
+      "AAG AA",
+      "TAG CC",
+      "TTC AC",
+      "AGG TG",
+      "TTG AA",
+      "ATC TG",
+      "AGC AC",
+      "TGC TG",
+      "AA ACC",
+      "ATG TG",
+      "TTTT CC",
+      "AG TTC",
+      "TCC TC",
+      "TATG G",
+      "AA TAC",
+      "AG TGG",
+      "TAG GC",
+      "AGC TC",
+      "AT AGG",
+      "TT ATC",
+      "TT AAG",
+      "T ACCC",
+      "TTTT TG",
+      "AAC AC",
+      "TGC TC",
+      "AG ATC",
+      "TCCC AGC",
+      "AGC TG",
+      "AA TAG",
+      "TC TTG",
+      "AGTG GC",
+      "ATT GG",
+      "TAC TC",
+      "TAA AC",
+      "AA TGG",
+      "AGG TC",
+      "AGG AC",
+      "TTG TG",
+      "TAT AC",
+      "ATT TTC",
+      "AT ATAA",
+      "AGGC TG",
+      "ATT TAA",
+      "AG TT",
+      "AG TAG",
+      "ATG AC",
+      "AA TGC",
+      "TCC AC",
+      "CC CC",
+      "ATG TC",
+      "AAC TC",
+      "TTTT TC",
+      "TAA GC",
+      "AAG TC",
+      "TGG TG",
+      "TAT AA",
+      "AG TGC",
+      "TAA GG",
+      "ACC TG",
+      "TT AGC",
+      "AA ATAA",
+      "TGCC TC",
+      "AA TCC",
+      "TTGG CC",
+      "TAG GG",
+      "TGG AC",
+      "TTG TC",
+      "AA CCC",
+      "TT ACC",
+      "TAA CC",
+      "AA TTTT",
+      "AA AGAA",
+      "ATT ATT",
+      "AGC G",
+      "AAAA AC",
+      "TAA TG",
+      "TTG AC",
+      "AG TCC",
+      "AAC TG",
+      "AG TTG",
+      "AA TTG",
+      "TC TGC",
+      "TT AGG",
+      "TAC AC",
+      "AGAA GG",
+      "AT ATTC",
+      "AAAA CC",
+      "AAAA GC",
+      "TG CCC",
+      "AC TGC",
+      "AGAA GC",
+      "TAA TAA",
+      "AA TATT",
+      "ACC ATG",
+      "TGG TC",
+      "TTTT GC",
+      "AAC G",
+      "TAC TG",
+      "ACAC ACAC",
+      "ATT TTG",
+      "TCC G",
+      "TGC G",
+      "AAAA TG",
+      "AC ATG",
+      "TC AGC",
+      "ATC G",
+      "AG TAC",
+      "TTTT GG",
+      "AA TAT",
+      "AG AGAA",
+      "TTC G",
+      "TCC AGCC",
+      "AT ATAC",
+      "TC ACC",
+      "AAAA GG",
+      "TGTG TGTG",
+      "TC ATC",
+      "TGC TGGG",
+      "TG AAG",
+      "TG TAG",
+      "TG TGG",
+      "AAAA ATT",
+      "AC TTC",
+      "TTCC CC",
+      "AT AGAA",
+      "TTG CCC",
+      "AGG AGG",
+      "TT TCCC",
+      "TAT ATT",
+      "ACC G",
+      "AC TAC",
+      "TCAC TGC",
+      "GC G",
+      "TT TGTG",
+      "AC AGC",
+      "TC ATG",
+      "AG TTTT",
+      "AGG AA",
+      "TT TATG",
+      "AT ATTG",
+      "TG ATG",
+      "TC TAA",
+      "TG TGC",
+      "AGG AAG",
+      "TT TGGG",
+      "TG TTC",
+      "AGCC CC",
+      "AG TTTC",
+      "AGGC TGG",
+      "TTTG CC",
+      "ATT TCC",
+      "AT ACAC",
+      "AAAA TAA",
+      "TAG AC",
+      "AGG AGAA",
+      "TG AGC",
+      "TGG AA",
+      "TTTT TAA",
+      "AGCC TCCC",
+      "ATG AA",
+      "TT TAAG",
+      "TC TGG",
+      "TT TATC",
+      "TT ATAA",
+      "TG ATT",
+      "AAC AA",
+      "TAGC TGGG",
+      "TC AAG",
+      "AAAA AA",
+      "ACTT TGGG",
+      "TATT CC",
+      "TC AGG",
+      "AAC AG",
+      "TTC TTC",
+      "TGTG GC",
+      "AT ATGC",
+      "ATTAC AGGC",
+      "AGGG GC",
+      "AGGG CC",
+      "TT ATTC",
+      "AT ATCC",
+      "TGTAA TCCCAGC",
+      "TAC G",
+      "AGAA AC",
+      "TG TCC",
+      "AG ATGG",
+      "TGTG CC",
+      "TTTC TC",
+      "TG AAC",
+      "AG TCTC",
+      "TG TTG",
+      "ATT TTTT",
+      "AAG AAG",
+      "TGGG GC",
+      "AGC AGC",
+      "G CCC",
+      "TTTG GC",
+      "AGGCTG AGGC",
+      "TGGG CC",
+      "TTC TCC",
+      "TAG AA",
+      "TGGAG TGC",
+      "ATT AA",
+      "AGTG CC",
+      "TG TCTC",
+      "AT ATGG",
+      "AC ATC",
+      "TGGG GG",
+      "TG ACC",
+      "AC TCC",
+      "TAA AAC",
+      "AG ATAA",
+      "TAA TTTT",
+      "TC AAC",
+      "TC TAC",
+      "TC TAG",
+      "G AG",
+      "TAA ATG",
+      "AGC AA",
+      "TAT ATG",
+      "ATAT ATAT",
+      "ATT TGC",
+      "TCC TCC",
+      "CCC AC",
+      "ATT TATT",
+      "TC TGCC",
+      "ATGG CC",
+      "TC GC",
+      "AG TATT",
+      "AGAA CC",
+      "TT AAAC",
+      "AA ATTC",
+      "AG AGAC",
+      "ATT TAC",
+      "ATTG CC",
+      "AAC AAC",
+      "TT TAAC",
+      "AC GG",
+      "AAG AAAA",
+      "TCTG GC",
+      "ATTC TCC",
+      "AGG TGG",
+      "TGC TGC",
+      "TTC AAG",
+      "AG AGGG",
+      "AC ACC",
+      "TC TTTT",
+      "AG AGGC",
+      "ATC ACC",
+      "TAA ATT",
+      "AAGG CC",
+      "TTGC AGTG",
+      "TG TAC",
+      "AA TTTC",
+      "ATCC CC",
+      "AC AAG",
+      "AC AGG",
+      "AC AAC",
+      "TGCC CC",
+      "AG ATTC",
+      "TT AGAA",
+      "TTGG GG",
+      "AG ACAC",
+      "TGG AAG",
+      "ACC TCC",
+      "ATG GGG",
+      "AGCC TCC",
+      "TT ATTG",
+      "TAA AAG",
+      "ATC TTC",
+      "ATC TCC",
+      "TGAA GC",
+      "TAA TC",
+      "AA ATGC",
+      "TTG TTG",
+      "ATT CCC",
+      "TAC TAAAA",
+      "AT AGTG",
+      "AA ATAC",
+      "TTGG GC",
+      "TAG AGAC",
+      "TG TTTT",
+      "TTC TGC",
+      "TGG CCC",
+      "TCTG TC",
+      "AGC TCC",
+      "AAC TCC",
+      "TT AGCC",
+      "AAAG TGCTGGG",
+      "AT AGAC",
+      "TATT TTTAG",
+      "AC TTG",
+      "ACC ACC",
+      "AA ACAC",
+      "G TGG",
+      "ATT TAG",
+      "AGG AGC",
+      "AGGC TGGAGTGC",
+      "AT ACCC",
+      "ATG TAA",
+      "AC GC",
+      "AG TAT",
+      "TT TACC",
+      "AC TAA",
+      "AGG CCC",
+      "AAGG GG",
+      "TCTC G",
+      "ATG AAG",
+      "AA AGAC",
+      "TG AAAA",
+      "AAGG GC",
+      "AT AGGC",
+      "AG AGTG",
+      "AGC TGC",
+      "ATG TTC",
+      "TATT TC",
+      "TG ATC",
+      "AG TTTG",
+      "AGC TAA",
+      "AG AGCC",
+      "TGC TTC",
+      "ATC ATC",
+      "AAC ATGG",
+      "AGC TTC",
+      "AAG AAC",
+      "TTTT TTG",
+      "AGGG GG",
+      "ATAA GC",
+      "TAAG CC",
+      "AC TGG",
+      "AC AAAA",
+      "ATC ATT",
+      "TC TTTC",
+      "ATG ATG",
+      "TGC AA",
+      "AGG TTC",
+      "AAC ATT",
+      "ATG GGC",
+      "AT AGAG",
+      "AA ATGG",
+      "AG TTCC",
+      "TT TAGC",
+      "AAC TTC",
+      "AGC AAG",
+      "AT AAAAC",
+      "AAAA TC",
+      "AGCC AC",
+      "AGG AAC",
+      "TTAA CC",
+      "TATT TATT",
+      "TTTC TG",
+      "ATAA GG",
+      "AGCC ACC",
+      "AG ATGC",
+      "TTAA GC",
+      "TTG TAA",
+      "AG TGTG",
+      "AACC CC",
+      "TTC ATT",
+      "ATC ATG",
+      "AA TGAA",
+      "AGG TGC",
+      "AAAAAAAA AAAAAAAA",
+      "AGG ATG",
+      "AGCC G",
+      "TGG TGG",
+      "AG TGGG",
+      "TGCAC TCCAGCC",
+      "TATT GC",
+      "TAG TC",
+      "CCC G",
+      "AAG TAA",
+      "TAG TG",
+      "TTTTTTTT TTTTTTTT",
+      "AGC ATT",
+      "ATC TGC",
+      "TCTC AC",
+      "AA ATTG",
+      "TT TAGG",
+      "AG ACCC",
+      "GGG CC",
+      "TCC TTC",
+      "AT AGGG",
+      "AA TATG",
+      "TT ATAC",
+      "TAG AAG",
+      "AA AGTG",
+      "AA ATCC",
+      "TTCC TC",
+      "TTTC AC",
+      "AG TATG",
+      "TACTAAAA ATAC",
+      "ATG TGC",
+      "AGG AGGC",
+      "TAT ATC",
+      "TTC TAA",
+      "TG AGGC",
+      "ACAC AC",
+      "TCC CCC",
+      "AAC ATC",
+      "AAGC G",
+      "AA TGGC",
+      "ACC CCC",
+      "AG ATAC",
+      "AT AAAAG",
+      "ATG ATT",
+      "TGG AGG",
+      "AG TTAA"
+    ]
+  }
+}

Grover_Pretrained/trainer_state.json

@@ -0,0 +1,1216 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 15.120586678763136,
+  "global_step": 200000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 5.7461,
+      "step": 1000
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 5.2435,
+      "step": 2000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.2e-05,
+      "loss": 5.1697,
+      "step": 3000
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 5.1422,
+      "step": 4000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 2e-05,
+      "loss": 5.1265,
+      "step": 5000
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 2.4e-05,
+      "loss": 5.1186,
+      "step": 6000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.8e-05,
+      "loss": 5.1159,
+      "step": 7000
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 5.11,
+      "step": 8000
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.6e-05,
+      "loss": 5.1084,
+      "step": 9000
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 4e-05,
+      "loss": 5.1073,
+      "step": 10000
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 3.978947368421053e-05,
+      "loss": 5.1033,
+      "step": 11000
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 3.9578947368421056e-05,
+      "loss": 5.103,
+      "step": 12000
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.936842105263158e-05,
+      "loss": 5.0998,
+      "step": 13000
+    },
+    {
+      "epoch": 1.06,
+      "learning_rate": 3.915789473684211e-05,
+      "loss": 5.099,
+      "step": 14000
+    },
+    {
+      "epoch": 1.13,
+      "learning_rate": 3.8947368421052636e-05,
+      "loss": 5.0958,
+      "step": 15000
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 3.873684210526316e-05,
+      "loss": 5.0956,
+      "step": 16000
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 3.852631578947369e-05,
+      "loss": 5.0925,
+      "step": 17000
+    },
+    {
+      "epoch": 1.36,
+      "learning_rate": 3.8315789473684216e-05,
+      "loss": 5.0947,
+      "step": 18000
+    },
+    {
+      "epoch": 1.44,
+      "learning_rate": 3.8105263157894735e-05,
+      "loss": 5.0925,
+      "step": 19000
+    },
+    {
+      "epoch": 1.51,
+      "learning_rate": 3.789473684210526e-05,
+      "loss": 5.093,
+      "step": 20000
+    },
+    {
+      "epoch": 1.59,
+      "learning_rate": 3.7684210526315795e-05,
+      "loss": 5.0922,
+      "step": 21000
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 3.747368421052632e-05,
+      "loss": 5.0894,
+      "step": 22000
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 3.726315789473685e-05,
+      "loss": 5.0699,
+      "step": 23000
+    },
+    {
+      "epoch": 1.81,
+      "learning_rate": 3.705263157894737e-05,
+      "loss": 5.037,
+      "step": 24000
+    },
+    {
+      "epoch": 1.89,
+      "learning_rate": 3.6842105263157895e-05,
+      "loss": 5.0146,
+      "step": 25000
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 3.663157894736842e-05,
+      "loss": 4.9928,
+      "step": 26000
+    },
+    {
+      "epoch": 2.04,
+      "learning_rate": 3.6421052631578955e-05,
+      "loss": 4.9687,
+      "step": 27000
+    },
+    {
+      "epoch": 2.12,
+      "learning_rate": 3.621052631578948e-05,
+      "loss": 4.9464,
+      "step": 28000
+    },
+    {
+      "epoch": 2.19,
+      "learning_rate": 3.6e-05,
+      "loss": 4.922,
+      "step": 29000
+    },
+    {
+      "epoch": 2.27,
+      "learning_rate": 3.578947368421053e-05,
+      "loss": 4.8983,
+      "step": 30000
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 3.5578947368421054e-05,
+      "loss": 4.8721,
+      "step": 31000
+    },
+    {
+      "epoch": 2.42,
+      "learning_rate": 3.536842105263158e-05,
+      "loss": 4.8441,
+      "step": 32000
+    },
+    {
+      "epoch": 2.49,
+      "learning_rate": 3.515789473684211e-05,
+      "loss": 4.8125,
+      "step": 33000
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 3.4947368421052634e-05,
+      "loss": 4.7796,
+      "step": 34000
+    },
+    {
+      "epoch": 2.65,
+      "learning_rate": 3.473684210526316e-05,
+      "loss": 4.7264,
+      "step": 35000
+    },
+    {
+      "epoch": 2.72,
+      "learning_rate": 3.452631578947369e-05,
+      "loss": 4.6306,
+      "step": 36000
+    },
+    {
+      "epoch": 2.8,
+      "learning_rate": 3.431578947368421e-05,
+      "loss": 4.5571,
+      "step": 37000
+    },
+    {
+      "epoch": 2.87,
+      "learning_rate": 3.410526315789474e-05,
+      "loss": 4.4757,
+      "step": 38000
+    },
+    {
+      "epoch": 2.95,
+      "learning_rate": 3.3894736842105266e-05,
+      "loss": 4.3942,
+      "step": 39000
+    },
+    {
+      "epoch": 3.02,
+      "learning_rate": 3.368421052631579e-05,
+      "loss": 4.3376,
+      "step": 40000
+    },
+    {
+      "epoch": 3.1,
+      "learning_rate": 3.347368421052632e-05,
+      "loss": 4.2928,
+      "step": 41000
+    },
+    {
+      "epoch": 3.18,
+      "learning_rate": 3.3263157894736846e-05,
+      "loss": 4.2614,
+      "step": 42000
+    },
+    {
+      "epoch": 3.25,
+      "learning_rate": 3.305263157894737e-05,
+      "loss": 4.2338,
+      "step": 43000
+    },
+    {
+      "epoch": 3.33,
+      "learning_rate": 3.28421052631579e-05,
+      "loss": 4.2183,
+      "step": 44000
+    },
+    {
+      "epoch": 3.4,
+      "learning_rate": 3.2631578947368426e-05,
+      "loss": 4.194,
+      "step": 45000
+    },
+    {
+      "epoch": 3.48,
+      "learning_rate": 3.2421052631578945e-05,
+      "loss": 4.1761,
+      "step": 46000
+    },
+    {
+      "epoch": 3.55,
+      "learning_rate": 3.221052631578947e-05,
+      "loss": 4.1644,
+      "step": 47000
+    },
+    {
+      "epoch": 3.63,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 4.1498,
+      "step": 48000
+    },
+    {
+      "epoch": 3.7,
+      "learning_rate": 3.178947368421053e-05,
+      "loss": 4.1397,
+      "step": 49000
+    },
+    {
+      "epoch": 3.78,
+      "learning_rate": 3.157894736842106e-05,
+      "loss": 4.1275,
+      "step": 50000
+    },
+    {
+      "epoch": 3.86,
+      "learning_rate": 3.136842105263158e-05,
+      "loss": 4.1146,
+      "step": 51000
+    },
+    {
+      "epoch": 3.93,
+      "learning_rate": 3.1157894736842105e-05,
+      "loss": 4.1074,
+      "step": 52000
+    },
+    {
+      "epoch": 4.01,
+      "learning_rate": 3.094736842105263e-05,
+      "loss": 4.1001,
+      "step": 53000
+    },
+    {
+      "epoch": 4.08,
+      "learning_rate": 3.073684210526316e-05,
+      "loss": 4.0887,
+      "step": 54000
+    },
+    {
+      "epoch": 4.16,
+      "learning_rate": 3.052631578947369e-05,
+      "loss": 4.0836,
+      "step": 55000
+    },
+    {
+      "epoch": 4.23,
+      "learning_rate": 3.0315789473684214e-05,
+      "loss": 4.0808,
+      "step": 56000
+    },
+    {
+      "epoch": 4.31,
+      "learning_rate": 3.010526315789474e-05,
+      "loss": 4.0691,
+      "step": 57000
+    },
+    {
+      "epoch": 4.38,
+      "learning_rate": 2.9894736842105264e-05,
+      "loss": 4.0624,
+      "step": 58000
+    },
+    {
+      "epoch": 4.46,
+      "learning_rate": 2.968421052631579e-05,
+      "loss": 4.0501,
+      "step": 59000
+    },
+    {
+      "epoch": 4.54,
+      "learning_rate": 2.9473684210526317e-05,
+      "loss": 4.0523,
+      "step": 60000
+    },
+    {
+      "epoch": 4.61,
+      "learning_rate": 2.9263157894736847e-05,
+      "loss": 4.0434,
+      "step": 61000
+    },
+    {
+      "epoch": 4.69,
+      "learning_rate": 2.9052631578947374e-05,
+      "loss": 4.0381,
+      "step": 62000
+    },
+    {
+      "epoch": 4.76,
+      "learning_rate": 2.8842105263157897e-05,
+      "loss": 4.0363,
+      "step": 63000
+    },
+    {
+      "epoch": 4.84,
+      "learning_rate": 2.8631578947368423e-05,
+      "loss": 4.0303,
+      "step": 64000
+    },
+    {
+      "epoch": 4.91,
+      "learning_rate": 2.842105263157895e-05,
+      "loss": 4.0211,
+      "step": 65000
+    },
+    {
+      "epoch": 4.99,
+      "learning_rate": 2.8210526315789476e-05,
+      "loss": 4.0168,
+      "step": 66000
+    },
+    {
+      "epoch": 5.07,
+      "learning_rate": 2.8e-05,
+      "loss": 4.0122,
+      "step": 67000
+    },
+    {
+      "epoch": 5.14,
+      "learning_rate": 2.778947368421053e-05,
+      "loss": 4.0091,
+      "step": 68000
+    },
+    {
+      "epoch": 5.22,
+      "learning_rate": 2.7578947368421056e-05,
+      "loss": 4.0019,
+      "step": 69000
+    },
+    {
+      "epoch": 5.29,
+      "learning_rate": 2.7368421052631583e-05,
+      "loss": 3.9973,
+      "step": 70000
+    },
+    {
+      "epoch": 5.37,
+      "learning_rate": 2.715789473684211e-05,
+      "loss": 3.9994,
+      "step": 71000
+    },
+    {
+      "epoch": 5.44,
+      "learning_rate": 2.6947368421052632e-05,
+      "loss": 3.9907,
+      "step": 72000
+    },
+    {
+      "epoch": 5.52,
+      "learning_rate": 2.673684210526316e-05,
+      "loss": 3.9873,
+      "step": 73000
+    },
+    {
+      "epoch": 5.59,
+      "learning_rate": 2.6526315789473685e-05,
+      "loss": 3.984,
+      "step": 74000
+    },
+    {
+      "epoch": 5.67,
+      "learning_rate": 2.6315789473684215e-05,
+      "loss": 3.9795,
+      "step": 75000
+    },
+    {
+      "epoch": 5.75,
+      "learning_rate": 2.610526315789474e-05,
+      "loss": 3.973,
+      "step": 76000
+    },
+    {
+      "epoch": 5.82,
+      "learning_rate": 2.5894736842105265e-05,
+      "loss": 3.9739,
+      "step": 77000
+    },
+    {
+      "epoch": 5.9,
+      "learning_rate": 2.568421052631579e-05,
+      "loss": 3.9637,
+      "step": 78000
+    },
+    {
+      "epoch": 5.97,
+      "learning_rate": 2.5473684210526318e-05,
+      "loss": 3.9668,
+      "step": 79000
+    },
+    {
+      "epoch": 6.05,
+      "learning_rate": 2.526315789473684e-05,
+      "loss": 3.9624,
+      "step": 80000
+    },
+    {
+      "epoch": 6.12,
+      "learning_rate": 2.5052631578947368e-05,
+      "loss": 3.9513,
+      "step": 81000
+    },
+    {
+      "epoch": 6.2,
+      "learning_rate": 2.4842105263157898e-05,
+      "loss": 3.9586,
+      "step": 82000
+    },
+    {
+      "epoch": 6.28,
+      "learning_rate": 2.4631578947368424e-05,
+      "loss": 3.9509,
+      "step": 83000
+    },
+    {
+      "epoch": 6.35,
+      "learning_rate": 2.442105263157895e-05,
+      "loss": 3.9526,
+      "step": 84000
+    },
+    {
+      "epoch": 6.43,
+      "learning_rate": 2.4210526315789474e-05,
+      "loss": 3.9427,
+      "step": 85000
+    },
+    {
+      "epoch": 6.5,
+      "learning_rate": 2.4e-05,
+      "loss": 3.9448,
+      "step": 86000
+    },
+    {
+      "epoch": 6.58,
+      "learning_rate": 2.3789473684210527e-05,
+      "loss": 3.9331,
+      "step": 87000
+    },
+    {
+      "epoch": 6.65,
+      "learning_rate": 2.3578947368421054e-05,
+      "loss": 3.9368,
+      "step": 88000
+    },
+    {
+      "epoch": 6.73,
+      "learning_rate": 2.3368421052631584e-05,
+      "loss": 3.9316,
+      "step": 89000
+    },
+    {
+      "epoch": 6.8,
+      "learning_rate": 2.3157894736842107e-05,
+      "loss": 3.932,
+      "step": 90000
+    },
+    {
+      "epoch": 6.88,
+      "learning_rate": 2.2947368421052633e-05,
+      "loss": 3.9249,
+      "step": 91000
+    },
+    {
+      "epoch": 6.96,
+      "learning_rate": 2.273684210526316e-05,
+      "loss": 3.9272,
+      "step": 92000
+    },
+    {
+      "epoch": 7.03,
+      "learning_rate": 2.2526315789473686e-05,
+      "loss": 3.9157,
+      "step": 93000
+    },
+    {
+      "epoch": 7.11,
+      "learning_rate": 2.231578947368421e-05,
+      "loss": 3.9139,
+      "step": 94000
+    },
+    {
+      "epoch": 7.18,
+      "learning_rate": 2.210526315789474e-05,
+      "loss": 3.9091,
+      "step": 95000
+    },
+    {
+      "epoch": 7.26,
+      "learning_rate": 2.1894736842105266e-05,
+      "loss": 3.9164,
+      "step": 96000
+    },
+    {
+      "epoch": 7.33,
+      "learning_rate": 2.1684210526315793e-05,
+      "loss": 3.9119,
+      "step": 97000
+    },
+    {
+      "epoch": 7.41,
+      "learning_rate": 2.147368421052632e-05,
+      "loss": 3.9058,
+      "step": 98000
+    },
+    {
+      "epoch": 7.48,
+      "learning_rate": 2.1263157894736842e-05,
+      "loss": 3.9131,
+      "step": 99000
+    },
+    {
+      "epoch": 7.56,
+      "learning_rate": 2.105263157894737e-05,
+      "loss": 3.9097,
+      "step": 100000
+    },
+    {
+      "epoch": 7.64,
+      "learning_rate": 2.0842105263157895e-05,
+      "loss": 3.9044,
+      "step": 101000
+    },
+    {
+      "epoch": 7.71,
+      "learning_rate": 2.0631578947368425e-05,
+      "loss": 3.9004,
+      "step": 102000
+    },
+    {
+      "epoch": 7.79,
+      "learning_rate": 2.0421052631578952e-05,
+      "loss": 3.8937,
+      "step": 103000
+    },
+    {
+      "epoch": 7.86,
+      "learning_rate": 2.0210526315789475e-05,
+      "loss": 3.9017,
+      "step": 104000
+    },
+    {
+      "epoch": 7.94,
+      "learning_rate": 2e-05,
+      "loss": 3.899,
+      "step": 105000
+    },
+    {
+      "epoch": 8.01,
+      "learning_rate": 1.9789473684210528e-05,
+      "loss": 3.8979,
+      "step": 106000
+    },
+    {
+      "epoch": 8.09,
+      "learning_rate": 1.9578947368421055e-05,
+      "loss": 3.8889,
+      "step": 107000
+    },
+    {
+      "epoch": 8.17,
+      "learning_rate": 1.936842105263158e-05,
+      "loss": 3.8857,
+      "step": 108000
+    },
+    {
+      "epoch": 8.24,
+      "learning_rate": 1.9157894736842108e-05,
+      "loss": 3.8892,
+      "step": 109000
+    },
+    {
+      "epoch": 8.32,
+      "learning_rate": 1.894736842105263e-05,
+      "loss": 3.887,
+      "step": 110000
+    },
+    {
+      "epoch": 8.39,
+      "learning_rate": 1.873684210526316e-05,
+      "loss": 3.8848,
+      "step": 111000
+    },
+    {
+      "epoch": 8.47,
+      "learning_rate": 1.8526315789473684e-05,
+      "loss": 3.8805,
+      "step": 112000
+    },
+    {
+      "epoch": 8.54,
+      "learning_rate": 1.831578947368421e-05,
+      "loss": 3.8819,
+      "step": 113000
+    },
+    {
+      "epoch": 8.62,
+      "learning_rate": 1.810526315789474e-05,
+      "loss": 3.8753,
+      "step": 114000
+    },
+    {
+      "epoch": 8.69,
+      "learning_rate": 1.7894736842105264e-05,
+      "loss": 3.881,
+      "step": 115000
+    },
+    {
+      "epoch": 8.77,
+      "learning_rate": 1.768421052631579e-05,
+      "loss": 3.8784,
+      "step": 116000
+    },
+    {
+      "epoch": 8.85,
+      "learning_rate": 1.7473684210526317e-05,
+      "loss": 3.8694,
+      "step": 117000
+    },
+    {
+      "epoch": 8.92,
+      "learning_rate": 1.7263157894736843e-05,
+      "loss": 3.8727,
+      "step": 118000
+    },
+    {
+      "epoch": 9.0,
+      "learning_rate": 1.705263157894737e-05,
+      "loss": 3.8701,
+      "step": 119000
+    },
+    {
+      "epoch": 9.07,
+      "learning_rate": 1.6842105263157896e-05,
+      "loss": 3.8676,
+      "step": 120000
+    },
+    {
+      "epoch": 9.15,
+      "learning_rate": 1.6631578947368423e-05,
+      "loss": 3.8632,
+      "step": 121000
+    },
+    {
+      "epoch": 9.22,
+      "learning_rate": 1.642105263157895e-05,
+      "loss": 3.8645,
+      "step": 122000
+    },
+    {
+      "epoch": 9.3,
+      "learning_rate": 1.6210526315789473e-05,
+      "loss": 3.8682,
+      "step": 123000
+    },
+    {
+      "epoch": 9.37,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 3.8575,
+      "step": 124000
+    },
+    {
+      "epoch": 9.45,
+      "learning_rate": 1.578947368421053e-05,
+      "loss": 3.8633,
+      "step": 125000
+    },
+    {
+      "epoch": 9.53,
+      "learning_rate": 1.5578947368421052e-05,
+      "loss": 3.8594,
+      "step": 126000
+    },
+    {
+      "epoch": 9.6,
+      "learning_rate": 1.536842105263158e-05,
+      "loss": 3.8552,
+      "step": 127000
+    },
+    {
+      "epoch": 9.68,
+      "learning_rate": 1.5157894736842107e-05,
+      "loss": 3.8563,
+      "step": 128000
+    },
+    {
+      "epoch": 9.75,
+      "learning_rate": 1.4947368421052632e-05,
+      "loss": 3.8596,
+      "step": 129000
+    },
+    {
+      "epoch": 9.83,
+      "learning_rate": 1.4736842105263159e-05,
+      "loss": 3.8565,
+      "step": 130000
+    },
+    {
+      "epoch": 9.9,
+      "learning_rate": 1.4526315789473687e-05,
+      "loss": 3.8512,
+      "step": 131000
+    },
+    {
+      "epoch": 9.98,
+      "learning_rate": 1.4315789473684212e-05,
+      "loss": 3.8531,
+      "step": 132000
+    },
+    {
+      "epoch": 10.06,
+      "learning_rate": 1.4105263157894738e-05,
+      "loss": 3.8538,
+      "step": 133000
+    },
+    {
+      "epoch": 10.13,
+      "learning_rate": 1.3894736842105265e-05,
+      "loss": 3.8477,
+      "step": 134000
+    },
+    {
+      "epoch": 10.21,
+      "learning_rate": 1.3684210526315791e-05,
+      "loss": 3.8497,
+      "step": 135000
+    },
+    {
+      "epoch": 10.28,
+      "learning_rate": 1.3473684210526316e-05,
+      "loss": 3.8455,
+      "step": 136000
+    },
+    {
+      "epoch": 10.36,
+      "learning_rate": 1.3263157894736843e-05,
+      "loss": 3.8442,
+      "step": 137000
+    },
+    {
+      "epoch": 10.43,
+      "learning_rate": 1.305263157894737e-05,
+      "loss": 3.8466,
+      "step": 138000
+    },
+    {
+      "epoch": 10.51,
+      "learning_rate": 1.2842105263157896e-05,
+      "loss": 3.8465,
+      "step": 139000
+    },
+    {
+      "epoch": 10.58,
+      "learning_rate": 1.263157894736842e-05,
+      "loss": 3.8443,
+      "step": 140000
+    },
+    {
+      "epoch": 10.66,
+      "learning_rate": 1.2421052631578949e-05,
+      "loss": 3.8407,
+      "step": 141000
+    },
+    {
+      "epoch": 10.74,
+      "learning_rate": 1.2210526315789475e-05,
+      "loss": 3.8414,
+      "step": 142000
+    },
+    {
+      "epoch": 10.81,
+      "learning_rate": 1.2e-05,
+      "loss": 3.835,
+      "step": 143000
+    },
+    {
+      "epoch": 10.89,
+      "learning_rate": 1.1789473684210527e-05,
+      "loss": 3.8321,
+      "step": 144000
+    },
+    {
+      "epoch": 10.96,
+      "learning_rate": 1.1578947368421053e-05,
+      "loss": 3.8366,
+      "step": 145000
+    },
+    {
+      "epoch": 11.04,
+      "learning_rate": 1.136842105263158e-05,
+      "loss": 3.8353,
+      "step": 146000
+    },
+    {
+      "epoch": 11.11,
+      "learning_rate": 1.1157894736842105e-05,
+      "loss": 3.8347,
+      "step": 147000
+    },
+    {
+      "epoch": 11.19,
+      "learning_rate": 1.0947368421052633e-05,
+      "loss": 3.833,
+      "step": 148000
+    },
+    {
+      "epoch": 11.26,
+      "learning_rate": 1.073684210526316e-05,
+      "loss": 3.8337,
+      "step": 149000
+    },
+    {
+      "epoch": 11.34,
+      "learning_rate": 1.0526315789473684e-05,
+      "loss": 3.8342,
+      "step": 150000
+    },
+    {
+      "epoch": 11.42,
+      "learning_rate": 1.0315789473684213e-05,
+      "loss": 3.8302,
+      "step": 151000
+    },
+    {
+      "epoch": 11.49,
+      "learning_rate": 1.0105263157894738e-05,
+      "loss": 3.8326,
+      "step": 152000
+    },
+    {
+      "epoch": 11.57,
+      "learning_rate": 9.894736842105264e-06,
+      "loss": 3.827,
+      "step": 153000
+    },
+    {
+      "epoch": 11.64,
+      "learning_rate": 9.68421052631579e-06,
+      "loss": 3.8309,
+      "step": 154000
+    },
+    {
+      "epoch": 11.72,
+      "learning_rate": 9.473684210526315e-06,
+      "loss": 3.8293,
+      "step": 155000
+    },
+    {
+      "epoch": 11.79,
+      "learning_rate": 9.263157894736842e-06,
+      "loss": 3.8277,
+      "step": 156000
+    },
+    {
+      "epoch": 11.87,
+      "learning_rate": 9.05263157894737e-06,
+      "loss": 3.8273,
+      "step": 157000
+    },
+    {
+      "epoch": 11.95,
+      "learning_rate": 8.842105263157895e-06,
+      "loss": 3.8199,
+      "step": 158000
+    },
+    {
+      "epoch": 12.02,
+      "learning_rate": 8.631578947368422e-06,
+      "loss": 3.8253,
+      "step": 159000
+    },
+    {
+      "epoch": 12.1,
+      "learning_rate": 8.421052631578948e-06,
+      "loss": 3.8267,
+      "step": 160000
+    },
+    {
+      "epoch": 12.17,
+      "learning_rate": 8.210526315789475e-06,
+      "loss": 3.8287,
+      "step": 161000
+    },
+    {
+      "epoch": 12.25,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 3.8196,
+      "step": 162000
+    },
+    {
+      "epoch": 12.32,
+      "learning_rate": 7.789473684210526e-06,
+      "loss": 3.8208,
+      "step": 163000
+    },
+    {
+      "epoch": 12.4,
+      "learning_rate": 7.578947368421054e-06,
+      "loss": 3.8287,
+      "step": 164000
+    },
+    {
+      "epoch": 12.47,
+      "learning_rate": 7.368421052631579e-06,
+      "loss": 3.8223,
+      "step": 165000
+    },
+    {
+      "epoch": 12.55,
+      "learning_rate": 7.157894736842106e-06,
+      "loss": 3.8103,
+      "step": 166000
+    },
+    {
+      "epoch": 12.63,
+      "learning_rate": 6.947368421052632e-06,
+      "loss": 3.8183,
+      "step": 167000
+    },
+    {
+      "epoch": 12.7,
+      "learning_rate": 6.736842105263158e-06,
+      "loss": 3.8174,
+      "step": 168000
+    },
+    {
+      "epoch": 12.78,
+      "learning_rate": 6.526315789473685e-06,
+      "loss": 3.8154,
+      "step": 169000
+    },
+    {
+      "epoch": 12.85,
+      "learning_rate": 6.31578947368421e-06,
+      "loss": 3.8204,
+      "step": 170000
+    },
+    {
+      "epoch": 12.93,
+      "learning_rate": 6.105263157894738e-06,
+      "loss": 3.8104,
+      "step": 171000
+    },
+    {
+      "epoch": 13.0,
+      "learning_rate": 5.8947368421052634e-06,
+      "loss": 3.8173,
+      "step": 172000
+    },
+    {
+      "epoch": 13.08,
+      "learning_rate": 5.68421052631579e-06,
+      "loss": 3.8139,
+      "step": 173000
+    },
+    {
+      "epoch": 13.15,
+      "learning_rate": 5.4736842105263165e-06,
+      "loss": 3.809,
+      "step": 174000
+    },
+    {
+      "epoch": 13.23,
+      "learning_rate": 5.263157894736842e-06,
+      "loss": 3.8151,
+      "step": 175000
+    },
+    {
+      "epoch": 13.31,
+      "learning_rate": 5.052631578947369e-06,
+      "loss": 3.8129,
+      "step": 176000
+    },
+    {
+      "epoch": 13.38,
+      "learning_rate": 4.842105263157895e-06,
+      "loss": 3.815,
+      "step": 177000
+    },
+    {
+      "epoch": 13.46,
+      "learning_rate": 4.631578947368421e-06,
+      "loss": 3.8175,
+      "step": 178000
+    },
+    {
+      "epoch": 13.53,
+      "learning_rate": 4.4210526315789476e-06,
+      "loss": 3.8125,
+      "step": 179000
+    },
+    {
+      "epoch": 13.61,
+      "learning_rate": 4.210526315789474e-06,
+      "loss": 3.8101,
+      "step": 180000
+    },
+    {
+      "epoch": 13.68,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 3.8079,
+      "step": 181000
+    },
+    {
+      "epoch": 13.76,
+      "learning_rate": 3.789473684210527e-06,
+      "loss": 3.8124,
+      "step": 182000
+    },
+    {
+      "epoch": 13.84,
+      "learning_rate": 3.578947368421053e-06,
+      "loss": 3.8134,
+      "step": 183000
+    },
+    {
+      "epoch": 13.91,
+      "learning_rate": 3.368421052631579e-06,
+      "loss": 3.8048,
+      "step": 184000
+    },
+    {
+      "epoch": 13.99,
+      "learning_rate": 3.157894736842105e-06,
+      "loss": 3.8167,
+      "step": 185000
+    },
+    {
+      "epoch": 14.06,
+      "learning_rate": 2.9473684210526317e-06,
+      "loss": 3.812,
+      "step": 186000
+    },
+    {
+      "epoch": 14.14,
+      "learning_rate": 2.7368421052631583e-06,
+      "loss": 3.8089,
+      "step": 187000
+    },
+    {
+      "epoch": 14.21,
+      "learning_rate": 2.5263157894736844e-06,
+      "loss": 3.8119,
+      "step": 188000
+    },
+    {
+      "epoch": 14.29,
+      "learning_rate": 2.3157894736842105e-06,
+      "loss": 3.8058,
+      "step": 189000
+    },
+    {
+      "epoch": 14.36,
+      "learning_rate": 2.105263157894737e-06,
+      "loss": 3.8089,
+      "step": 190000
+    },
+    {
+      "epoch": 14.44,
+      "learning_rate": 1.8947368421052634e-06,
+      "loss": 3.8058,
+      "step": 191000
+    },
+    {
+      "epoch": 14.52,
+      "learning_rate": 1.6842105263157895e-06,
+      "loss": 3.8099,
+      "step": 192000
+    },
+    {
+      "epoch": 14.59,
+      "learning_rate": 1.4736842105263159e-06,
+      "loss": 3.7993,
+      "step": 193000
+    },
+    {
+      "epoch": 14.67,
+      "learning_rate": 1.2631578947368422e-06,
+      "loss": 3.8072,
+      "step": 194000
+    },
+    {
+      "epoch": 14.74,
+      "learning_rate": 1.0526315789473685e-06,
+      "loss": 3.806,
+      "step": 195000
+    },
+    {
+      "epoch": 14.82,
+      "learning_rate": 8.421052631578948e-07,
+      "loss": 3.8084,
+      "step": 196000
+    },
+    {
+      "epoch": 14.89,
+      "learning_rate": 6.315789473684211e-07,
+      "loss": 3.8074,
+      "step": 197000
+    },
+    {
+      "epoch": 14.97,
+      "learning_rate": 4.210526315789474e-07,
+      "loss": 3.8013,
+      "step": 198000
+    },
+    {
+      "epoch": 15.04,
+      "learning_rate": 2.105263157894737e-07,
+      "loss": 3.8081,
+      "step": 199000
+    },
+    {
+      "epoch": 15.12,
+      "learning_rate": 0.0,
+      "loss": 3.8009,
+      "step": 200000
+    }
+  ],
+  "max_steps": 200000,
+  "num_train_epochs": 16,
+  "total_flos": 5.051724710578883e+18,
+  "trial_name": null,
+  "trial_params": null
+}

Grover_Pretrained/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06ffcd57892162760bf8f1bd73a1a42cf2c64618c52c361b5fe38a722735c150
+size 4945

model_ablation_no_length/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

model_ablation_no_length/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 5120
+}

model_ablation_no_length/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "max_length": 512,
+  "pad_token_id": 0,
+  "transformers_version": "4.35.2"
+}

model_ablation_no_length/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06ce30b74288caa2d3abc415a67862967721c8b2a308332cf9fb00849481ff64
+size 359944256

model_ablation_no_length/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9371f8c50f85cd5bc9cfede91f8c222b9db45489698bbfb84f64fd81d25ea115
+size 720010699

model_ablation_no_length/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dfd380ff13cdf644746f177c474d8f73957dad2c31e564b41dc4f6165318a82
+size 14709

model_ablation_no_length/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3018e461ce1631f9bf04479faa74b8f3d9fd86bf68d2cd93702db04c8a02131
+size 1465

model_ablation_no_length/trainer_state.json

@@ -0,0 +1,619 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 27.19608376393799,
+  "eval_steps": 500,
+  "global_step": 100000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 7.3758,
+      "step": 1000
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 6.6082,
+      "step": 2000
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.2e-05,
+      "loss": 6.4181,
+      "step": 3000
+    },
+    {
+      "epoch": 1.09,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 6.325,
+      "step": 4000
+    },
+    {
+      "epoch": 1.36,
+      "learning_rate": 2e-05,
+      "loss": 6.2866,
+      "step": 5000
+    },
+    {
+      "epoch": 1.63,
+      "learning_rate": 2.4e-05,
+      "loss": 6.2655,
+      "step": 6000
+    },
+    {
+      "epoch": 1.9,
+      "learning_rate": 2.8e-05,
+      "loss": 6.2543,
+      "step": 7000
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 6.2456,
+      "step": 8000
+    },
+    {
+      "epoch": 2.45,
+      "learning_rate": 3.6e-05,
+      "loss": 6.2373,
+      "step": 9000
+    },
+    {
+      "epoch": 2.72,
+      "learning_rate": 4e-05,
+      "loss": 6.2309,
+      "step": 10000
+    },
+    {
+      "epoch": 2.99,
+      "learning_rate": 3.955555555555556e-05,
+      "loss": 6.2262,
+      "step": 11000
+    },
+    {
+      "epoch": 3.26,
+      "learning_rate": 3.9111111111111115e-05,
+      "loss": 6.221,
+      "step": 12000
+    },
+    {
+      "epoch": 3.54,
+      "learning_rate": 3.866666666666667e-05,
+      "loss": 6.2168,
+      "step": 13000
+    },
+    {
+      "epoch": 3.81,
+      "learning_rate": 3.8222222222222226e-05,
+      "loss": 6.2153,
+      "step": 14000
+    },
+    {
+      "epoch": 4.08,
+      "learning_rate": 3.777777777777778e-05,
+      "loss": 6.2099,
+      "step": 15000
+    },
+    {
+      "epoch": 4.35,
+      "learning_rate": 3.733333333333334e-05,
+      "loss": 6.209,
+      "step": 16000
+    },
+    {
+      "epoch": 4.62,
+      "learning_rate": 3.6888888888888896e-05,
+      "loss": 6.2097,
+      "step": 17000
+    },
+    {
+      "epoch": 4.9,
+      "learning_rate": 3.644444444444445e-05,
+      "loss": 6.2056,
+      "step": 18000
+    },
+    {
+      "epoch": 5.17,
+      "learning_rate": 3.6e-05,
+      "loss": 6.2036,
+      "step": 19000
+    },
+    {
+      "epoch": 5.44,
+      "learning_rate": 3.555555555555555e-05,
+      "loss": 6.2015,
+      "step": 20000
+    },
+    {
+      "epoch": 5.71,
+      "learning_rate": 3.511111111111111e-05,
+      "loss": 6.2014,
+      "step": 21000
+    },
+    {
+      "epoch": 5.98,
+      "learning_rate": 3.466666666666667e-05,
+      "loss": 6.1992,
+      "step": 22000
+    },
+    {
+      "epoch": 6.26,
+      "learning_rate": 3.4222222222222224e-05,
+      "loss": 6.1994,
+      "step": 23000
+    },
+    {
+      "epoch": 6.53,
+      "learning_rate": 3.377777777777778e-05,
+      "loss": 6.1886,
+      "step": 24000
+    },
+    {
+      "epoch": 6.8,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 6.111,
+      "step": 25000
+    },
+    {
+      "epoch": 7.07,
+      "learning_rate": 3.288888888888889e-05,
+      "loss": 6.0575,
+      "step": 26000
+    },
+    {
+      "epoch": 7.34,
+      "learning_rate": 3.2444444444444446e-05,
+      "loss": 6.0078,
+      "step": 27000
+    },
+    {
+      "epoch": 7.61,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 5.9631,
+      "step": 28000
+    },
+    {
+      "epoch": 7.89,
+      "learning_rate": 3.155555555555556e-05,
+      "loss": 5.922,
+      "step": 29000
+    },
+    {
+      "epoch": 8.16,
+      "learning_rate": 3.111111111111112e-05,
+      "loss": 5.8761,
+      "step": 30000
+    },
+    {
+      "epoch": 8.43,
+      "learning_rate": 3.066666666666667e-05,
+      "loss": 5.832,
+      "step": 31000
+    },
+    {
+      "epoch": 8.7,
+      "learning_rate": 3.0222222222222225e-05,
+      "loss": 5.7851,
+      "step": 32000
+    },
+    {
+      "epoch": 8.97,
+      "learning_rate": 2.977777777777778e-05,
+      "loss": 5.7381,
+      "step": 33000
+    },
+    {
+      "epoch": 9.25,
+      "learning_rate": 2.9333333333333333e-05,
+      "loss": 5.6705,
+      "step": 34000
+    },
+    {
+      "epoch": 9.52,
+      "learning_rate": 2.888888888888889e-05,
+      "loss": 5.4784,
+      "step": 35000
+    },
+    {
+      "epoch": 9.79,
+      "learning_rate": 2.8444444444444447e-05,
+      "loss": 5.1969,
+      "step": 36000
+    },
+    {
+      "epoch": 10.06,
+      "learning_rate": 2.8e-05,
+      "loss": 5.0725,
+      "step": 37000
+    },
+    {
+      "epoch": 10.33,
+      "learning_rate": 2.755555555555556e-05,
+      "loss": 5.0063,
+      "step": 38000
+    },
+    {
+      "epoch": 10.61,
+      "learning_rate": 2.7111111111111114e-05,
+      "loss": 4.9628,
+      "step": 39000
+    },
+    {
+      "epoch": 10.88,
+      "learning_rate": 2.6666666666666667e-05,
+      "loss": 4.9296,
+      "step": 40000
+    },
+    {
+      "epoch": 11.15,
+      "learning_rate": 2.6222222222222226e-05,
+      "loss": 4.899,
+      "step": 41000
+    },
+    {
+      "epoch": 11.42,
+      "learning_rate": 2.577777777777778e-05,
+      "loss": 4.8777,
+      "step": 42000
+    },
+    {
+      "epoch": 11.69,
+      "learning_rate": 2.5333333333333334e-05,
+      "loss": 4.8581,
+      "step": 43000
+    },
+    {
+      "epoch": 11.97,
+      "learning_rate": 2.4888888888888893e-05,
+      "loss": 4.8393,
+      "step": 44000
+    },
+    {
+      "epoch": 12.24,
+      "learning_rate": 2.444444444444445e-05,
+      "loss": 4.8243,
+      "step": 45000
+    },
+    {
+      "epoch": 12.51,
+      "learning_rate": 2.4e-05,
+      "loss": 4.8118,
+      "step": 46000
+    },
+    {
+      "epoch": 12.78,
+      "learning_rate": 2.355555555555556e-05,
+      "loss": 4.799,
+      "step": 47000
+    },
+    {
+      "epoch": 13.05,
+      "learning_rate": 2.3111111111111112e-05,
+      "loss": 4.7892,
+      "step": 48000
+    },
+    {
+      "epoch": 13.33,
+      "learning_rate": 2.2666666666666668e-05,
+      "loss": 4.7799,
+      "step": 49000
+    },
+    {
+      "epoch": 13.6,
+      "learning_rate": 2.2222222222222227e-05,
+      "loss": 4.7639,
+      "step": 50000
+    },
+    {
+      "epoch": 13.87,
+      "learning_rate": 2.177777777777778e-05,
+      "loss": 4.7586,
+      "step": 51000
+    },
+    {
+      "epoch": 14.14,
+      "learning_rate": 2.1333333333333335e-05,
+      "loss": 4.7515,
+      "step": 52000
+    },
+    {
+      "epoch": 14.41,
+      "learning_rate": 2.088888888888889e-05,
+      "loss": 4.7425,
+      "step": 53000
+    },
+    {
+      "epoch": 14.69,
+      "learning_rate": 2.0444444444444446e-05,
+      "loss": 4.7301,
+      "step": 54000
+    },
+    {
+      "epoch": 14.96,
+      "learning_rate": 2e-05,
+      "loss": 4.7283,
+      "step": 55000
+    },
+    {
+      "epoch": 15.23,
+      "learning_rate": 1.9555555555555557e-05,
+      "loss": 4.7201,
+      "step": 56000
+    },
+    {
+      "epoch": 15.5,
+      "learning_rate": 1.9111111111111113e-05,
+      "loss": 4.7155,
+      "step": 57000
+    },
+    {
+      "epoch": 15.77,
+      "learning_rate": 1.866666666666667e-05,
+      "loss": 4.7073,
+      "step": 58000
+    },
+    {
+      "epoch": 16.05,
+      "learning_rate": 1.8222222222222224e-05,
+      "loss": 4.7043,
+      "step": 59000
+    },
+    {
+      "epoch": 16.32,
+      "learning_rate": 1.7777777777777777e-05,
+      "loss": 4.6977,
+      "step": 60000
+    },
+    {
+      "epoch": 16.59,
+      "learning_rate": 1.7333333333333336e-05,
+      "loss": 4.6925,
+      "step": 61000
+    },
+    {
+      "epoch": 16.86,
+      "learning_rate": 1.688888888888889e-05,
+      "loss": 4.6867,
+      "step": 62000
+    },
+    {
+      "epoch": 17.13,
+      "learning_rate": 1.6444444444444444e-05,
+      "loss": 4.6806,
+      "step": 63000
+    },
+    {
+      "epoch": 17.41,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 4.6754,
+      "step": 64000
+    },
+    {
+      "epoch": 17.68,
+      "learning_rate": 1.555555555555556e-05,
+      "loss": 4.6791,
+      "step": 65000
+    },
+    {
+      "epoch": 17.95,
+      "learning_rate": 1.5111111111111112e-05,
+      "loss": 4.6689,
+      "step": 66000
+    },
+    {
+      "epoch": 18.22,
+      "learning_rate": 1.4666666666666666e-05,
+      "loss": 4.6677,
+      "step": 67000
+    },
+    {
+      "epoch": 18.49,
+      "learning_rate": 1.4222222222222224e-05,
+      "loss": 4.6618,
+      "step": 68000
+    },
+    {
+      "epoch": 18.77,
+      "learning_rate": 1.377777777777778e-05,
+      "loss": 4.6565,
+      "step": 69000
+    },
+    {
+      "epoch": 19.04,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 4.6544,
+      "step": 70000
+    },
+    {
+      "epoch": 19.31,
+      "learning_rate": 1.288888888888889e-05,
+      "loss": 4.6488,
+      "step": 71000
+    },
+    {
+      "epoch": 19.58,
+      "learning_rate": 1.2444444444444446e-05,
+      "loss": 4.6467,
+      "step": 72000
+    },
+    {
+      "epoch": 19.85,
+      "learning_rate": 1.2e-05,
+      "loss": 4.6478,
+      "step": 73000
+    },
+    {
+      "epoch": 20.13,
+      "learning_rate": 1.1555555555555556e-05,
+      "loss": 4.6429,
+      "step": 74000
+    },
+    {
+      "epoch": 20.4,
+      "learning_rate": 1.1111111111111113e-05,
+      "loss": 4.6436,
+      "step": 75000
+    },
+    {
+      "epoch": 20.67,
+      "learning_rate": 1.0666666666666667e-05,
+      "loss": 4.6352,
+      "step": 76000
+    },
+    {
+      "epoch": 20.94,
+      "learning_rate": 1.0222222222222223e-05,
+      "loss": 4.6338,
+      "step": 77000
+    },
+    {
+      "epoch": 21.21,
+      "learning_rate": 9.777777777777779e-06,
+      "loss": 4.6292,
+      "step": 78000
+    },
+    {
+      "epoch": 21.48,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 4.6305,
+      "step": 79000
+    },
+    {
+      "epoch": 21.76,
+      "learning_rate": 8.888888888888888e-06,
+      "loss": 4.6269,
+      "step": 80000
+    },
+    {
+      "epoch": 22.03,
+      "learning_rate": 8.444444444444446e-06,
+      "loss": 4.6256,
+      "step": 81000
+    },
+    {
+      "epoch": 22.3,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 4.6254,
+      "step": 82000
+    },
+    {
+      "epoch": 22.57,
+      "learning_rate": 7.555555555555556e-06,
+      "loss": 4.6209,
+      "step": 83000
+    },
+    {
+      "epoch": 22.84,
+      "learning_rate": 7.111111111111112e-06,
+      "loss": 4.6196,
+      "step": 84000
+    },
+    {
+      "epoch": 23.12,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 4.6162,
+      "step": 85000
+    },
+    {
+      "epoch": 23.39,
+      "learning_rate": 6.222222222222223e-06,
+      "loss": 4.6128,
+      "step": 86000
+    },
+    {
+      "epoch": 23.66,
+      "learning_rate": 5.777777777777778e-06,
+      "loss": 4.6184,
+      "step": 87000
+    },
+    {
+      "epoch": 23.93,
+      "learning_rate": 5.333333333333334e-06,
+      "loss": 4.6133,
+      "step": 88000
+    },
+    {
+      "epoch": 24.2,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 4.6098,
+      "step": 89000
+    },
+    {
+      "epoch": 24.48,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 4.6101,
+      "step": 90000
+    },
+    {
+      "epoch": 24.75,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 4.6099,
+      "step": 91000
+    },
+    {
+      "epoch": 25.02,
+      "learning_rate": 3.555555555555556e-06,
+      "loss": 4.6092,
+      "step": 92000
+    },
+    {
+      "epoch": 25.29,
+      "learning_rate": 3.1111111111111116e-06,
+      "loss": 4.6061,
+      "step": 93000
+    },
+    {
+      "epoch": 25.56,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 4.6072,
+      "step": 94000
+    },
+    {
+      "epoch": 25.84,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 4.6041,
+      "step": 95000
+    },
+    {
+      "epoch": 26.11,
+      "learning_rate": 1.777777777777778e-06,
+      "loss": 4.6047,
+      "step": 96000
+    },
+    {
+      "epoch": 26.38,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 4.6079,
+      "step": 97000
+    },
+    {
+      "epoch": 26.65,
+      "learning_rate": 8.88888888888889e-07,
+      "loss": 4.6021,
+      "step": 98000
+    },
+    {
+      "epoch": 26.92,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 4.6046,
+      "step": 99000
+    },
+    {
+      "epoch": 27.2,
+      "learning_rate": 0.0,
+      "loss": 4.6024,
+      "step": 100000
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 100000,
+  "num_train_epochs": 28,
+  "save_steps": 1000,
+  "total_flos": 7.577490197651128e+18,
+  "trial_name": null,
+  "trial_params": null
+}

model_ablation_no_length/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a92a5186604d956d275ed158205ded7a647f0e4efa7ab68d02a9ada345e3d97e
+size 5265

model_ablation_no_partition/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

model_ablation_no_partition/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 5120
+}

model_ablation_no_partition/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "max_length": 512,
+  "pad_token_id": 0,
+  "transformers_version": "4.35.2"
+}

model_ablation_no_partition/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6677fb2024ace21bf31d9cbd95a9fa578900f93b8536cc6392accdc9286758ae
+size 359944256

model_ablation_no_partition/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:769b8cf35fbbb7ee3dedb845b68db80a829abb981fbacdfc00ef5682160dad3c
+size 720010699

model_ablation_no_partition/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a6929480e8da34ff65d3764cc7933f69943b6db1fa0ac74e54af2b7e389043c
+size 14709

model_ablation_no_partition/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3018e461ce1631f9bf04479faa74b8f3d9fd86bf68d2cd93702db04c8a02131
+size 1465

model_ablation_no_partition/trainer_state.json

@@ -0,0 +1,619 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 30.257186081694403,
+  "eval_steps": 500,
+  "global_step": 100000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 7.7084,
+      "step": 1000
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 7.1347,
+      "step": 2000
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2e-05,
+      "loss": 6.9286,
+      "step": 3000
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 6.8404,
+      "step": 4000
+    },
+    {
+      "epoch": 1.51,
+      "learning_rate": 2e-05,
+      "loss": 6.811,
+      "step": 5000
+    },
+    {
+      "epoch": 1.82,
+      "learning_rate": 2.4e-05,
+      "loss": 6.7884,
+      "step": 6000
+    },
+    {
+      "epoch": 2.12,
+      "learning_rate": 2.8e-05,
+      "loss": 6.7671,
+      "step": 7000
+    },
+    {
+      "epoch": 2.42,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 6.754,
+      "step": 8000
+    },
+    {
+      "epoch": 2.72,
+      "learning_rate": 3.6e-05,
+      "loss": 6.7459,
+      "step": 9000
+    },
+    {
+      "epoch": 3.03,
+      "learning_rate": 4e-05,
+      "loss": 6.7364,
+      "step": 10000
+    },
+    {
+      "epoch": 3.33,
+      "learning_rate": 3.955555555555556e-05,
+      "loss": 6.7283,
+      "step": 11000
+    },
+    {
+      "epoch": 3.63,
+      "learning_rate": 3.9111111111111115e-05,
+      "loss": 6.7226,
+      "step": 12000
+    },
+    {
+      "epoch": 3.93,
+      "learning_rate": 3.866666666666667e-05,
+      "loss": 6.7186,
+      "step": 13000
+    },
+    {
+      "epoch": 4.24,
+      "learning_rate": 3.8222222222222226e-05,
+      "loss": 6.715,
+      "step": 14000
+    },
+    {
+      "epoch": 4.54,
+      "learning_rate": 3.777777777777778e-05,
+      "loss": 6.7114,
+      "step": 15000
+    },
+    {
+      "epoch": 4.84,
+      "learning_rate": 3.733333333333334e-05,
+      "loss": 6.7088,
+      "step": 16000
+    },
+    {
+      "epoch": 5.14,
+      "learning_rate": 3.6888888888888896e-05,
+      "loss": 6.7072,
+      "step": 17000
+    },
+    {
+      "epoch": 5.45,
+      "learning_rate": 3.644444444444445e-05,
+      "loss": 6.7059,
+      "step": 18000
+    },
+    {
+      "epoch": 5.75,
+      "learning_rate": 3.6e-05,
+      "loss": 6.7039,
+      "step": 19000
+    },
+    {
+      "epoch": 6.05,
+      "learning_rate": 3.555555555555555e-05,
+      "loss": 6.7023,
+      "step": 20000
+    },
+    {
+      "epoch": 6.35,
+      "learning_rate": 3.511111111111111e-05,
+      "loss": 6.7002,
+      "step": 21000
+    },
+    {
+      "epoch": 6.66,
+      "learning_rate": 3.466666666666667e-05,
+      "loss": 6.698,
+      "step": 22000
+    },
+    {
+      "epoch": 6.96,
+      "learning_rate": 3.4222222222222224e-05,
+      "loss": 6.6975,
+      "step": 23000
+    },
+    {
+      "epoch": 7.26,
+      "learning_rate": 3.377777777777778e-05,
+      "loss": 6.6959,
+      "step": 24000
+    },
+    {
+      "epoch": 7.56,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 6.6929,
+      "step": 25000
+    },
+    {
+      "epoch": 7.87,
+      "learning_rate": 3.288888888888889e-05,
+      "loss": 6.6936,
+      "step": 26000
+    },
+    {
+      "epoch": 8.17,
+      "learning_rate": 3.2444444444444446e-05,
+      "loss": 6.6917,
+      "step": 27000
+    },
+    {
+      "epoch": 8.47,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 6.6915,
+      "step": 28000
+    },
+    {
+      "epoch": 8.77,
+      "learning_rate": 3.155555555555556e-05,
+      "loss": 6.69,
+      "step": 29000
+    },
+    {
+      "epoch": 9.08,
+      "learning_rate": 3.111111111111112e-05,
+      "loss": 6.6884,
+      "step": 30000
+    },
+    {
+      "epoch": 9.38,
+      "learning_rate": 3.066666666666667e-05,
+      "loss": 6.6864,
+      "step": 31000
+    },
+    {
+      "epoch": 19.37,
+      "learning_rate": 3.0222222222222225e-05,
+      "loss": 6.6229,
+      "step": 32000
+    },
+    {
+      "epoch": 19.97,
+      "learning_rate": 2.977777777777778e-05,
+      "loss": 6.5542,
+      "step": 33000
+    },
+    {
+      "epoch": 20.58,
+      "learning_rate": 2.9333333333333333e-05,
+      "loss": 6.5168,
+      "step": 34000
+    },
+    {
+      "epoch": 21.19,
+      "learning_rate": 2.888888888888889e-05,
+      "loss": 6.4593,
+      "step": 35000
+    },
+    {
+      "epoch": 21.79,
+      "learning_rate": 2.8444444444444447e-05,
+      "loss": 6.3693,
+      "step": 36000
+    },
+    {
+      "epoch": 22.4,
+      "learning_rate": 2.8e-05,
+      "loss": 6.2642,
+      "step": 37000
+    },
+    {
+      "epoch": 23.0,
+      "learning_rate": 2.755555555555556e-05,
+      "loss": 6.151,
+      "step": 38000
+    },
+    {
+      "epoch": 23.61,
+      "learning_rate": 2.7111111111111114e-05,
+      "loss": 6.0063,
+      "step": 39000
+    },
+    {
+      "epoch": 24.21,
+      "learning_rate": 2.6666666666666667e-05,
+      "loss": 5.9027,
+      "step": 40000
+    },
+    {
+      "epoch": 24.82,
+      "learning_rate": 2.6222222222222226e-05,
+      "loss": 5.826,
+      "step": 41000
+    },
+    {
+      "epoch": 25.42,
+      "learning_rate": 2.577777777777778e-05,
+      "loss": 5.7664,
+      "step": 42000
+    },
+    {
+      "epoch": 26.03,
+      "learning_rate": 2.5333333333333334e-05,
+      "loss": 5.7222,
+      "step": 43000
+    },
+    {
+      "epoch": 26.63,
+      "learning_rate": 2.4888888888888893e-05,
+      "loss": 5.6801,
+      "step": 44000
+    },
+    {
+      "epoch": 27.24,
+      "learning_rate": 2.444444444444445e-05,
+      "loss": 5.6484,
+      "step": 45000
+    },
+    {
+      "epoch": 27.84,
+      "learning_rate": 2.4e-05,
+      "loss": 5.6185,
+      "step": 46000
+    },
+    {
+      "epoch": 28.45,
+      "learning_rate": 2.355555555555556e-05,
+      "loss": 5.5924,
+      "step": 47000
+    },
+    {
+      "epoch": 29.05,
+      "learning_rate": 2.3111111111111112e-05,
+      "loss": 5.5701,
+      "step": 48000
+    },
+    {
+      "epoch": 29.66,
+      "learning_rate": 2.2666666666666668e-05,
+      "loss": 5.5486,
+      "step": 49000
+    },
+    {
+      "epoch": 30.26,
+      "learning_rate": 2.2222222222222227e-05,
+      "loss": 5.5285,
+      "step": 50000
+    },
+    {
+      "epoch": 30.87,
+      "learning_rate": 2.177777777777778e-05,
+      "loss": 5.5128,
+      "step": 51000
+    },
+    {
+      "epoch": 31.47,
+      "learning_rate": 2.1333333333333335e-05,
+      "loss": 5.4946,
+      "step": 52000
+    },
+    {
+      "epoch": 32.08,
+      "learning_rate": 2.088888888888889e-05,
+      "loss": 5.4802,
+      "step": 53000
+    },
+    {
+      "epoch": 32.68,
+      "learning_rate": 2.0444444444444446e-05,
+      "loss": 5.4654,
+      "step": 54000
+    },
+    {
+      "epoch": 33.29,
+      "learning_rate": 2e-05,
+      "loss": 5.4553,
+      "step": 55000
+    },
+    {
+      "epoch": 16.94,
+      "learning_rate": 1.9555555555555557e-05,
+      "loss": 5.4459,
+      "step": 56000
+    },
+    {
+      "epoch": 17.25,
+      "learning_rate": 1.9111111111111113e-05,
+      "loss": 5.4354,
+      "step": 57000
+    },
+    {
+      "epoch": 17.55,
+      "learning_rate": 1.866666666666667e-05,
+      "loss": 5.4323,
+      "step": 58000
+    },
+    {
+      "epoch": 17.85,
+      "learning_rate": 1.8222222222222224e-05,
+      "loss": 5.4245,
+      "step": 59000
+    },
+    {
+      "epoch": 18.15,
+      "learning_rate": 1.7777777777777777e-05,
+      "loss": 5.4162,
+      "step": 60000
+    },
+    {
+      "epoch": 18.46,
+      "learning_rate": 1.7333333333333336e-05,
+      "loss": 5.4092,
+      "step": 61000
+    },
+    {
+      "epoch": 18.76,
+      "learning_rate": 1.688888888888889e-05,
+      "loss": 5.402,
+      "step": 62000
+    },
+    {
+      "epoch": 19.06,
+      "learning_rate": 1.6444444444444444e-05,
+      "loss": 5.3934,
+      "step": 63000
+    },
+    {
+      "epoch": 19.36,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 5.392,
+      "step": 64000
+    },
+    {
+      "epoch": 19.67,
+      "learning_rate": 1.555555555555556e-05,
+      "loss": 5.3835,
+      "step": 65000
+    },
+    {
+      "epoch": 19.97,
+      "learning_rate": 1.5111111111111112e-05,
+      "loss": 5.3828,
+      "step": 66000
+    },
+    {
+      "epoch": 20.27,
+      "learning_rate": 1.4666666666666666e-05,
+      "loss": 5.377,
+      "step": 67000
+    },
+    {
+      "epoch": 20.57,
+      "learning_rate": 1.4222222222222224e-05,
+      "loss": 5.3695,
+      "step": 68000
+    },
+    {
+      "epoch": 20.88,
+      "learning_rate": 1.377777777777778e-05,
+      "loss": 5.3634,
+      "step": 69000
+    },
+    {
+      "epoch": 21.18,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 5.358,
+      "step": 70000
+    },
+    {
+      "epoch": 21.48,
+      "learning_rate": 1.288888888888889e-05,
+      "loss": 5.3554,
+      "step": 71000
+    },
+    {
+      "epoch": 21.79,
+      "learning_rate": 1.2444444444444446e-05,
+      "loss": 5.3537,
+      "step": 72000
+    },
+    {
+      "epoch": 22.09,
+      "learning_rate": 1.2e-05,
+      "loss": 5.3486,
+      "step": 73000
+    },
+    {
+      "epoch": 22.39,
+      "learning_rate": 1.1555555555555556e-05,
+      "loss": 5.3431,
+      "step": 74000
+    },
+    {
+      "epoch": 22.69,
+      "learning_rate": 1.1111111111111113e-05,
+      "loss": 5.3404,
+      "step": 75000
+    },
+    {
+      "epoch": 23.0,
+      "learning_rate": 1.0666666666666667e-05,
+      "loss": 5.3377,
+      "step": 76000
+    },
+    {
+      "epoch": 23.3,
+      "learning_rate": 1.0222222222222223e-05,
+      "loss": 5.3317,
+      "step": 77000
+    },
+    {
+      "epoch": 23.6,
+      "learning_rate": 9.777777777777779e-06,
+      "loss": 5.3319,
+      "step": 78000
+    },
+    {
+      "epoch": 23.9,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 5.3299,
+      "step": 79000
+    },
+    {
+      "epoch": 24.21,
+      "learning_rate": 8.888888888888888e-06,
+      "loss": 5.3232,
+      "step": 80000
+    },
+    {
+      "epoch": 24.51,
+      "learning_rate": 8.444444444444446e-06,
+      "loss": 5.3249,
+      "step": 81000
+    },
+    {
+      "epoch": 24.81,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 5.3187,
+      "step": 82000
+    },
+    {
+      "epoch": 25.11,
+      "learning_rate": 7.555555555555556e-06,
+      "loss": 5.3174,
+      "step": 83000
+    },
+    {
+      "epoch": 25.42,
+      "learning_rate": 7.111111111111112e-06,
+      "loss": 5.3139,
+      "step": 84000
+    },
+    {
+      "epoch": 25.72,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 5.3119,
+      "step": 85000
+    },
+    {
+      "epoch": 26.02,
+      "learning_rate": 6.222222222222223e-06,
+      "loss": 5.3143,
+      "step": 86000
+    },
+    {
+      "epoch": 26.32,
+      "learning_rate": 5.777777777777778e-06,
+      "loss": 5.3063,
+      "step": 87000
+    },
+    {
+      "epoch": 26.63,
+      "learning_rate": 5.333333333333334e-06,
+      "loss": 5.3063,
+      "step": 88000
+    },
+    {
+      "epoch": 26.93,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 5.3096,
+      "step": 89000
+    },
+    {
+      "epoch": 27.23,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 5.2996,
+      "step": 90000
+    },
+    {
+      "epoch": 27.53,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 5.3011,
+      "step": 91000
+    },
+    {
+      "epoch": 27.84,
+      "learning_rate": 3.555555555555556e-06,
+      "loss": 5.3027,
+      "step": 92000
+    },
+    {
+      "epoch": 28.14,
+      "learning_rate": 3.1111111111111116e-06,
+      "loss": 5.3007,
+      "step": 93000
+    },
+    {
+      "epoch": 28.44,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 5.297,
+      "step": 94000
+    },
+    {
+      "epoch": 28.74,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 5.2979,
+      "step": 95000
+    },
+    {
+      "epoch": 29.05,
+      "learning_rate": 1.777777777777778e-06,
+      "loss": 5.2982,
+      "step": 96000
+    },
+    {
+      "epoch": 29.35,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 5.2977,
+      "step": 97000
+    },
+    {
+      "epoch": 29.65,
+      "learning_rate": 8.88888888888889e-07,
+      "loss": 5.2955,
+      "step": 98000
+    },
+    {
+      "epoch": 29.95,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 5.2938,
+      "step": 99000
+    },
+    {
+      "epoch": 30.26,
+      "learning_rate": 0.0,
+      "loss": 5.2937,
+      "step": 100000
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 100000,
+  "num_train_epochs": 31,
+  "save_steps": 1000,
+  "total_flos": 9.396281398647587e+18,
+  "trial_name": null,
+  "trial_params": null
+}

model_ablation_no_partition/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fedbea814b8e0698bec087c4313eaab91fc0ee4ef38d1f1581968325185357b
+size 5265