Training in progress, step 1000, checkpoint

last-checkpoint/config.json

@@ -0,0 +1,47 @@
+{
+  "_name_or_path": "CocoRoF/KoModernBERT-large-mlm-v3",
+  "architectures": [
+    "ModernBertForMaskedLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 50281,
+  "classifier_activation": "gelu",
+  "classifier_bias": false,
+  "classifier_dropout": 0.0,
+  "classifier_pooling": "mean",
+  "cls_token_id": 50281,
+  "decoder_bias": true,
+  "deterministic_flash_attn": false,
+  "embedding_dropout": 0.0,
+  "eos_token_id": 50282,
+  "global_attn_every_n_layers": 3,
+  "global_rope_theta": 160000.0,
+  "gradient_checkpointing": false,
+  "hidden_activation": "gelu",
+  "hidden_size": 1024,
+  "initializer_cutoff_factor": 2.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 2624,
+  "layer_norm_eps": 1e-05,
+  "local_attention": 128,
+  "local_rope_theta": 10000.0,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mlp_dropout": 0.0,
+  "model_type": "modernbert",
+  "norm_bias": false,
+  "norm_eps": 1e-05,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "pad_token_id": 50283,
+  "position_embedding_type": "absolute",
+  "reference_compile": false,
+  "repad_logits_with_grad": false,
+  "sep_token_id": 50282,
+  "sparse_pred_ignore_index": -100,
+  "sparse_prediction": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.49.0",
+  "vocab_size": 50411
+}

last-checkpoint/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c71a7729ac4a2d1a47e86393840dc65c758773a0098e1237ae0e7077cfb8ae0e
+size 791869518

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f7191baea0d93efcc055c1b3e98f679be63211ee15b418d2aa66ffeebc8979e
+size 2375752250

last-checkpoint/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74386f26f36ed67f56395205881e5db2d0c28ffcbeed50dd95b28771d2dac588
+size 15984

last-checkpoint/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41c88f9de084200454883a13c3717941ea3fd433e2f8735507fc30611f9c5501
+size 15984

last-checkpoint/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:965b00d4cb4710ebab57c8787b9925bb3f77b8eeba94a186ec4bc1c2f326ef3f
+size 15984

last-checkpoint/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5dc374b8b9a4c45c950f9d136feab85a767081fa59f0c7d68ed3a62060c4949
+size 15984

last-checkpoint/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c7c212fb779217f1edac0baf44f67b608eefc1e0e4e3f5a9dd7eb557032c1bc
+size 15984

last-checkpoint/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86e1effd626ce1e95dd68a0c8089fe19218f2b24dfe9e45ed2cab1c0ebc10ba1
+size 15984

last-checkpoint/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:799cc83f60dfc1c4243cfd6403592112414a2eec494e6832f10221c96ff62c20
+size 15984

last-checkpoint/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:586777c398770c3255d3a1f48c7fef44ea9d89117c627c9ea490e16bfd9a49ba
+size 15984

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e74977310703748551c5dc1da92b0842afc2c410245fb89bf45d068c870d5fe6
+size 1000

last-checkpoint/special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

last-checkpoint/tokenizer_config.json

@@ -0,0 +1,945 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50278": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50279": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50280": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50281": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50282": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50283": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50284": {
+      "content": "[MASK]",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50328": {
+      "content": "[unused0]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50329": {
+      "content": "[unused1]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50330": {
+      "content": "[unused2]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50331": {
+      "content": "[unused3]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50332": {
+      "content": "[unused4]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50333": {
+      "content": "[unused5]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50334": {
+      "content": "[unused6]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50335": {
+      "content": "[unused7]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50336": {
+      "content": "[unused8]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50337": {
+      "content": "[unused9]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50338": {
+      "content": "[unused10]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50339": {
+      "content": "[unused11]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50340": {
+      "content": "[unused12]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50341": {
+      "content": "[unused13]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50342": {
+      "content": "[unused14]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50343": {
+      "content": "[unused15]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50344": {
+      "content": "[unused16]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50345": {
+      "content": "[unused17]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50346": {
+      "content": "[unused18]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50347": {
+      "content": "[unused19]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50348": {
+      "content": "[unused20]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50349": {
+      "content": "[unused21]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50350": {
+      "content": "[unused22]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50351": {
+      "content": "[unused23]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50352": {
+      "content": "[unused24]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50353": {
+      "content": "[unused25]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50354": {
+      "content": "[unused26]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50355": {
+      "content": "[unused27]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50356": {
+      "content": "[unused28]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50357": {
+      "content": "[unused29]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50358": {
+      "content": "[unused30]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50359": {
+      "content": "[unused31]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50360": {
+      "content": "[unused32]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50361": {
+      "content": "[unused33]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50362": {
+      "content": "[unused34]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50363": {
+      "content": "[unused35]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50364": {
+      "content": "[unused36]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50365": {
+      "content": "[unused37]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50366": {
+      "content": "[unused38]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50367": {
+      "content": "[unused39]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50368": {
+      "content": "[unused40]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50369": {
+      "content": "[unused41]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50370": {
+      "content": "[unused42]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50371": {
+      "content": "[unused43]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50372": {
+      "content": "[unused44]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50373": {
+      "content": "[unused45]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50374": {
+      "content": "[unused46]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50375": {
+      "content": "[unused47]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50376": {
+      "content": "[unused48]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50377": {
+      "content": "[unused49]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50378": {
+      "content": "[unused50]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50379": {
+      "content": "[unused51]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50380": {
+      "content": "[unused52]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50381": {
+      "content": "[unused53]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50382": {
+      "content": "[unused54]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50383": {
+      "content": "[unused55]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50384": {
+      "content": "[unused56]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50385": {
+      "content": "[unused57]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50386": {
+      "content": "[unused58]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50387": {
+      "content": "[unused59]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50388": {
+      "content": "[unused60]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50389": {
+      "content": "[unused61]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50390": {
+      "content": "[unused62]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50391": {
+      "content": "[unused63]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50392": {
+      "content": "[unused64]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50393": {
+      "content": "[unused65]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50394": {
+      "content": "[unused66]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50395": {
+      "content": "[unused67]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50396": {
+      "content": "[unused68]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50397": {
+      "content": "[unused69]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50398": {
+      "content": "[unused70]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50399": {
+      "content": "[unused71]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50400": {
+      "content": "[unused72]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50401": {
+      "content": "[unused73]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50402": {
+      "content": "[unused74]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50403": {
+      "content": "[unused75]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50404": {
+      "content": "[unused76]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50405": {
+      "content": "[unused77]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50406": {
+      "content": "[unused78]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50407": {
+      "content": "[unused79]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50408": {
+      "content": "[unused80]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50409": {
+      "content": "[unused81]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50410": {
+      "content": "[unused82]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 8192,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizer",
+  "unk_token": "[UNK]"
+}

last-checkpoint/trainer_state.json

@@ -0,0 +1,1449 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.2815860333327467,
+  "eval_steps": 500,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0014079301666637335,
+      "grad_norm": 114.625,
+      "learning_rate": 2.8089887640449436e-08,
+      "loss": 85.8853,
+      "step": 5
+    },
+    {
+      "epoch": 0.002815860333327467,
+      "grad_norm": 111.6875,
+      "learning_rate": 5.617977528089887e-08,
+      "loss": 85.9658,
+      "step": 10
+    },
+    {
+      "epoch": 0.0042237904999912,
+      "grad_norm": 112.0625,
+      "learning_rate": 8.426966292134831e-08,
+      "loss": 84.4067,
+      "step": 15
+    },
+    {
+      "epoch": 0.005631720666654934,
+      "grad_norm": 114.375,
+      "learning_rate": 1.1235955056179774e-07,
+      "loss": 85.7776,
+      "step": 20
+    },
+    {
+      "epoch": 0.007039650833318667,
+      "grad_norm": 110.25,
+      "learning_rate": 1.4044943820224718e-07,
+      "loss": 83.5889,
+      "step": 25
+    },
+    {
+      "epoch": 0.0084475809999824,
+      "grad_norm": 112.375,
+      "learning_rate": 1.6853932584269663e-07,
+      "loss": 84.3387,
+      "step": 30
+    },
+    {
+      "epoch": 0.009855511166646135,
+      "grad_norm": 109.5625,
+      "learning_rate": 1.9662921348314607e-07,
+      "loss": 84.4387,
+      "step": 35
+    },
+    {
+      "epoch": 0.011263441333309868,
+      "grad_norm": 114.8125,
+      "learning_rate": 2.2471910112359549e-07,
+      "loss": 83.7717,
+      "step": 40
+    },
+    {
+      "epoch": 0.012671371499973601,
+      "grad_norm": 110.875,
+      "learning_rate": 2.5280898876404493e-07,
+      "loss": 83.5306,
+      "step": 45
+    },
+    {
+      "epoch": 0.014079301666637335,
+      "grad_norm": 106.3125,
+      "learning_rate": 2.8089887640449437e-07,
+      "loss": 83.6832,
+      "step": 50
+    },
+    {
+      "epoch": 0.015487231833301068,
+      "grad_norm": 112.25,
+      "learning_rate": 3.0898876404494376e-07,
+      "loss": 83.6929,
+      "step": 55
+    },
+    {
+      "epoch": 0.0168951619999648,
+      "grad_norm": 111.8125,
+      "learning_rate": 3.3707865168539325e-07,
+      "loss": 84.1018,
+      "step": 60
+    },
+    {
+      "epoch": 0.018303092166628534,
+      "grad_norm": 109.3125,
+      "learning_rate": 3.651685393258427e-07,
+      "loss": 84.1149,
+      "step": 65
+    },
+    {
+      "epoch": 0.01971102233329227,
+      "grad_norm": 109.875,
+      "learning_rate": 3.9325842696629214e-07,
+      "loss": 82.8841,
+      "step": 70
+    },
+    {
+      "epoch": 0.021118952499956004,
+      "grad_norm": 107.9375,
+      "learning_rate": 4.2134831460674153e-07,
+      "loss": 84.9085,
+      "step": 75
+    },
+    {
+      "epoch": 0.022526882666619737,
+      "grad_norm": 113.875,
+      "learning_rate": 4.4943820224719097e-07,
+      "loss": 83.177,
+      "step": 80
+    },
+    {
+      "epoch": 0.02393481283328347,
+      "grad_norm": 112.5625,
+      "learning_rate": 4.775280898876405e-07,
+      "loss": 84.905,
+      "step": 85
+    },
+    {
+      "epoch": 0.025342742999947203,
+      "grad_norm": 111.125,
+      "learning_rate": 5.056179775280899e-07,
+      "loss": 83.7053,
+      "step": 90
+    },
+    {
+      "epoch": 0.026750673166610936,
+      "grad_norm": 110.3125,
+      "learning_rate": 5.337078651685392e-07,
+      "loss": 84.249,
+      "step": 95
+    },
+    {
+      "epoch": 0.02815860333327467,
+      "grad_norm": 115.0,
+      "learning_rate": 5.617977528089887e-07,
+      "loss": 83.1926,
+      "step": 100
+    },
+    {
+      "epoch": 0.029566533499938402,
+      "grad_norm": 113.25,
+      "learning_rate": 5.898876404494381e-07,
+      "loss": 84.164,
+      "step": 105
+    },
+    {
+      "epoch": 0.030974463666602135,
+      "grad_norm": 112.9375,
+      "learning_rate": 6.179775280898875e-07,
+      "loss": 83.7127,
+      "step": 110
+    },
+    {
+      "epoch": 0.03238239383326587,
+      "grad_norm": 114.5,
+      "learning_rate": 6.460674157303371e-07,
+      "loss": 85.8182,
+      "step": 115
+    },
+    {
+      "epoch": 0.0337903239999296,
+      "grad_norm": 114.125,
+      "learning_rate": 6.741573033707865e-07,
+      "loss": 86.1688,
+      "step": 120
+    },
+    {
+      "epoch": 0.035198254166593335,
+      "grad_norm": 108.6875,
+      "learning_rate": 7.02247191011236e-07,
+      "loss": 83.0914,
+      "step": 125
+    },
+    {
+      "epoch": 0.03660618433325707,
+      "grad_norm": 109.75,
+      "learning_rate": 7.303370786516854e-07,
+      "loss": 86.4069,
+      "step": 130
+    },
+    {
+      "epoch": 0.0380141144999208,
+      "grad_norm": 111.8125,
+      "learning_rate": 7.584269662921348e-07,
+      "loss": 85.471,
+      "step": 135
+    },
+    {
+      "epoch": 0.03942204466658454,
+      "grad_norm": 109.75,
+      "learning_rate": 7.865168539325843e-07,
+      "loss": 84.2073,
+      "step": 140
+    },
+    {
+      "epoch": 0.040829974833248274,
+      "grad_norm": 112.6875,
+      "learning_rate": 8.146067415730337e-07,
+      "loss": 85.0642,
+      "step": 145
+    },
+    {
+      "epoch": 0.04223790499991201,
+      "grad_norm": 112.5,
+      "learning_rate": 8.426966292134831e-07,
+      "loss": 86.0327,
+      "step": 150
+    },
+    {
+      "epoch": 0.04364583516657574,
+      "grad_norm": 111.625,
+      "learning_rate": 8.707865168539326e-07,
+      "loss": 86.5769,
+      "step": 155
+    },
+    {
+      "epoch": 0.04505376533323947,
+      "grad_norm": 110.1875,
+      "learning_rate": 8.988764044943819e-07,
+      "loss": 87.2292,
+      "step": 160
+    },
+    {
+      "epoch": 0.046461695499903206,
+      "grad_norm": 108.4375,
+      "learning_rate": 9.269662921348314e-07,
+      "loss": 87.8398,
+      "step": 165
+    },
+    {
+      "epoch": 0.04786962566656694,
+      "grad_norm": 107.5,
+      "learning_rate": 9.55056179775281e-07,
+      "loss": 87.6348,
+      "step": 170
+    },
+    {
+      "epoch": 0.04927755583323067,
+      "grad_norm": 115.625,
+      "learning_rate": 9.831460674157302e-07,
+      "loss": 87.7073,
+      "step": 175
+    },
+    {
+      "epoch": 0.050685485999894406,
+      "grad_norm": 115.125,
+      "learning_rate": 1.0112359550561797e-06,
+      "loss": 86.4599,
+      "step": 180
+    },
+    {
+      "epoch": 0.05209341616655814,
+      "grad_norm": 109.625,
+      "learning_rate": 1.0393258426966292e-06,
+      "loss": 85.6867,
+      "step": 185
+    },
+    {
+      "epoch": 0.05350134633322187,
+      "grad_norm": 111.1875,
+      "learning_rate": 1.0674157303370785e-06,
+      "loss": 88.4886,
+      "step": 190
+    },
+    {
+      "epoch": 0.054909276499885605,
+      "grad_norm": 113.6875,
+      "learning_rate": 1.095505617977528e-06,
+      "loss": 89.7883,
+      "step": 195
+    },
+    {
+      "epoch": 0.05631720666654934,
+      "grad_norm": 112.1875,
+      "learning_rate": 1.1235955056179775e-06,
+      "loss": 86.8666,
+      "step": 200
+    },
+    {
+      "epoch": 0.05772513683321307,
+      "grad_norm": 113.875,
+      "learning_rate": 1.151685393258427e-06,
+      "loss": 88.1594,
+      "step": 205
+    },
+    {
+      "epoch": 0.059133066999876804,
+      "grad_norm": 112.5625,
+      "learning_rate": 1.1797752808988763e-06,
+      "loss": 87.9299,
+      "step": 210
+    },
+    {
+      "epoch": 0.06054099716654054,
+      "grad_norm": 111.1875,
+      "learning_rate": 1.2078651685393258e-06,
+      "loss": 88.8051,
+      "step": 215
+    },
+    {
+      "epoch": 0.06194892733320427,
+      "grad_norm": 108.125,
+      "learning_rate": 1.235955056179775e-06,
+      "loss": 86.5864,
+      "step": 220
+    },
+    {
+      "epoch": 0.06335685749986801,
+      "grad_norm": 115.5625,
+      "learning_rate": 1.2640449438202247e-06,
+      "loss": 89.1844,
+      "step": 225
+    },
+    {
+      "epoch": 0.06476478766653174,
+      "grad_norm": 112.875,
+      "learning_rate": 1.2921348314606742e-06,
+      "loss": 89.5238,
+      "step": 230
+    },
+    {
+      "epoch": 0.06617271783319548,
+      "grad_norm": 111.1875,
+      "learning_rate": 1.3202247191011235e-06,
+      "loss": 89.9019,
+      "step": 235
+    },
+    {
+      "epoch": 0.0675806479998592,
+      "grad_norm": 114.0625,
+      "learning_rate": 1.348314606741573e-06,
+      "loss": 89.5135,
+      "step": 240
+    },
+    {
+      "epoch": 0.06898857816652294,
+      "grad_norm": 111.9375,
+      "learning_rate": 1.3764044943820223e-06,
+      "loss": 89.6118,
+      "step": 245
+    },
+    {
+      "epoch": 0.07039650833318667,
+      "grad_norm": 113.875,
+      "learning_rate": 1.404494382022472e-06,
+      "loss": 89.3294,
+      "step": 250
+    },
+    {
+      "epoch": 0.07180443849985041,
+      "grad_norm": 108.6875,
+      "learning_rate": 1.4325842696629213e-06,
+      "loss": 91.4349,
+      "step": 255
+    },
+    {
+      "epoch": 0.07321236866651414,
+      "grad_norm": 115.125,
+      "learning_rate": 1.4606741573033708e-06,
+      "loss": 89.6878,
+      "step": 260
+    },
+    {
+      "epoch": 0.07462029883317788,
+      "grad_norm": 111.3125,
+      "learning_rate": 1.48876404494382e-06,
+      "loss": 91.3542,
+      "step": 265
+    },
+    {
+      "epoch": 0.0760282289998416,
+      "grad_norm": 114.0625,
+      "learning_rate": 1.5168539325842696e-06,
+      "loss": 90.9409,
+      "step": 270
+    },
+    {
+      "epoch": 0.07743615916650534,
+      "grad_norm": 110.875,
+      "learning_rate": 1.544943820224719e-06,
+      "loss": 91.2244,
+      "step": 275
+    },
+    {
+      "epoch": 0.07884408933316908,
+      "grad_norm": 114.125,
+      "learning_rate": 1.5730337078651686e-06,
+      "loss": 90.888,
+      "step": 280
+    },
+    {
+      "epoch": 0.08025201949983281,
+      "grad_norm": 113.4375,
+      "learning_rate": 1.6011235955056178e-06,
+      "loss": 91.2134,
+      "step": 285
+    },
+    {
+      "epoch": 0.08165994966649655,
+      "grad_norm": 108.375,
+      "learning_rate": 1.6292134831460673e-06,
+      "loss": 91.87,
+      "step": 290
+    },
+    {
+      "epoch": 0.08306787983316027,
+      "grad_norm": 108.6875,
+      "learning_rate": 1.6573033707865166e-06,
+      "loss": 91.9418,
+      "step": 295
+    },
+    {
+      "epoch": 0.08447580999982401,
+      "grad_norm": 108.1875,
+      "learning_rate": 1.6853932584269661e-06,
+      "loss": 91.6362,
+      "step": 300
+    },
+    {
+      "epoch": 0.08588374016648774,
+      "grad_norm": 114.9375,
+      "learning_rate": 1.7134831460674158e-06,
+      "loss": 92.6335,
+      "step": 305
+    },
+    {
+      "epoch": 0.08729167033315148,
+      "grad_norm": 107.1875,
+      "learning_rate": 1.741573033707865e-06,
+      "loss": 92.5334,
+      "step": 310
+    },
+    {
+      "epoch": 0.0886996004998152,
+      "grad_norm": 114.6875,
+      "learning_rate": 1.7696629213483146e-06,
+      "loss": 91.7963,
+      "step": 315
+    },
+    {
+      "epoch": 0.09010753066647895,
+      "grad_norm": 109.1875,
+      "learning_rate": 1.7977528089887639e-06,
+      "loss": 92.5923,
+      "step": 320
+    },
+    {
+      "epoch": 0.09151546083314267,
+      "grad_norm": 114.9375,
+      "learning_rate": 1.8258426966292134e-06,
+      "loss": 93.9665,
+      "step": 325
+    },
+    {
+      "epoch": 0.09292339099980641,
+      "grad_norm": 118.1875,
+      "learning_rate": 1.8539325842696629e-06,
+      "loss": 91.7156,
+      "step": 330
+    },
+    {
+      "epoch": 0.09433132116647014,
+      "grad_norm": 116.0,
+      "learning_rate": 1.8820224719101124e-06,
+      "loss": 94.3289,
+      "step": 335
+    },
+    {
+      "epoch": 0.09573925133313388,
+      "grad_norm": 116.75,
+      "learning_rate": 1.910112359550562e-06,
+      "loss": 94.187,
+      "step": 340
+    },
+    {
+      "epoch": 0.0971471814997976,
+      "grad_norm": 111.875,
+      "learning_rate": 1.938202247191011e-06,
+      "loss": 93.7581,
+      "step": 345
+    },
+    {
+      "epoch": 0.09855511166646135,
+      "grad_norm": 114.9375,
+      "learning_rate": 1.9662921348314604e-06,
+      "loss": 93.413,
+      "step": 350
+    },
+    {
+      "epoch": 0.09996304183312507,
+      "grad_norm": 112.625,
+      "learning_rate": 1.99438202247191e-06,
+      "loss": 94.2749,
+      "step": 355
+    },
+    {
+      "epoch": 0.10137097199978881,
+      "grad_norm": 107.6875,
+      "learning_rate": 1.9974960876369326e-06,
+      "loss": 94.6595,
+      "step": 360
+    },
+    {
+      "epoch": 0.10277890216645254,
+      "grad_norm": 115.125,
+      "learning_rate": 1.9943661971830985e-06,
+      "loss": 94.8831,
+      "step": 365
+    },
+    {
+      "epoch": 0.10418683233311628,
+      "grad_norm": 111.9375,
+      "learning_rate": 1.9912363067292643e-06,
+      "loss": 94.5983,
+      "step": 370
+    },
+    {
+      "epoch": 0.10559476249978002,
+      "grad_norm": 113.875,
+      "learning_rate": 1.98810641627543e-06,
+      "loss": 96.0132,
+      "step": 375
+    },
+    {
+      "epoch": 0.10700269266644374,
+      "grad_norm": 111.8125,
+      "learning_rate": 1.984976525821596e-06,
+      "loss": 96.9668,
+      "step": 380
+    },
+    {
+      "epoch": 0.10841062283310748,
+      "grad_norm": 109.5625,
+      "learning_rate": 1.981846635367762e-06,
+      "loss": 94.2092,
+      "step": 385
+    },
+    {
+      "epoch": 0.10981855299977121,
+      "grad_norm": 113.6875,
+      "learning_rate": 1.9787167449139278e-06,
+      "loss": 93.9071,
+      "step": 390
+    },
+    {
+      "epoch": 0.11122648316643495,
+      "grad_norm": 113.625,
+      "learning_rate": 1.9755868544600936e-06,
+      "loss": 94.9607,
+      "step": 395
+    },
+    {
+      "epoch": 0.11263441333309868,
+      "grad_norm": 110.75,
+      "learning_rate": 1.9724569640062595e-06,
+      "loss": 92.9988,
+      "step": 400
+    },
+    {
+      "epoch": 0.11404234349976242,
+      "grad_norm": 114.25,
+      "learning_rate": 1.9693270735524258e-06,
+      "loss": 95.0537,
+      "step": 405
+    },
+    {
+      "epoch": 0.11545027366642614,
+      "grad_norm": 110.9375,
+      "learning_rate": 1.9661971830985916e-06,
+      "loss": 94.5083,
+      "step": 410
+    },
+    {
+      "epoch": 0.11685820383308988,
+      "grad_norm": 113.375,
+      "learning_rate": 1.9630672926447575e-06,
+      "loss": 94.8912,
+      "step": 415
+    },
+    {
+      "epoch": 0.11826613399975361,
+      "grad_norm": 112.125,
+      "learning_rate": 1.9599374021909234e-06,
+      "loss": 95.3204,
+      "step": 420
+    },
+    {
+      "epoch": 0.11967406416641735,
+      "grad_norm": 113.625,
+      "learning_rate": 1.9568075117370892e-06,
+      "loss": 95.2065,
+      "step": 425
+    },
+    {
+      "epoch": 0.12108199433308107,
+      "grad_norm": 111.5625,
+      "learning_rate": 1.953677621283255e-06,
+      "loss": 94.5451,
+      "step": 430
+    },
+    {
+      "epoch": 0.12248992449974482,
+      "grad_norm": 108.75,
+      "learning_rate": 1.950547730829421e-06,
+      "loss": 95.1698,
+      "step": 435
+    },
+    {
+      "epoch": 0.12389785466640854,
+      "grad_norm": 107.9375,
+      "learning_rate": 1.947417840375587e-06,
+      "loss": 94.7059,
+      "step": 440
+    },
+    {
+      "epoch": 0.12530578483307228,
+      "grad_norm": 108.875,
+      "learning_rate": 1.9442879499217527e-06,
+      "loss": 94.4242,
+      "step": 445
+    },
+    {
+      "epoch": 0.12671371499973602,
+      "grad_norm": 110.875,
+      "learning_rate": 1.9411580594679185e-06,
+      "loss": 95.4208,
+      "step": 450
+    },
+    {
+      "epoch": 0.12812164516639973,
+      "grad_norm": 112.4375,
+      "learning_rate": 1.9380281690140844e-06,
+      "loss": 96.0422,
+      "step": 455
+    },
+    {
+      "epoch": 0.12952957533306347,
+      "grad_norm": 111.25,
+      "learning_rate": 1.9348982785602502e-06,
+      "loss": 93.9782,
+      "step": 460
+    },
+    {
+      "epoch": 0.1309375054997272,
+      "grad_norm": 112.4375,
+      "learning_rate": 1.931768388106416e-06,
+      "loss": 93.49,
+      "step": 465
+    },
+    {
+      "epoch": 0.13234543566639095,
+      "grad_norm": 111.25,
+      "learning_rate": 1.928638497652582e-06,
+      "loss": 93.6705,
+      "step": 470
+    },
+    {
+      "epoch": 0.1337533658330547,
+      "grad_norm": 109.75,
+      "learning_rate": 1.925508607198748e-06,
+      "loss": 94.0692,
+      "step": 475
+    },
+    {
+      "epoch": 0.1351612959997184,
+      "grad_norm": 106.875,
+      "learning_rate": 1.922378716744914e-06,
+      "loss": 94.4334,
+      "step": 480
+    },
+    {
+      "epoch": 0.13656922616638215,
+      "grad_norm": 110.4375,
+      "learning_rate": 1.9192488262910796e-06,
+      "loss": 95.9341,
+      "step": 485
+    },
+    {
+      "epoch": 0.13797715633304589,
+      "grad_norm": 108.5,
+      "learning_rate": 1.9161189358372454e-06,
+      "loss": 94.6785,
+      "step": 490
+    },
+    {
+      "epoch": 0.13938508649970963,
+      "grad_norm": 109.5,
+      "learning_rate": 1.9129890453834113e-06,
+      "loss": 95.5102,
+      "step": 495
+    },
+    {
+      "epoch": 0.14079301666637334,
+      "grad_norm": 110.3125,
+      "learning_rate": 1.909859154929577e-06,
+      "loss": 93.6705,
+      "step": 500
+    },
+    {
+      "epoch": 0.14079301666637334,
+      "eval_loss": 2.953507661819458,
+      "eval_runtime": 174.8095,
+      "eval_samples_per_second": 1094.883,
+      "eval_steps_per_second": 34.22,
+      "step": 500
+    },
+    {
+      "epoch": 0.14220094683303708,
+      "grad_norm": 116.3125,
+      "learning_rate": 1.9067292644757434e-06,
+      "loss": 93.6612,
+      "step": 505
+    },
+    {
+      "epoch": 0.14360887699970082,
+      "grad_norm": 107.5625,
+      "learning_rate": 1.9035993740219093e-06,
+      "loss": 93.6054,
+      "step": 510
+    },
+    {
+      "epoch": 0.14501680716636456,
+      "grad_norm": 107.4375,
+      "learning_rate": 1.900469483568075e-06,
+      "loss": 93.6585,
+      "step": 515
+    },
+    {
+      "epoch": 0.14642473733302827,
+      "grad_norm": 111.3125,
+      "learning_rate": 1.8973395931142408e-06,
+      "loss": 93.5845,
+      "step": 520
+    },
+    {
+      "epoch": 0.147832667499692,
+      "grad_norm": 109.8125,
+      "learning_rate": 1.8942097026604067e-06,
+      "loss": 96.1556,
+      "step": 525
+    },
+    {
+      "epoch": 0.14924059766635575,
+      "grad_norm": 112.3125,
+      "learning_rate": 1.8910798122065727e-06,
+      "loss": 95.0694,
+      "step": 530
+    },
+    {
+      "epoch": 0.1506485278330195,
+      "grad_norm": 108.0625,
+      "learning_rate": 1.8879499217527386e-06,
+      "loss": 94.7092,
+      "step": 535
+    },
+    {
+      "epoch": 0.1520564579996832,
+      "grad_norm": 107.0,
+      "learning_rate": 1.8848200312989044e-06,
+      "loss": 92.7553,
+      "step": 540
+    },
+    {
+      "epoch": 0.15346438816634694,
+      "grad_norm": 110.4375,
+      "learning_rate": 1.8816901408450703e-06,
+      "loss": 95.0633,
+      "step": 545
+    },
+    {
+      "epoch": 0.15487231833301068,
+      "grad_norm": 105.8125,
+      "learning_rate": 1.8785602503912362e-06,
+      "loss": 93.2634,
+      "step": 550
+    },
+    {
+      "epoch": 0.15628024849967442,
+      "grad_norm": 109.9375,
+      "learning_rate": 1.8754303599374022e-06,
+      "loss": 94.7607,
+      "step": 555
+    },
+    {
+      "epoch": 0.15768817866633816,
+      "grad_norm": 107.8125,
+      "learning_rate": 1.872300469483568e-06,
+      "loss": 94.321,
+      "step": 560
+    },
+    {
+      "epoch": 0.15909610883300188,
+      "grad_norm": 104.5625,
+      "learning_rate": 1.8691705790297338e-06,
+      "loss": 94.108,
+      "step": 565
+    },
+    {
+      "epoch": 0.16050403899966562,
+      "grad_norm": 106.25,
+      "learning_rate": 1.8660406885758996e-06,
+      "loss": 94.7094,
+      "step": 570
+    },
+    {
+      "epoch": 0.16191196916632936,
+      "grad_norm": 103.8125,
+      "learning_rate": 1.8629107981220657e-06,
+      "loss": 95.3564,
+      "step": 575
+    },
+    {
+      "epoch": 0.1633198993329931,
+      "grad_norm": 109.375,
+      "learning_rate": 1.8597809076682315e-06,
+      "loss": 94.0117,
+      "step": 580
+    },
+    {
+      "epoch": 0.1647278294996568,
+      "grad_norm": 111.0,
+      "learning_rate": 1.8566510172143974e-06,
+      "loss": 94.3991,
+      "step": 585
+    },
+    {
+      "epoch": 0.16613575966632055,
+      "grad_norm": 107.9375,
+      "learning_rate": 1.8535211267605633e-06,
+      "loss": 93.3577,
+      "step": 590
+    },
+    {
+      "epoch": 0.1675436898329843,
+      "grad_norm": 113.0,
+      "learning_rate": 1.8503912363067291e-06,
+      "loss": 92.6764,
+      "step": 595
+    },
+    {
+      "epoch": 0.16895161999964803,
+      "grad_norm": 108.875,
+      "learning_rate": 1.8472613458528952e-06,
+      "loss": 93.5743,
+      "step": 600
+    },
+    {
+      "epoch": 0.17035955016631174,
+      "grad_norm": 110.75,
+      "learning_rate": 1.844131455399061e-06,
+      "loss": 95.4879,
+      "step": 605
+    },
+    {
+      "epoch": 0.17176748033297548,
+      "grad_norm": 108.1875,
+      "learning_rate": 1.841001564945227e-06,
+      "loss": 94.0815,
+      "step": 610
+    },
+    {
+      "epoch": 0.17317541049963922,
+      "grad_norm": 107.6875,
+      "learning_rate": 1.8378716744913928e-06,
+      "loss": 93.5862,
+      "step": 615
+    },
+    {
+      "epoch": 0.17458334066630296,
+      "grad_norm": 110.6875,
+      "learning_rate": 1.8347417840375584e-06,
+      "loss": 94.7766,
+      "step": 620
+    },
+    {
+      "epoch": 0.17599127083296667,
+      "grad_norm": 107.75,
+      "learning_rate": 1.8316118935837245e-06,
+      "loss": 93.7132,
+      "step": 625
+    },
+    {
+      "epoch": 0.1773992009996304,
+      "grad_norm": 108.5,
+      "learning_rate": 1.8284820031298904e-06,
+      "loss": 95.441,
+      "step": 630
+    },
+    {
+      "epoch": 0.17880713116629415,
+      "grad_norm": 109.875,
+      "learning_rate": 1.8253521126760562e-06,
+      "loss": 94.4422,
+      "step": 635
+    },
+    {
+      "epoch": 0.1802150613329579,
+      "grad_norm": 108.9375,
+      "learning_rate": 1.822222222222222e-06,
+      "loss": 92.7065,
+      "step": 640
+    },
+    {
+      "epoch": 0.1816229914996216,
+      "grad_norm": 108.0,
+      "learning_rate": 1.819092331768388e-06,
+      "loss": 95.6058,
+      "step": 645
+    },
+    {
+      "epoch": 0.18303092166628535,
+      "grad_norm": 113.4375,
+      "learning_rate": 1.815962441314554e-06,
+      "loss": 93.1739,
+      "step": 650
+    },
+    {
+      "epoch": 0.18443885183294909,
+      "grad_norm": 105.375,
+      "learning_rate": 1.8128325508607199e-06,
+      "loss": 94.3629,
+      "step": 655
+    },
+    {
+      "epoch": 0.18584678199961283,
+      "grad_norm": 109.3125,
+      "learning_rate": 1.8097026604068857e-06,
+      "loss": 93.1163,
+      "step": 660
+    },
+    {
+      "epoch": 0.18725471216627657,
+      "grad_norm": 109.75,
+      "learning_rate": 1.8065727699530516e-06,
+      "loss": 91.9717,
+      "step": 665
+    },
+    {
+      "epoch": 0.18866264233294028,
+      "grad_norm": 107.0,
+      "learning_rate": 1.8034428794992173e-06,
+      "loss": 93.1724,
+      "step": 670
+    },
+    {
+      "epoch": 0.19007057249960402,
+      "grad_norm": 111.125,
+      "learning_rate": 1.8003129890453833e-06,
+      "loss": 93.8705,
+      "step": 675
+    },
+    {
+      "epoch": 0.19147850266626776,
+      "grad_norm": 106.6875,
+      "learning_rate": 1.7971830985915492e-06,
+      "loss": 95.3723,
+      "step": 680
+    },
+    {
+      "epoch": 0.1928864328329315,
+      "grad_norm": 105.75,
+      "learning_rate": 1.794053208137715e-06,
+      "loss": 94.0826,
+      "step": 685
+    },
+    {
+      "epoch": 0.1942943629995952,
+      "grad_norm": 108.8125,
+      "learning_rate": 1.790923317683881e-06,
+      "loss": 93.8173,
+      "step": 690
+    },
+    {
+      "epoch": 0.19570229316625895,
+      "grad_norm": 108.5625,
+      "learning_rate": 1.7877934272300468e-06,
+      "loss": 93.0598,
+      "step": 695
+    },
+    {
+      "epoch": 0.1971102233329227,
+      "grad_norm": 105.1875,
+      "learning_rate": 1.7846635367762128e-06,
+      "loss": 93.9662,
+      "step": 700
+    },
+    {
+      "epoch": 0.19851815349958643,
+      "grad_norm": 109.0625,
+      "learning_rate": 1.7815336463223787e-06,
+      "loss": 91.824,
+      "step": 705
+    },
+    {
+      "epoch": 0.19992608366625014,
+      "grad_norm": 109.625,
+      "learning_rate": 1.7784037558685446e-06,
+      "loss": 93.3993,
+      "step": 710
+    },
+    {
+      "epoch": 0.20133401383291388,
+      "grad_norm": 108.125,
+      "learning_rate": 1.7752738654147104e-06,
+      "loss": 92.4667,
+      "step": 715
+    },
+    {
+      "epoch": 0.20274194399957762,
+      "grad_norm": 111.0,
+      "learning_rate": 1.7721439749608763e-06,
+      "loss": 92.8489,
+      "step": 720
+    },
+    {
+      "epoch": 0.20414987416624136,
+      "grad_norm": 109.125,
+      "learning_rate": 1.7690140845070422e-06,
+      "loss": 93.2275,
+      "step": 725
+    },
+    {
+      "epoch": 0.20555780433290508,
+      "grad_norm": 108.0625,
+      "learning_rate": 1.765884194053208e-06,
+      "loss": 92.2952,
+      "step": 730
+    },
+    {
+      "epoch": 0.20696573449956882,
+      "grad_norm": 107.375,
+      "learning_rate": 1.7627543035993739e-06,
+      "loss": 93.8364,
+      "step": 735
+    },
+    {
+      "epoch": 0.20837366466623256,
+      "grad_norm": 110.0625,
+      "learning_rate": 1.7596244131455397e-06,
+      "loss": 93.7801,
+      "step": 740
+    },
+    {
+      "epoch": 0.2097815948328963,
+      "grad_norm": 109.8125,
+      "learning_rate": 1.7564945226917056e-06,
+      "loss": 93.1315,
+      "step": 745
+    },
+    {
+      "epoch": 0.21118952499956004,
+      "grad_norm": 111.0625,
+      "learning_rate": 1.7533646322378717e-06,
+      "loss": 91.7111,
+      "step": 750
+    },
+    {
+      "epoch": 0.21259745516622375,
+      "grad_norm": 108.25,
+      "learning_rate": 1.7502347417840375e-06,
+      "loss": 93.8666,
+      "step": 755
+    },
+    {
+      "epoch": 0.2140053853328875,
+      "grad_norm": 108.875,
+      "learning_rate": 1.7471048513302034e-06,
+      "loss": 91.9754,
+      "step": 760
+    },
+    {
+      "epoch": 0.21541331549955123,
+      "grad_norm": 108.0,
+      "learning_rate": 1.7439749608763693e-06,
+      "loss": 94.4781,
+      "step": 765
+    },
+    {
+      "epoch": 0.21682124566621497,
+      "grad_norm": 109.1875,
+      "learning_rate": 1.7408450704225351e-06,
+      "loss": 92.3431,
+      "step": 770
+    },
+    {
+      "epoch": 0.21822917583287868,
+      "grad_norm": 108.0625,
+      "learning_rate": 1.7377151799687012e-06,
+      "loss": 93.1383,
+      "step": 775
+    },
+    {
+      "epoch": 0.21963710599954242,
+      "grad_norm": 107.4375,
+      "learning_rate": 1.7345852895148668e-06,
+      "loss": 93.4093,
+      "step": 780
+    },
+    {
+      "epoch": 0.22104503616620616,
+      "grad_norm": 107.875,
+      "learning_rate": 1.7314553990610327e-06,
+      "loss": 93.3635,
+      "step": 785
+    },
+    {
+      "epoch": 0.2224529663328699,
+      "grad_norm": 109.0625,
+      "learning_rate": 1.7283255086071986e-06,
+      "loss": 90.5487,
+      "step": 790
+    },
+    {
+      "epoch": 0.2238608964995336,
+      "grad_norm": 105.9375,
+      "learning_rate": 1.7251956181533644e-06,
+      "loss": 93.1931,
+      "step": 795
+    },
+    {
+      "epoch": 0.22526882666619735,
+      "grad_norm": 109.5625,
+      "learning_rate": 1.7220657276995305e-06,
+      "loss": 93.2803,
+      "step": 800
+    },
+    {
+      "epoch": 0.2266767568328611,
+      "grad_norm": 105.75,
+      "learning_rate": 1.7189358372456964e-06,
+      "loss": 92.0815,
+      "step": 805
+    },
+    {
+      "epoch": 0.22808468699952483,
+      "grad_norm": 106.0625,
+      "learning_rate": 1.7158059467918622e-06,
+      "loss": 92.8134,
+      "step": 810
+    },
+    {
+      "epoch": 0.22949261716618855,
+      "grad_norm": 105.0625,
+      "learning_rate": 1.712676056338028e-06,
+      "loss": 91.2653,
+      "step": 815
+    },
+    {
+      "epoch": 0.23090054733285229,
+      "grad_norm": 107.4375,
+      "learning_rate": 1.709546165884194e-06,
+      "loss": 92.2337,
+      "step": 820
+    },
+    {
+      "epoch": 0.23230847749951603,
+      "grad_norm": 105.75,
+      "learning_rate": 1.70641627543036e-06,
+      "loss": 91.8069,
+      "step": 825
+    },
+    {
+      "epoch": 0.23371640766617977,
+      "grad_norm": 112.0,
+      "learning_rate": 1.7032863849765259e-06,
+      "loss": 90.8042,
+      "step": 830
+    },
+    {
+      "epoch": 0.23512433783284348,
+      "grad_norm": 107.25,
+      "learning_rate": 1.7001564945226915e-06,
+      "loss": 92.4549,
+      "step": 835
+    },
+    {
+      "epoch": 0.23653226799950722,
+      "grad_norm": 104.0625,
+      "learning_rate": 1.6970266040688574e-06,
+      "loss": 93.1807,
+      "step": 840
+    },
+    {
+      "epoch": 0.23794019816617096,
+      "grad_norm": 108.75,
+      "learning_rate": 1.6938967136150232e-06,
+      "loss": 92.848,
+      "step": 845
+    },
+    {
+      "epoch": 0.2393481283328347,
+      "grad_norm": 110.9375,
+      "learning_rate": 1.6907668231611893e-06,
+      "loss": 91.5934,
+      "step": 850
+    },
+    {
+      "epoch": 0.24075605849949844,
+      "grad_norm": 105.1875,
+      "learning_rate": 1.6876369327073552e-06,
+      "loss": 91.8347,
+      "step": 855
+    },
+    {
+      "epoch": 0.24216398866616215,
+      "grad_norm": 109.6875,
+      "learning_rate": 1.684507042253521e-06,
+      "loss": 91.4695,
+      "step": 860
+    },
+    {
+      "epoch": 0.2435719188328259,
+      "grad_norm": 108.3125,
+      "learning_rate": 1.681377151799687e-06,
+      "loss": 92.2532,
+      "step": 865
+    },
+    {
+      "epoch": 0.24497984899948963,
+      "grad_norm": 105.9375,
+      "learning_rate": 1.678247261345853e-06,
+      "loss": 91.6567,
+      "step": 870
+    },
+    {
+      "epoch": 0.24638777916615337,
+      "grad_norm": 107.4375,
+      "learning_rate": 1.6751173708920188e-06,
+      "loss": 91.7404,
+      "step": 875
+    },
+    {
+      "epoch": 0.24779570933281708,
+      "grad_norm": 108.125,
+      "learning_rate": 1.6719874804381847e-06,
+      "loss": 91.8094,
+      "step": 880
+    },
+    {
+      "epoch": 0.24920363949948082,
+      "grad_norm": 105.8125,
+      "learning_rate": 1.6688575899843503e-06,
+      "loss": 91.615,
+      "step": 885
+    },
+    {
+      "epoch": 0.25061156966614456,
+      "grad_norm": 108.0625,
+      "learning_rate": 1.6657276995305162e-06,
+      "loss": 93.0504,
+      "step": 890
+    },
+    {
+      "epoch": 0.2520194998328083,
+      "grad_norm": 112.375,
+      "learning_rate": 1.6625978090766823e-06,
+      "loss": 91.0786,
+      "step": 895
+    },
+    {
+      "epoch": 0.25342742999947204,
+      "grad_norm": 107.875,
+      "learning_rate": 1.6594679186228481e-06,
+      "loss": 90.6298,
+      "step": 900
+    },
+    {
+      "epoch": 0.2548353601661358,
+      "grad_norm": 106.1875,
+      "learning_rate": 1.656338028169014e-06,
+      "loss": 91.8793,
+      "step": 905
+    },
+    {
+      "epoch": 0.25624329033279947,
+      "grad_norm": 110.5,
+      "learning_rate": 1.6532081377151799e-06,
+      "loss": 91.6597,
+      "step": 910
+    },
+    {
+      "epoch": 0.2576512204994632,
+      "grad_norm": 106.125,
+      "learning_rate": 1.6500782472613457e-06,
+      "loss": 91.8785,
+      "step": 915
+    },
+    {
+      "epoch": 0.25905915066612695,
+      "grad_norm": 106.0625,
+      "learning_rate": 1.6469483568075118e-06,
+      "loss": 90.6928,
+      "step": 920
+    },
+    {
+      "epoch": 0.2604670808327907,
+      "grad_norm": 105.4375,
+      "learning_rate": 1.6438184663536777e-06,
+      "loss": 90.5155,
+      "step": 925
+    },
+    {
+      "epoch": 0.2618750109994544,
+      "grad_norm": 105.4375,
+      "learning_rate": 1.6406885758998435e-06,
+      "loss": 91.1331,
+      "step": 930
+    },
+    {
+      "epoch": 0.26328294116611817,
+      "grad_norm": 104.875,
+      "learning_rate": 1.6375586854460094e-06,
+      "loss": 92.0714,
+      "step": 935
+    },
+    {
+      "epoch": 0.2646908713327819,
+      "grad_norm": 106.375,
+      "learning_rate": 1.634428794992175e-06,
+      "loss": 91.0242,
+      "step": 940
+    },
+    {
+      "epoch": 0.26609880149944565,
+      "grad_norm": 108.6875,
+      "learning_rate": 1.631298904538341e-06,
+      "loss": 91.612,
+      "step": 945
+    },
+    {
+      "epoch": 0.2675067316661094,
+      "grad_norm": 104.6875,
+      "learning_rate": 1.628169014084507e-06,
+      "loss": 90.1659,
+      "step": 950
+    },
+    {
+      "epoch": 0.26891466183277307,
+      "grad_norm": 105.1875,
+      "learning_rate": 1.6250391236306728e-06,
+      "loss": 91.2171,
+      "step": 955
+    },
+    {
+      "epoch": 0.2703225919994368,
+      "grad_norm": 107.375,
+      "learning_rate": 1.6219092331768387e-06,
+      "loss": 91.4,
+      "step": 960
+    },
+    {
+      "epoch": 0.27173052216610055,
+      "grad_norm": 105.5625,
+      "learning_rate": 1.6187793427230045e-06,
+      "loss": 90.2747,
+      "step": 965
+    },
+    {
+      "epoch": 0.2731384523327643,
+      "grad_norm": 113.0,
+      "learning_rate": 1.6156494522691706e-06,
+      "loss": 90.9026,
+      "step": 970
+    },
+    {
+      "epoch": 0.27454638249942803,
+      "grad_norm": 108.8125,
+      "learning_rate": 1.6125195618153365e-06,
+      "loss": 90.2174,
+      "step": 975
+    },
+    {
+      "epoch": 0.27595431266609177,
+      "grad_norm": 107.0625,
+      "learning_rate": 1.6093896713615023e-06,
+      "loss": 92.164,
+      "step": 980
+    },
+    {
+      "epoch": 0.2773622428327555,
+      "grad_norm": 109.0,
+      "learning_rate": 1.6062597809076682e-06,
+      "loss": 90.952,
+      "step": 985
+    },
+    {
+      "epoch": 0.27877017299941925,
+      "grad_norm": 109.3125,
+      "learning_rate": 1.6031298904538339e-06,
+      "loss": 91.6762,
+      "step": 990
+    },
+    {
+      "epoch": 0.28017810316608294,
+      "grad_norm": 108.6875,
+      "learning_rate": 1.6e-06,
+      "loss": 89.3236,
+      "step": 995
+    },
+    {
+      "epoch": 0.2815860333327467,
+      "grad_norm": 105.0,
+      "learning_rate": 1.5968701095461658e-06,
+      "loss": 89.8639,
+      "step": 1000
+    },
+    {
+      "epoch": 0.2815860333327467,
+      "eval_loss": 2.838271141052246,
+      "eval_runtime": 172.5248,
+      "eval_samples_per_second": 1109.383,
+      "eval_steps_per_second": 34.673,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 3551,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.332357992788787e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60b40db795e654ff7cc6763d0a4e65ffde9f8eeb4a5da37c02cffa1d394588ed
+size 5752