Upload model checkpoint

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "roberta-large",
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 50265
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0265fc0baf777b0e8df143572f06deb46a5efe4bd895c4b24fc0df85477ee03
+size 1421696540

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3c16a5a27b755d1cd881c36d14cdff09cddacb2a662734b798458e68f340aff
+size 2843626160

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:213d67ef22ce8b2800bbe6e53fdef8f747e56dce8bf6a96a171c8a7a44ad5577
+size 14244

roberta_excl_no_struct_no_token_training.log

@@ -0,0 +1,12 @@
+2024-08-22 20:48:08,618 - INFO - Step 25000: training loss = 2.2576
+2024-08-22 21:06:59,564 - INFO - Step 25000: validation loss = 1.9398
+2024-08-22 22:21:01,448 - INFO - Step 50000: training loss = 1.9854
+2024-08-22 22:40:00,762 - INFO - Step 50000: validation loss = 1.7355
+2024-08-22 23:53:58,432 - INFO - Step 75000: training loss = 1.8002
+2024-08-23 00:12:23,862 - INFO - Step 75000: validation loss = 1.5858
+2024-08-23 01:26:19,158 - INFO - Step 100000: training loss = 1.6281
+2024-08-23 01:44:22,764 - INFO - Step 100000: validation loss = 1.4415
+2024-08-23 02:58:17,873 - INFO - Step 125000: training loss = 1.5041
+2024-08-23 03:16:07,162 - INFO - Step 125000: validation loss = 1.3530
+2024-08-23 04:30:05,030 - INFO - Step 150000: training loss = 1.4019
+2024-08-23 04:47:38,762 - INFO - Step 150000: validation loss = 1.2703

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37c8591dc2f03beba7e48f10e0e6391e27a4406f566d9e4ed95b2fbd4d72bc84
+size 1064

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50264": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "RobertaTokenizer",
+  "unk_token": "<unk>"
+}

trainer_state.json

@@ -0,0 +1,123 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.7863309952245419,
+  "eval_steps": 25000,
+  "global_step": 150000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.2977218325374236,
+      "grad_norm": 14.776897430419922,
+      "learning_rate": 4.729071698412804e-05,
+      "loss": 2.2576,
+      "step": 25000
+    },
+    {
+      "epoch": 0.2977218325374236,
+      "eval_loss": 1.9398064613342285,
+      "eval_runtime": 1130.9439,
+      "eval_samples_per_second": 148.496,
+      "eval_steps_per_second": 1.161,
+      "step": 25000
+    },
+    {
+      "epoch": 0.5954436650748473,
+      "grad_norm": 15.924115180969238,
+      "learning_rate": 3.9023268738380516e-05,
+      "loss": 1.9854,
+      "step": 50000
+    },
+    {
+      "epoch": 0.5954436650748473,
+      "eval_loss": 1.7355190515518188,
+      "eval_runtime": 1139.3123,
+      "eval_samples_per_second": 147.406,
+      "eval_steps_per_second": 1.152,
+      "step": 50000
+    },
+    {
+      "epoch": 0.8931654976122709,
+      "grad_norm": 18.3256893157959,
+      "learning_rate": 3.075648210020709e-05,
+      "loss": 1.8002,
+      "step": 75000
+    },
+    {
+      "epoch": 0.8931654976122709,
+      "eval_loss": 1.5857831239700317,
+      "eval_runtime": 1105.4287,
+      "eval_samples_per_second": 151.924,
+      "eval_steps_per_second": 1.188,
+      "step": 75000
+    },
+    {
+      "epoch": 1.1908873301496945,
+      "grad_norm": 24.324203491210938,
+      "learning_rate": 2.2488703050672525e-05,
+      "loss": 1.6281,
+      "step": 100000
+    },
+    {
+      "epoch": 1.1908873301496945,
+      "eval_loss": 1.4414993524551392,
+      "eval_runtime": 1083.6047,
+      "eval_samples_per_second": 154.984,
+      "eval_steps_per_second": 1.212,
+      "step": 100000
+    },
+    {
+      "epoch": 1.4886091626871183,
+      "grad_norm": 21.457475662231445,
+      "learning_rate": 1.4222247216286134e-05,
+      "loss": 1.5041,
+      "step": 125000
+    },
+    {
+      "epoch": 1.4886091626871183,
+      "eval_loss": 1.3530445098876953,
+      "eval_runtime": 1069.2866,
+      "eval_samples_per_second": 157.059,
+      "eval_steps_per_second": 1.228,
+      "step": 125000
+    },
+    {
+      "epoch": 1.7863309952245419,
+      "grad_norm": 8.10311222076416,
+      "learning_rate": 5.9544681667515734e-06,
+      "loss": 1.4019,
+      "step": 150000
+    },
+    {
+      "epoch": 1.7863309952245419,
+      "eval_loss": 1.2703442573547363,
+      "eval_runtime": 1053.7311,
+      "eval_samples_per_second": 159.377,
+      "eval_steps_per_second": 1.246,
+      "step": 150000
+    }
+  ],
+  "logging_steps": 25000,
+  "max_steps": 167942,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 25000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1184982650879437e+18,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da9f964c1055e87bd8c8047f16b32dbe63d38ce8cca37a48013a5a4bc38b15f6
+size 5112