Training in progress, step 100, checkpoint

last-checkpoint/config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "mgh6/TCS_MLM_All",
+  "_name_or_path": "facebook/esm2_t12_35M_UR50D",
   "architectures": [
     "EsmForMaskedLM"
   ],
@@ -23,7 +23,7 @@
   "position_embedding_type": "rotary",
   "token_dropout": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.41.1",
+  "transformers_version": "4.45.2",
   "use_cache": true,
   "vocab_list": null,
   "vocab_size": 33

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:059d13c27bf9d6ab11c2dfe23b5144a5b66888cdab247e126eeed36eaf2ddac5
+oid sha256:260f33e18a57d6a52a92153fced5f4433b825cbae7e74e13a6bc31c6c3890bb7
 size 136000488

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37d67377845895ac8a961852d630a9818de94d18690b3a4dd5b15055542ad775
+oid sha256:90a09eabfc08ce01b91af2e622183e0ef79e272762919610e00fff525c6e193b
 size 268176506

last-checkpoint/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dcfee971f88c3cda33b81baee0cbbee275ada59edce084b10f143d24d1066de
+size 15024

last-checkpoint/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf7fb54143717feb75da48fb46d5104be02f6922f4165bb1f8dc901d2d29dfe4
+size 15024

last-checkpoint/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e20d60dc689485036fab9f46d0c8f9988c61adf50aea162ec6b58003dc5f271
+size 15024

last-checkpoint/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cd11604e085d709114e3ae610473be24a56ca9f858b53106ddf99cb2d836153
+size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fbcad846aab1f58cf860b8c192e9988ce56fad568e6f4990d5c6c9cee42b3501
+oid sha256:507aa2c5f1b24139884502ecf564787c3e6d78876a1f9bae04804ba26898f238
 size 1064

last-checkpoint/trainer_state.json

@@ -1,307 +1,44 @@
 {
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 9.875948452366615,
-  "eval_steps": 500,
-  "global_step": 41000,
+  "best_metric": 1.250386118888855,
+  "best_model_checkpoint": "mgh6/TCS_MLM_All/checkpoint-100",
+  "epoch": 0.9523809523809523,
+  "eval_steps": 100,
+  "global_step": 100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.24087679152113695,
-      "grad_norm": 0.6258379220962524,
-      "learning_rate": 4.153472996796067e-06,
-      "loss": 0.7631,
-      "step": 1000
+      "epoch": 0.9523809523809523,
+      "grad_norm": 0.18039299547672272,
+      "learning_rate": 0.0009047619047619047,
+      "loss": 1.5254,
+      "step": 100
     },
     {
-      "epoch": 0.4817535830422739,
-      "grad_norm": 0.6458311676979065,
-      "learning_rate": 4.050943423930205e-06,
-      "loss": 0.7552,
-      "step": 2000
-    },
-    {
-      "epoch": 0.7226303745634108,
-      "grad_norm": 0.645728349685669,
-      "learning_rate": 3.948413851064344e-06,
-      "loss": 0.7559,
-      "step": 3000
-    },
-    {
-      "epoch": 0.9635071660845478,
-      "grad_norm": 0.6008012294769287,
-      "learning_rate": 3.845884278198482e-06,
-      "loss": 0.7608,
-      "step": 4000
-    },
-    {
-      "epoch": 1.2043839576056847,
-      "grad_norm": 0.7096830606460571,
-      "learning_rate": 3.743354705332619e-06,
-      "loss": 0.7574,
-      "step": 5000
-    },
-    {
-      "epoch": 1.4452607491268217,
-      "grad_norm": 0.5911938548088074,
-      "learning_rate": 3.640825132466757e-06,
-      "loss": 0.7549,
-      "step": 6000
-    },
-    {
-      "epoch": 1.6861375406479586,
-      "grad_norm": 0.7647172212600708,
-      "learning_rate": 3.5382955596008955e-06,
-      "loss": 0.7557,
-      "step": 7000
-    },
-    {
-      "epoch": 1.9270143321690956,
-      "grad_norm": 0.5935486555099487,
-      "learning_rate": 3.435765986735034e-06,
-      "loss": 0.7594,
-      "step": 8000
-    },
-    {
-      "epoch": 2.1678911236902323,
-      "grad_norm": 0.6902479529380798,
-      "learning_rate": 3.333236413869172e-06,
-      "loss": 0.7593,
-      "step": 9000
-    },
-    {
-      "epoch": 2.4087679152113695,
-      "grad_norm": 0.650104284286499,
-      "learning_rate": 3.2307068410033098e-06,
-      "loss": 0.7583,
-      "step": 10000
-    },
-    {
-      "epoch": 2.649644706732506,
-      "grad_norm": 0.5951535701751709,
-      "learning_rate": 3.1281772681374477e-06,
-      "loss": 0.7506,
-      "step": 11000
-    },
-    {
-      "epoch": 2.8905214982536434,
-      "grad_norm": 0.643113911151886,
-      "learning_rate": 3.0256476952715857e-06,
-      "loss": 0.7577,
-      "step": 12000
-    },
-    {
-      "epoch": 3.13139828977478,
-      "grad_norm": 0.5796828866004944,
-      "learning_rate": 2.923118122405724e-06,
-      "loss": 0.7594,
-      "step": 13000
-    },
-    {
-      "epoch": 3.3722750812959172,
-      "grad_norm": 0.6443358063697815,
-      "learning_rate": 2.820588549539862e-06,
-      "loss": 0.7539,
-      "step": 14000
-    },
-    {
-      "epoch": 3.613151872817054,
-      "grad_norm": 0.5989753603935242,
-      "learning_rate": 2.718058976674e-06,
-      "loss": 0.7578,
-      "step": 15000
-    },
-    {
-      "epoch": 3.854028664338191,
-      "grad_norm": 0.5480288863182068,
-      "learning_rate": 2.615529403808138e-06,
-      "loss": 0.7531,
-      "step": 16000
-    },
-    {
-      "epoch": 4.094905455859328,
-      "grad_norm": 0.6654678583145142,
-      "learning_rate": 2.512999830942276e-06,
-      "loss": 0.7579,
-      "step": 17000
-    },
-    {
-      "epoch": 4.335782247380465,
-      "grad_norm": 0.6215384006500244,
-      "learning_rate": 2.4104702580764137e-06,
-      "loss": 0.7545,
-      "step": 18000
-    },
-    {
-      "epoch": 4.576659038901602,
-      "grad_norm": 0.6459180116653442,
-      "learning_rate": 2.3079406852105525e-06,
-      "loss": 0.7566,
-      "step": 19000
-    },
-    {
-      "epoch": 4.817535830422739,
-      "grad_norm": 0.6795477867126465,
-      "learning_rate": 2.2054111123446905e-06,
-      "loss": 0.7555,
-      "step": 20000
-    },
-    {
-      "epoch": 5.058412621943876,
-      "grad_norm": 0.6410796046257019,
-      "learning_rate": 2.1028815394788284e-06,
-      "loss": 0.7503,
-      "step": 21000
-    },
-    {
-      "epoch": 5.299289413465012,
-      "grad_norm": 0.5807074308395386,
-      "learning_rate": 2.0003519666129664e-06,
-      "loss": 0.7529,
-      "step": 22000
-    },
-    {
-      "epoch": 5.54016620498615,
-      "grad_norm": 0.5771316885948181,
-      "learning_rate": 1.8978223937471045e-06,
-      "loss": 0.751,
-      "step": 23000
-    },
-    {
-      "epoch": 5.781042996507287,
-      "grad_norm": 0.6301162838935852,
-      "learning_rate": 1.7952928208812425e-06,
-      "loss": 0.7534,
-      "step": 24000
-    },
-    {
-      "epoch": 6.021919788028423,
-      "grad_norm": 0.5753132104873657,
-      "learning_rate": 1.6927632480153806e-06,
-      "loss": 0.7511,
-      "step": 25000
-    },
-    {
-      "epoch": 6.26279657954956,
-      "grad_norm": 0.6592470407485962,
-      "learning_rate": 1.5902336751495186e-06,
-      "loss": 0.7554,
-      "step": 26000
-    },
-    {
-      "epoch": 6.503673371070697,
-      "grad_norm": 0.6203874349594116,
-      "learning_rate": 1.4877041022836565e-06,
-      "loss": 0.7568,
-      "step": 27000
-    },
-    {
-      "epoch": 6.7445501625918345,
-      "grad_norm": 0.6534298062324524,
-      "learning_rate": 1.385174529417795e-06,
-      "loss": 0.7553,
-      "step": 28000
-    },
-    {
-      "epoch": 6.985426954112971,
-      "grad_norm": 0.6775331497192383,
-      "learning_rate": 1.2826449565519328e-06,
-      "loss": 0.7525,
-      "step": 29000
-    },
-    {
-      "epoch": 7.226303745634108,
-      "grad_norm": 0.640843391418457,
-      "learning_rate": 1.180115383686071e-06,
-      "loss": 0.7559,
-      "step": 30000
-    },
-    {
-      "epoch": 7.4671805371552455,
-      "grad_norm": 0.6803774833679199,
-      "learning_rate": 1.077585810820209e-06,
-      "loss": 0.7508,
-      "step": 31000
-    },
-    {
-      "epoch": 7.708057328676382,
-      "grad_norm": 0.6412643790245056,
-      "learning_rate": 9.75056237954347e-07,
-      "loss": 0.7534,
-      "step": 32000
-    },
-    {
-      "epoch": 7.948934120197519,
-      "grad_norm": 0.6142516136169434,
-      "learning_rate": 8.725266650884852e-07,
-      "loss": 0.7531,
-      "step": 33000
-    },
-    {
-      "epoch": 8.189810911718656,
-      "grad_norm": 0.670280933380127,
-      "learning_rate": 7.699970922226231e-07,
-      "loss": 0.7506,
-      "step": 34000
-    },
-    {
-      "epoch": 8.430687703239792,
-      "grad_norm": 0.6336104869842529,
-      "learning_rate": 6.674675193567612e-07,
-      "loss": 0.7538,
-      "step": 35000
-    },
-    {
-      "epoch": 8.67156449476093,
-      "grad_norm": 0.6225976347923279,
-      "learning_rate": 5.649379464908993e-07,
-      "loss": 0.7521,
-      "step": 36000
-    },
-    {
-      "epoch": 8.912441286282068,
-      "grad_norm": 0.7998696565628052,
-      "learning_rate": 4.624083736250373e-07,
-      "loss": 0.7559,
-      "step": 37000
-    },
-    {
-      "epoch": 9.153318077803204,
-      "grad_norm": 0.6381024122238159,
-      "learning_rate": 3.598788007591754e-07,
-      "loss": 0.7547,
-      "step": 38000
-    },
-    {
-      "epoch": 9.394194869324341,
-      "grad_norm": 0.558660089969635,
-      "learning_rate": 2.5734922789331346e-07,
-      "loss": 0.7539,
-      "step": 39000
-    },
-    {
-      "epoch": 9.635071660845478,
-      "grad_norm": 0.7711436748504639,
-      "learning_rate": 1.5481965502745152e-07,
-      "loss": 0.7484,
-      "step": 40000
-    },
-    {
-      "epoch": 9.875948452366615,
-      "grad_norm": 0.6147646903991699,
-      "learning_rate": 5.229008216158959e-08,
-      "loss": 0.7509,
-      "step": 41000
+      "epoch": 0.9523809523809523,
+      "eval_loss": 1.250386118888855,
+      "eval_runtime": 0.218,
+      "eval_samples_per_second": 87.168,
+      "eval_steps_per_second": 4.588,
+      "step": 100
     }
   ],
-  "logging_steps": 1000,
-  "max_steps": 41510,
+  "logging_steps": 100,
+  "max_steps": 1050,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 10,
-  "save_steps": 1000,
+  "save_steps": 100,
   "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 5,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
     "TrainerControl": {
       "args": {
         "should_epoch_stop": false,
@@ -313,7 +50,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.101497938646139e+17,
+  "total_flos": 2931727227944960.0,
   "train_batch_size": 64,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b19c50ebceb96bf9ab86b4ad80b203101b877c903c2c529277d18a83643e787
-size 5112
+oid sha256:873e2b625b704f36c39ea8938bc18e37074ce987126ad27c9b364f4977d71919
+size 5240