Training in progress, epoch 4, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7fcef207828e97d334020207b7c4e0ce8b10911f91e884313cc27540bcf22215
+oid sha256:790e59fe9066d0dd6221cfe2764f4275cd9c904136ac1745ccc0f3eb48a5ed1b
 size 1713050034

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a3ed2b4efa016d58457f3d8931b71a9ebaaae4412d1d353411975961be30dbac
+oid sha256:df630dcd81b9b194bc0ac1be2a6a9fef794e8226952b0be6694b2f40bb7e1a82
 size 816721594

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ba1195da48c8391083911b2a9cc3e15ab8d82f8f8936d669e0116eb962ce738
+oid sha256:8693c95f0f09ea2f300c00a9d97d4ecc249e7fd41ef6cc52cb95a982ef985bca
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:edebf099edd988740678361da4ee055b665f3729f5f42b95a748d6bab1b73604
+oid sha256:414d81b0296706ee8544c28eaf6013748a0dfc834c3d7fbd1015bc57e89aeb34
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 6.0006157582392916e-05,
-  "best_model_checkpoint": "ProtChem_ESM2_MolGen_Decoder/checkpoint-15889",
-  "epoch": 4.0,
+  "best_metric": 5.9128127759322524e-05,
+  "best_model_checkpoint": "ProtChem_ESM2_MolGen_Decoder/checkpoint-19861",
+  "epoch": 4.999937063377179,
   "eval_steps": 500,
-  "global_step": 15889,
+  "global_step": 19861,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -256,6 +256,70 @@
       "eval_samples_per_second": 15.238,
       "eval_steps_per_second": 0.952,
       "step": 15889
+    },
+    {
+      "epoch": 4.027943860532444,
+      "grad_norm": 0.007171630859375,
+      "learning_rate": 1.393812617541025e-05,
+      "loss": 0.0001,
+      "step": 16000
+    },
+    {
+      "epoch": 4.153817106174083,
+      "grad_norm": 0.001190185546875,
+      "learning_rate": 1.3552038517004991e-05,
+      "loss": 0.0001,
+      "step": 16500
+    },
+    {
+      "epoch": 4.279690351815722,
+      "grad_norm": 0.003082275390625,
+      "learning_rate": 1.3159791843212542e-05,
+      "loss": 0.0001,
+      "step": 17000
+    },
+    {
+      "epoch": 4.405563597457361,
+      "grad_norm": 0.002593994140625,
+      "learning_rate": 1.2762066285537071e-05,
+      "loss": 0.0001,
+      "step": 17500
+    },
+    {
+      "epoch": 4.531436843099,
+      "grad_norm": 0.01129150390625,
+      "learning_rate": 1.2359551475529e-05,
+      "loss": 0.0001,
+      "step": 18000
+    },
+    {
+      "epoch": 4.657310088740639,
+      "grad_norm": 0.002166748046875,
+      "learning_rate": 1.1952945349006455e-05,
+      "loss": 0.0001,
+      "step": 18500
+    },
+    {
+      "epoch": 4.783183334382277,
+      "grad_norm": 0.006622314453125,
+      "learning_rate": 1.1542952935877703e-05,
+      "loss": 0.0001,
+      "step": 19000
+    },
+    {
+      "epoch": 4.909056580023916,
+      "grad_norm": 0.0029449462890625,
+      "learning_rate": 1.1130285137662811e-05,
+      "loss": 0.0001,
+      "step": 19500
+    },
+    {
+      "epoch": 4.999937063377179,
+      "eval_loss": 5.9128127759322524e-05,
+      "eval_runtime": 16790.9197,
+      "eval_samples_per_second": 15.24,
+      "eval_steps_per_second": 0.952,
+      "step": 19861
     }
   ],
   "logging_steps": 500,
@@ -284,7 +348,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.0669336571309322e+19,
+  "total_flos": 1.3336670714136654e+19,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null