Training in progress, epoch 4, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:219be41537bbb98fca70ca3f58664027f22ac8b626937fc54ed0c58b1a583287
+oid sha256:7fcef207828e97d334020207b7c4e0ce8b10911f91e884313cc27540bcf22215
 size 1713050034

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c557980c0526bbe9b748ef020546f7bc8e22ad8fcbd68d484140a76b913f895
+oid sha256:a3ed2b4efa016d58457f3d8931b71a9ebaaae4412d1d353411975961be30dbac
 size 816721594

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f5f4c3829d3c61d60d4aa81f39b1ae90c914023d099d2c2879c131506416ca01
+oid sha256:6ba1195da48c8391083911b2a9cc3e15ab8d82f8f8936d669e0116eb962ce738
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81c3e6445408f158b3d87cbf8d2d8e36840ad25379ff7117c45407306acac4e6
+oid sha256:edebf099edd988740678361da4ee055b665f3729f5f42b95a748d6bab1b73604
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 6.317481165751815e-05,
-  "best_model_checkpoint": "ProtChem_ESM2_MolGen_Decoder/checkpoint-11916",
-  "epoch": 2.9998111901315374,
+  "best_metric": 6.0006157582392916e-05,
+  "best_model_checkpoint": "ProtChem_ESM2_MolGen_Decoder/checkpoint-15889",
+  "epoch": 4.0,
   "eval_steps": 500,
-  "global_step": 11916,
+  "global_step": 15889,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -192,6 +192,70 @@
       "eval_samples_per_second": 15.248,
       "eval_steps_per_second": 0.953,
       "step": 11916
+    },
+    {
+      "epoch": 3.020957895399333,
+      "grad_norm": 0.0028076171875,
+      "learning_rate": 1.672753744810037e-05,
+      "loss": 0.0002,
+      "step": 12000
+    },
+    {
+      "epoch": 3.146831141040972,
+      "grad_norm": 0.00225830078125,
+      "learning_rate": 1.6413686942993405e-05,
+      "loss": 0.0001,
+      "step": 12500
+    },
+    {
+      "epoch": 3.2727043866826104,
+      "grad_norm": 0.00579833984375,
+      "learning_rate": 1.608871550088606e-05,
+      "loss": 0.0002,
+      "step": 13000
+    },
+    {
+      "epoch": 3.3985776323242494,
+      "grad_norm": 0.0086669921875,
+      "learning_rate": 1.5753186602186207e-05,
+      "loss": 0.0002,
+      "step": 13500
+    },
+    {
+      "epoch": 3.5244508779658883,
+      "grad_norm": 0.0022735595703125,
+      "learning_rate": 1.540768203327934e-05,
+      "loss": 0.0001,
+      "step": 14000
+    },
+    {
+      "epoch": 3.6503241236075272,
+      "grad_norm": 0.01202392578125,
+      "learning_rate": 1.5052800877746915e-05,
+      "loss": 0.0001,
+      "step": 14500
+    },
+    {
+      "epoch": 3.776197369249166,
+      "grad_norm": 0.01190185546875,
+      "learning_rate": 1.4689158477592433e-05,
+      "loss": 0.0001,
+      "step": 15000
+    },
+    {
+      "epoch": 3.902070614890805,
+      "grad_norm": 0.017822265625,
+      "learning_rate": 1.4317385366276393e-05,
+      "loss": 0.0001,
+      "step": 15500
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 6.0006157582392916e-05,
+      "eval_runtime": 16792.4882,
+      "eval_samples_per_second": 15.238,
+      "eval_steps_per_second": 0.952,
+      "step": 15889
     }
   ],
   "logging_steps": 500,
@@ -220,7 +284,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 8.002002428481992e+18,
+  "total_flos": 1.0669336571309322e+19,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null