Training in progress, step 500000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb6c56954e78dc0009a37a49227c9d09ca6b44ee0bf58f8054ecb3de57d1f0ca
+oid sha256:6cfa7c7e92be438dcf3db7b213e9dec445fc69325f8af229476e4fade60f1c4f
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48c7a81d5ad9f4939a3a394ee2ec0cee99260435a555fb2a922706840415be41
+oid sha256:47400866d2b3874aa8db515822b34e6309789728f282f9bb86b1432c96816298
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:851e62cc221f5b89fc632a61a046dd22f6c0b1983df71dc5cf6a85d362878b7f
+oid sha256:6bb40df1d2cacfe96f8a1f89857bbc3220c52b2d5fca642cb88aa8e9fbbc97a5
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c87680c6a6ccbae9d154900996c944b2bf164f9842479579468f42223a09d2f
+oid sha256:a6052cb7a9c038f97f0bff232e594126132c8dd3530545ea97f7bd3c6962c6a4
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.9890306854029252,
   "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-Trial-15/checkpoint-480000",
-  "epoch": 45.88014981273408,
+  "epoch": 46.81647940074907,
   "eval_steps": 10000,
-  "global_step": 490000,
+  "global_step": 500000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4235,6 +4235,118 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 1.0,
       "step": 490000
+    },
+    {
+      "epoch": 45.97378277153558,
+      "grad_norm": 0.012753269635140896,
+      "learning_rate": 8.518086768107509e-06,
+      "loss": 0.0003,
+      "step": 491000
+    },
+    {
+      "epoch": 46.06741573033708,
+      "grad_norm": 0.0015431916108354926,
+      "learning_rate": 1.510764106820829e-05,
+      "loss": 0.0003,
+      "step": 492000
+    },
+    {
+      "epoch": 46.161048689138575,
+      "grad_norm": 0.02155626006424427,
+      "learning_rate": 2.25881865139301e-05,
+      "loss": 0.0004,
+      "step": 493000
+    },
+    {
+      "epoch": 46.254681647940075,
+      "grad_norm": 0.01683797501027584,
+      "learning_rate": 3.0298952869963953e-05,
+      "loss": 0.0003,
+      "step": 494000
+    },
+    {
+      "epoch": 46.348314606741575,
+      "grad_norm": 0.009855197742581367,
+      "learning_rate": 3.750390492464676e-05,
+      "loss": 0.0003,
+      "step": 495000
+    },
+    {
+      "epoch": 46.441947565543074,
+      "grad_norm": 0.003276103874668479,
+      "learning_rate": 4.351529023344395e-05,
+      "loss": 0.0003,
+      "step": 496000
+    },
+    {
+      "epoch": 46.53558052434457,
+      "grad_norm": 0.001958276843652129,
+      "learning_rate": 4.775928888612517e-05,
+      "loss": 0.0003,
+      "step": 497000
+    },
+    {
+      "epoch": 46.62921348314607,
+      "grad_norm": 0.0038878321647644043,
+      "learning_rate": 4.983891000015541e-05,
+      "loss": 0.0003,
+      "step": 498000
+    },
+    {
+      "epoch": 46.72284644194757,
+      "grad_norm": 0.0057247113436460495,
+      "learning_rate": 4.816788519950127e-07,
+      "loss": 0.0003,
+      "step": 499000
+    },
+    {
+      "epoch": 46.81647940074907,
+      "grad_norm": 0.004408022854477167,
+      "learning_rate": 3.17712320846264e-06,
+      "loss": 0.0003,
+      "step": 500000
+    },
+    {
+      "epoch": 46.81647940074907,
+      "eval_all_ligands_equal": 0.9880269572698595,
+      "eval_e3_equal": 0.9950530542013192,
+      "eval_e3_graph_edit_distance": Infinity,
+      "eval_e3_graph_edit_distance_norm": Infinity,
+      "eval_e3_has_attachment_point(s)": 1.0,
+      "eval_e3_heavy_atoms_difference": 0.004086607398910238,
+      "eval_e3_heavy_atoms_difference_norm": 0.00021304401339568784,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 1.0,
+      "eval_has_all_attachment_points": 1.0,
+      "eval_has_three_substructures": 1.0,
+      "eval_heavy_atoms_difference": -0.00014338973329509606,
+      "eval_heavy_atoms_difference_norm": -2.106863305601476e-06,
+      "eval_linker_equal": 0.9881703470031545,
+      "eval_linker_graph_edit_distance": 0.11578720963579008,
+      "eval_linker_graph_edit_distance_norm": Infinity,
+      "eval_linker_has_attachment_point(s)": 1.0,
+      "eval_linker_heavy_atoms_difference": -0.016633209062231143,
+      "eval_linker_heavy_atoms_difference_norm": -0.0027458937792369477,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 1.0,
+      "eval_loss": 0.00021886541799176484,
+      "eval_num_fragments": 3.0,
+      "eval_poi_equal": 0.9930455979351879,
+      "eval_poi_graph_edit_distance": Infinity,
+      "eval_poi_graph_edit_distance_norm": Infinity,
+      "eval_poi_has_attachment_point(s)": 1.0,
+      "eval_poi_heavy_atoms_difference": 0.01240321193002581,
+      "eval_poi_heavy_atoms_difference_norm": 0.00025383141098379934,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 1.0,
+      "eval_reassembly": 0.9997132205334098,
+      "eval_reassembly_nostereo": 0.9997132205334098,
+      "eval_runtime": 2712.3666,
+      "eval_samples_per_second": 5.142,
+      "eval_steps_per_second": 0.08,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 1.0,
+      "step": 500000
     }
   ],
   "logging_steps": 1000,
@@ -4249,12 +4361,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.214983293542787e+18,
+  "total_flos": 4.3010984555114865e+18,
   "train_batch_size": 128,
   "trial_name": null,
   "trial_params": null