Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8809287208bd2abeabde5d7e2d1f0a62d8eef9063b9a057f1498e2b16b9b84bd
+oid sha256:b58e752952aed4677522ad8389334c4861f2539aa80e417fc458133fc3e9c5ab
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d32e08deec2215967ea668a7f9f52073adc701a6c1145cdd6c0aaa59ce9724a9
+oid sha256:1d7ecb2ac19189bcf2c233b0cead6a8a4acbb6f35226ebd36c13702b9b92fd23
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79113e7c8e5f940ec750e9a6bfaeee63dab08300d78af556570f03137e343283
+oid sha256:51be3507e4858df120a8f607c701c436099c60ecffd12f4bdd16f4abede4cc55
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e45aaec203a60beabc34853347eaed388f12f650d48920d9a6930327facf81ef
+oid sha256:8f51f9073d820deca33496502793f5bfc1491af5bcb6df0fcc33f6c729a1edcd
 size 1192

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.5277065841802916,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=7-learning_rate=7.9e-05-warmup_ratio=0.060-min_lr=0.000-factor=0.920/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.5534246575342465,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=7-learning_rate=7.9e-05-warmup_ratio=0.060-min_lr=0.000-factor=0.920/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.9274414494034467,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.10655809193849564,
+      "learning_rate": 7.897721300792183e-05,
+      "loss": 0.0054,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.10990337282419205,
+      "learning_rate": 7.897721300792183e-05,
+      "loss": 0.0048,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.0940764844417572,
+      "learning_rate": 7.897721300792183e-05,
+      "loss": 0.0064,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.10571976006031036,
+      "learning_rate": 7.897721300792183e-05,
+      "loss": 0.0034,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.08957740664482117,
+      "learning_rate": 7.897721300792183e-05,
+      "loss": 0.003,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.5560760053026955,
+      "eval_e3_equal": 0.8172337604949182,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.989924878479894,
+      "eval_e3_heavy_atoms_difference": 0.5365444100751215,
+      "eval_e3_heavy_atoms_difference_norm": 0.011299086189019656,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.989924878479894,
+      "eval_has_all_attachment_points": 0.9816173221387539,
+      "eval_has_three_substructures": 0.9993813521873619,
+      "eval_heavy_atoms_difference": 6.706407423773752,
+      "eval_heavy_atoms_difference_norm": 0.08881506981006522,
+      "eval_linker_equal": 0.782235969951392,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9935483870967742,
+      "eval_linker_heavy_atoms_difference": 0.3836500220945647,
+      "eval_linker_heavy_atoms_difference_norm": 0.003922412538702962,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9935483870967742,
+      "eval_loss": 0.30979764461517334,
+      "eval_num_fragments": 2.999734865223155,
+      "eval_poi_equal": 0.7688024745912505,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9324790101634998,
+      "eval_poi_heavy_atoms_difference": 1.9796730004418912,
+      "eval_poi_heavy_atoms_difference_norm": 0.0652935121657418,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9324790101634998,
+      "eval_reassembly": 0.563323022536456,
+      "eval_reassembly_nostereo": 0.5840919133893062,
+      "eval_runtime": 1489.7161,
+      "eval_samples_per_second": 7.595,
+      "eval_steps_per_second": 0.119,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.919222271321255,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.10187249630689621,
+      "learning_rate": 7.897721300792183e-05,
+      "loss": 0.0031,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.07569529116153717,
+      "learning_rate": 7.897721300792183e-05,
+      "loss": 0.0034,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.07957209646701813,
+      "learning_rate": 7.897721300792183e-05,
+      "loss": 0.0031,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.07916957885026932,
+      "learning_rate": 7.897721300792183e-05,
+      "loss": 0.0028,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.06219175085425377,
+      "learning_rate": 7.897721300792183e-05,
+      "loss": 0.003,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.5534246575342465,
+      "eval_e3_equal": 0.8113124171453823,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9819708351745471,
+      "eval_e3_heavy_atoms_difference": 0.6880247459125055,
+      "eval_e3_heavy_atoms_difference_norm": 0.015301653091196767,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9819708351745471,
+      "eval_has_all_attachment_points": 0.9765797613787008,
+      "eval_has_three_substructures": 0.999116217410517,
+      "eval_heavy_atoms_difference": 8.727883340698188,
+      "eval_heavy_atoms_difference_norm": 0.1127822284643258,
+      "eval_linker_equal": 0.7893946089262042,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9933716305788776,
+      "eval_linker_heavy_atoms_difference": 0.420415377817057,
+      "eval_linker_heavy_atoms_difference_norm": 0.006358583874060475,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9933716305788776,
+      "eval_loss": 0.31512290239334106,
+      "eval_num_fragments": 3.000353513035793,
+      "eval_poi_equal": 0.7712770658418029,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9185152452496685,
+      "eval_poi_heavy_atoms_difference": 2.5423773751657093,
+      "eval_poi_heavy_atoms_difference_norm": 0.07831832886442663,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9185152452496685,
+      "eval_reassembly": 0.5626159964648696,
+      "eval_reassembly_nostereo": 0.5887759611135661,
+      "eval_runtime": 1512.8952,
+      "eval_samples_per_second": 7.479,
+      "eval_steps_per_second": 0.117,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.8976579761378701,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=7-learning_rate=7.9e-05-warmup_ratio=0.060-min_lr=0.000-factor=0.920",
   "trial_params": {