Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d17eba1d4428ff7f75170a2fd7218311fc090b5313a3b06432bc714a80f4ca87
+oid sha256:2655a7c5a6f3b44280a8b2d96529887f85baa3433f1588eb504291c7bb228836
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2cd717910fab37a42e66b8e23bd3807dc15ac6f75a947703418ac9d703eccc3f
+oid sha256:cb06b002543c91ea316a35c24a2821d6397a7c06b11933b2c6ccf7ddbb23effc
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53443eed9703ff20591c804b00994138e91d65f45131a4345c68e1e1a48b80c4
+oid sha256:edfb0b0889cbd2250b35734bcab9cc469c772472998ce7fd321d378f0e09ea2f
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:541251d5a59f16adf59a3ce8eba9361feb886d367a1c21f5e8b0a05c6ce28501
+oid sha256:6092aaf0f5bb75d5e667503b11558e227dd128e7e1ea91b5de70f7cbae1e6e1c
 size 1192

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.00821917808219178,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=24-learning_rate=7.7e-06-warmup_ratio=0.040-min_lr=0.000-factor=0.570/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.1942554131683606,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=24-learning_rate=7.7e-06-warmup_ratio=0.040-min_lr=0.000-factor=0.570/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.8996022978347327,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.4523199796676636,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.1138,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.4032362997531891,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.1022,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.39051586389541626,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.0913,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.4568197429180145,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.0824,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.4104657769203186,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.0734,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.09226690234202387,
+      "eval_e3_equal": 0.5859478568272205,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9982324348210341,
+      "eval_e3_heavy_atoms_difference": -0.40927971718957135,
+      "eval_e3_heavy_atoms_difference_norm": -0.028732096583262583,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9982324348210341,
+      "eval_has_all_attachment_points": 0.9905435262925321,
+      "eval_has_three_substructures": 0.9989394608926204,
+      "eval_heavy_atoms_difference": 4.900309323906319,
+      "eval_heavy_atoms_difference_norm": 0.06346159241850167,
+      "eval_linker_equal": 0.17065841802916482,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9969951391957579,
+      "eval_linker_heavy_atoms_difference": 0.9197525408749447,
+      "eval_linker_heavy_atoms_difference_norm": -0.004279007320364481,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9969951391957579,
+      "eval_loss": 0.30911630392074585,
+      "eval_num_fragments": 2.999646486964207,
+      "eval_poi_equal": 0.5859478568272205,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9417587273530711,
+      "eval_poi_heavy_atoms_difference": 1.571895713654441,
+      "eval_poi_heavy_atoms_difference_norm": 0.04295033310659916,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9417587273530711,
+      "eval_reassembly": 0.09368095448519664,
+      "eval_reassembly_nostereo": 0.11162174105170128,
+      "eval_runtime": 1561.5722,
+      "eval_samples_per_second": 7.246,
+      "eval_steps_per_second": 0.113,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9394608926204154,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.3263263702392578,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.0658,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.31009504199028015,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.0597,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.3627758324146271,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.0542,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.2838134765625,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.0493,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.3240777254104614,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.0454,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.1942554131683606,
+      "eval_e3_equal": 0.6628369421122404,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.997348652231551,
+      "eval_e3_heavy_atoms_difference": -0.17587273530711445,
+      "eval_e3_heavy_atoms_difference_norm": -0.018808788002665174,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.997348652231551,
+      "eval_has_all_attachment_points": 0.9904551480335837,
+      "eval_has_three_substructures": 0.9993813521873619,
+      "eval_heavy_atoms_difference": 5.831462660185594,
+      "eval_heavy_atoms_difference_norm": 0.07443880795953683,
+      "eval_linker_equal": 0.3456473707467963,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9966416261599647,
+      "eval_linker_heavy_atoms_difference": 0.8881131241714538,
+      "eval_linker_heavy_atoms_difference_norm": 0.011275755194650126,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9966416261599647,
+      "eval_loss": 0.3081931471824646,
+      "eval_num_fragments": 2.9999116217410515,
+      "eval_poi_equal": 0.6042421564295183,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.942907644719399,
+      "eval_poi_heavy_atoms_difference": 2.1913389306230666,
+      "eval_poi_heavy_atoms_difference_norm": 0.06304879570876758,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.942907644719399,
+      "eval_reassembly": 0.19787892178524083,
+      "eval_reassembly_nostereo": 0.23057887759611137,
+      "eval_runtime": 1505.1145,
+      "eval_samples_per_second": 7.518,
+      "eval_steps_per_second": 0.118,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9386654882898807,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.364959997072179e+16,
+  "total_flos": 8.729673191023411e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=24-learning_rate=7.7e-06-warmup_ratio=0.040-min_lr=0.000-factor=0.570",
   "trial_params": {