Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9663ba3ab5044bc90a7b3d1b75f640ff55f2b5c41c418701a4ae83c867c824bb
+oid sha256:848ca9af313856a53fa408bf838552535448daed0213ecd04bae8a6027f45bbe
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72cc2c8c672b182b4bcc420e9df023d1155dae42a4b06a811b042e577bd5ae25
+oid sha256:172819277237957e3a00d9cc349ead604ad6e059c1276e3c37c779350a30604a
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e527a24b032ccf5b3ad099345582f9a572dd9ca9f0a87b20d04145f903aac542
+oid sha256:a85a6fbc133998925e6017d6cc12830910d2733615f99b9c6f66d217c32440ec
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d92c5a2febec6a3debc54e07f631908af44ee341af8b138c2ac3072c6cfbde8
+oid sha256:f9622860b2ff3b52b1dc058cb630ef6381def4ff9274a76cbef21acbf286488a
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=21-learning_rate=1.6e-06-warmup_ratio=0.080-num_cycles=1.500/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.47980556783031375,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=22-learning_rate=7.8e-05-warmup_ratio=0.050-num_cycles=7.000/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 4.178253650665283,
-      "learning_rate": 1.1556051821104526e-06,
-      "loss": 6.3981,
+      "grad_norm": 0.8338723182678223,
+      "learning_rate": 5.546499519621046e-05,
+      "loss": 2.0504,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.9749269485473633,
-      "learning_rate": 1.5782537021726088e-06,
-      "loss": 2.3749,
+      "grad_norm": 0.520139217376709,
+      "learning_rate": 4.446545588867061e-05,
+      "loss": 0.2196,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 1.4726980924606323,
-      "learning_rate": 1.3637510062898264e-06,
-      "loss": 1.4006,
+      "grad_norm": 0.34758761525154114,
+      "learning_rate": 7.815797870732129e-06,
+      "loss": 0.0808,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 1.4031742811203003,
-      "learning_rate": 1.0095554213221014e-06,
-      "loss": 1.1642,
+      "grad_norm": 0.3090643882751465,
+      "learning_rate": 7.720759640693211e-05,
+      "loss": 0.0659,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 1.421210527420044,
-      "learning_rate": 6.046599668215422e-07,
-      "loss": 1.0397,
+      "grad_norm": 0.20644567906856537,
+      "learning_rate": 1.4921919604851097e-05,
+      "loss": 0.0344,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 1.4061120748519897,
-      "learning_rate": 2.5079620192718024e-07,
-      "loss": 0.9784,
+      "grad_norm": 0.23131480813026428,
+      "learning_rate": 3.433390507333742e-05,
+      "loss": 0.0214,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 1.4008420705795288,
-      "learning_rate": 3.687377510596774e-08,
-      "loss": 0.9519,
+      "grad_norm": 0.22179295122623444,
+      "learning_rate": 6.894294973215218e-05,
+      "loss": 0.0231,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 1.39585280418396,
-      "learning_rate": 1.664152886385891e-08,
-      "loss": 0.9482,
+      "grad_norm": 0.11722031235694885,
+      "learning_rate": 1.8712978537945652e-07,
+      "loss": 0.0131,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 1.3957849740982056,
-      "learning_rate": 1.9518289383020806e-07,
-      "loss": 0.9436,
+      "grad_norm": 0.16297303140163422,
+      "learning_rate": 6.360667367381643e-05,
+      "loss": 0.0105,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 1.4641075134277344,
-      "learning_rate": 5.276386570231308e-07,
-      "loss": 0.9277,
+      "grad_norm": 0.12827661633491516,
+      "learning_rate": 4.194209897608062e-05,
+      "loss": 0.0142,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.023243482103402564,
+      "eval_all_ligands_equal": 0.47980556783031375,
+      "eval_e3_equal": 0.7869200176756518,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.7854175872735307,
-      "eval_e3_heavy_atoms_difference": 7.005744586831639,
-      "eval_e3_heavy_atoms_difference_norm": 0.19202268846027856,
+      "eval_e3_has_attachment_point(s)": 0.9908970393283252,
+      "eval_e3_heavy_atoms_difference": 0.4931506849315068,
+      "eval_e3_heavy_atoms_difference_norm": 0.00838361371746141,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.7854175872735307,
-      "eval_has_all_attachment_points": 0.4845779938135219,
-      "eval_has_three_substructures": 0.7926646045072913,
-      "eval_heavy_atoms_difference": 71.70234202386213,
-      "eval_heavy_atoms_difference_norm": 0.9403941983073605,
-      "eval_linker_equal": 0.0007954043305346885,
+      "eval_e3_valid": 0.9908970393283252,
+      "eval_has_all_attachment_points": 0.984887317719841,
+      "eval_has_three_substructures": 0.9995581087052585,
+      "eval_heavy_atoms_difference": 7.6450729120636325,
+      "eval_heavy_atoms_difference_norm": 0.09998915701426427,
+      "eval_linker_equal": 0.6953601414052143,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.7095006628369421,
-      "eval_linker_heavy_atoms_difference": 11.820945647370747,
-      "eval_linker_heavy_atoms_difference_norm": 0.6561966279132151,
+      "eval_linker_has_attachment_point(s)": 0.9977905435262925,
+      "eval_linker_heavy_atoms_difference": 0.7192222713212549,
+      "eval_linker_heavy_atoms_difference_norm": 0.018969491788470037,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.7095006628369421,
-      "eval_loss": 0.9539049863815308,
-      "eval_num_fragments": 3.1619089703932834,
-      "eval_poi_equal": 0.0,
+      "eval_linker_valid": 0.9977905435262925,
+      "eval_loss": 0.28961092233657837,
+      "eval_num_fragments": 3.000265134776845,
+      "eval_poi_equal": 0.7358373840035352,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.15510384445426426,
-      "eval_poi_heavy_atoms_difference": 29.169686257180732,
-      "eval_poi_heavy_atoms_difference_norm": 0.9225394246917481,
+      "eval_poi_has_attachment_point(s)": 0.9250552364118427,
+      "eval_poi_heavy_atoms_difference": 2.3874502872293415,
+      "eval_poi_heavy_atoms_difference_norm": 0.07483521916137179,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.15510384445426426,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 2245.7826,
-      "eval_samples_per_second": 5.038,
-      "eval_steps_per_second": 0.079,
+      "eval_poi_valid": 0.9250552364118427,
+      "eval_reassembly": 0.4869642068051259,
+      "eval_reassembly_nostereo": 0.5148917366327883,
+      "eval_runtime": 1495.7927,
+      "eval_samples_per_second": 7.565,
+      "eval_steps_per_second": 0.118,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.14653115333627928,
+      "eval_valid": 0.9155987627043747,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 1.294830083847046,
-      "learning_rate": 9.304780137412219e-07,
-      "loss": 0.8837,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 1.3187594413757324,
-      "learning_rate": 1.302486007417367e-06,
-      "loss": 0.8177,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 1.3196697235107422,
-      "learning_rate": 1.5501941814704978e-06,
-      "loss": 0.739,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 1.3168965578079224,
-      "learning_rate": 1.6113648928799203e-06,
-      "loss": 0.6635,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 1.1013790369033813,
-      "learning_rate": 1.4706287622758396e-06,
-      "loss": 0.598,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.06637207247017234,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9723376049491825,
-      "eval_e3_heavy_atoms_difference": 0.5682722050375607,
-      "eval_e3_heavy_atoms_difference_norm": -0.02346762473633578,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9723376049491825,
-      "eval_has_all_attachment_points": 0.7742819266460451,
-      "eval_has_three_substructures": 0.9734865223155104,
-      "eval_heavy_atoms_difference": 61.96756517896598,
-      "eval_heavy_atoms_difference_norm": 0.8021929226054672,
-      "eval_linker_equal": 0.0015024304021210783,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.943791427308882,
-      "eval_linker_heavy_atoms_difference": 8.018294299602298,
-      "eval_linker_heavy_atoms_difference_norm": 0.38053119346826625,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.943791427308882,
-      "eval_loss": 0.6559651494026184,
-      "eval_num_fragments": 3.012372956252762,
-      "eval_poi_equal": 0.0026513477684489617,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.2973928413610252,
-      "eval_poi_heavy_atoms_difference": 25.526115775519223,
-      "eval_poi_heavy_atoms_difference_norm": 0.7772625095843685,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.2973928413610252,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1361.2908,
-      "eval_samples_per_second": 8.312,
-      "eval_steps_per_second": 0.13,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.2877596111356606,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 1.2238191366195679,
-      "learning_rate": 1.1633462903366549e-06,
-      "loss": 0.5506,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 1.3292711973190308,
-      "learning_rate": 7.667233943405157e-07,
-      "loss": 0.5139,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 1.052363634109497,
-      "learning_rate": 3.804131208984208e-07,
-      "loss": 0.4937,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 1.1418038606643677,
-      "learning_rate": 1.014774300592085e-07,
-      "loss": 0.4831,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.9714767932891846,
-      "learning_rate": 0.0,
-      "loss": 0.4799,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.08590366769774635,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9799381352187362,
-      "eval_e3_heavy_atoms_difference": -0.6433937251436147,
-      "eval_e3_heavy_atoms_difference_norm": -0.060812794107813935,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9799381352187362,
-      "eval_has_all_attachment_points": 0.8074237737516571,
-      "eval_has_three_substructures": 0.981352187361909,
-      "eval_heavy_atoms_difference": 55.751480335837385,
-      "eval_heavy_atoms_difference_norm": 0.7168565547640369,
-      "eval_linker_equal": 0.0015024304021210783,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9653557224922669,
-      "eval_linker_heavy_atoms_difference": 7.463190455148034,
-      "eval_linker_heavy_atoms_difference_norm": 0.34341321095264715,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9653557224922669,
-      "eval_loss": 0.5779388546943665,
-      "eval_num_fragments": 3.00415377817057,
-      "eval_poi_equal": 0.011312417145382236,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.371011931064958,
-      "eval_poi_heavy_atoms_difference": 23.019619973486524,
-      "eval_poi_heavy_atoms_difference_norm": 0.6868493188139538,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.371011931064958,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1413.0201,
-      "eval_samples_per_second": 8.008,
-      "eval_steps_per_second": 0.125,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.3641184268669907,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=21-learning_rate=1.6e-06-warmup_ratio=0.080-num_cycles=1.500",
+  "trial_name": "trial-number=22-learning_rate=7.8e-05-warmup_ratio=0.050-num_cycles=7.000",
   "trial_params": {
-    "learning_rate": 1.6155360445904128e-06,
-    "num_cycles": 1.5,
-    "warmup_ratio": 0.08
+    "learning_rate": 7.754006328430222e-05,
+    "num_cycles": 7.0,
+    "warmup_ratio": 0.05
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:840779dc69cef9253cfbecd1665a91f7f3e4ecfabfa209711398b6be673cceed
+oid sha256:11eaea15c2a35b1a070cf894a39e7bc90d63d5cf9b469b3d9d6752ab49318484
 size 7416