Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85d494d15e9fec950bb907757580874cee023971b1e8199d17f51819891885e4
+oid sha256:d17eba1d4428ff7f75170a2fd7218311fc090b5313a3b06432bc714a80f4ca87
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:658e99f72bfed205ea7d8fc48f9965df9efd3373a67ca5e05ff48925affdb7e4
+oid sha256:2cd717910fab37a42e66b8e23bd3807dc15ac6f75a947703418ac9d703eccc3f
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21b96083d4a355f271c2e85fba80bbec1893f0ff3ec6a81677b4a6a0eb36dd48
+oid sha256:53443eed9703ff20591c804b00994138e91d65f45131a4345c68e1e1a48b80c4
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5fbe293b04d7589da67f35a38d8078b1d8ca015aff4a1a221bd5312708f58432
+oid sha256:541251d5a59f16adf59a3ce8eba9361feb886d367a1c21f5e8b0a05c6ce28501
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.5573133009279717,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=23-learning_rate=9.9e-05-warmup_ratio=0.070-min_lr=0.000-factor=0.120/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.00821917808219178,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=24-learning_rate=7.7e-06-warmup_ratio=0.040-min_lr=0.000-factor=0.570/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.6064334511756897,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.7751,
+      "grad_norm": 1.0838514566421509,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 1.9521,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.31307706236839294,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.1148,
+      "grad_norm": 1.154284954071045,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.727,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.19195418059825897,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0417,
+      "grad_norm": 1.0525119304656982,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.4868,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.19379402697086334,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0241,
+      "grad_norm": 0.8092649579048157,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.3584,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.13165819644927979,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0166,
+      "grad_norm": 0.8515424728393555,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.2778,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.1423509120941162,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0129,
+      "grad_norm": 0.5962498784065247,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.2258,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.14043006300926208,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0102,
+      "grad_norm": 0.44913268089294434,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.1889,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.12152795493602753,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0094,
+      "grad_norm": 0.5444086194038391,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.1641,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.10019189119338989,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.009,
+      "grad_norm": 0.48740506172180176,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.1436,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.09819556027650833,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0069,
+      "grad_norm": 0.4699327051639557,
+      "learning_rate": 7.729384753693693e-06,
+      "loss": 0.1276,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.5213433495360141,
-      "eval_e3_equal": 0.8005302695536898,
+      "eval_all_ligands_equal": 0.00821917808219178,
+      "eval_e3_equal": 0.426336721166593,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9891294741493593,
-      "eval_e3_heavy_atoms_difference": 0.4190897039328325,
-      "eval_e3_heavy_atoms_difference_norm": 0.008621549797286895,
+      "eval_e3_has_attachment_point(s)": 0.9975254087494476,
+      "eval_e3_heavy_atoms_difference": -0.7306230667255855,
+      "eval_e3_heavy_atoms_difference_norm": -0.043387643547083826,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9891294741493593,
-      "eval_has_all_attachment_points": 0.9900132567388422,
-      "eval_has_three_substructures": 0.9992929739284137,
-      "eval_heavy_atoms_difference": 4.418559434379143,
-      "eval_heavy_atoms_difference_norm": 0.05692426841147494,
-      "eval_linker_equal": 0.7637649138311975,
+      "eval_e3_valid": 0.9975254087494476,
+      "eval_has_all_attachment_points": 0.9786124613345117,
+      "eval_has_three_substructures": 0.9981440565620857,
+      "eval_heavy_atoms_difference": 10.570216526734423,
+      "eval_heavy_atoms_difference_norm": 0.13624537211132287,
+      "eval_linker_equal": 0.028369421122403888,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9954043305346885,
-      "eval_linker_heavy_atoms_difference": 0.4836058329650906,
-      "eval_linker_heavy_atoms_difference_norm": 0.006199250631660274,
+      "eval_linker_has_attachment_point(s)": 0.9939019001325674,
+      "eval_linker_heavy_atoms_difference": 3.5113566062748562,
+      "eval_linker_heavy_atoms_difference_norm": 0.143718666575472,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9954043305346885,
-      "eval_loss": 0.31019213795661926,
-      "eval_num_fragments": 2.999646486964207,
-      "eval_poi_equal": 0.755899248784799,
+      "eval_linker_valid": 0.9939019001325674,
+      "eval_loss": 0.33331799507141113,
+      "eval_num_fragments": 2.9986743261157756,
+      "eval_poi_equal": 0.45223155103844453,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9647370746796288,
-      "eval_poi_heavy_atoms_difference": 1.0888201502430401,
-      "eval_poi_heavy_atoms_difference_norm": 0.026170453342413177,
+      "eval_poi_has_attachment_point(s)": 0.9035793194874061,
+      "eval_poi_heavy_atoms_difference": 3.2608042421564294,
+      "eval_poi_heavy_atoms_difference_norm": 0.0900747544488257,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9647370746796288,
-      "eval_reassembly": 0.5285019885108263,
-      "eval_reassembly_nostereo": 0.5531595227574017,
-      "eval_runtime": 1519.1579,
-      "eval_samples_per_second": 7.448,
-      "eval_steps_per_second": 0.117,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9515687140963323,
-      "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.11416710168123245,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.007,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.09157449007034302,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.006,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.0833706334233284,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0055,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.09237861633300781,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0055,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.11140145361423492,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0061,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.5434379142730889,
-      "eval_e3_equal": 0.8167918692001768,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9913389306230668,
-      "eval_e3_heavy_atoms_difference": 0.42456915598762707,
-      "eval_e3_heavy_atoms_difference_norm": 0.010198341107667014,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9913389306230668,
-      "eval_has_all_attachment_points": 0.9916040653999116,
-      "eval_has_three_substructures": 0.9994697304463102,
-      "eval_heavy_atoms_difference": 5.391604065399911,
-      "eval_heavy_atoms_difference_norm": 0.06824124017588187,
-      "eval_linker_equal": 0.7818824569155988,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9948740609809986,
-      "eval_linker_heavy_atoms_difference": 0.521873619089704,
-      "eval_linker_heavy_atoms_difference_norm": 0.01070563611327301,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9948740609809986,
-      "eval_loss": 0.315396785736084,
-      "eval_num_fragments": 2.99946973044631,
-      "eval_poi_equal": 0.7551038444542643,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9546619531595227,
-      "eval_poi_heavy_atoms_difference": 1.4819266460450728,
-      "eval_poi_heavy_atoms_difference_norm": 0.03638325355870011,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9546619531595227,
-      "eval_reassembly": 0.5498895271763147,
-      "eval_reassembly_nostereo": 0.5710119310649581,
-      "eval_runtime": 1602.6716,
-      "eval_samples_per_second": 7.06,
-      "eval_steps_per_second": 0.11,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9425541316836058,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.09053564816713333,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0043,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.10203088074922562,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0041,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.07406218349933624,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.004,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.07578085362911224,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0041,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.06606390327215195,
-      "learning_rate": 9.886132722858021e-05,
-      "loss": 0.0043,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.5573133009279717,
-      "eval_e3_equal": 0.8178524083075563,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9900132567388422,
-      "eval_e3_heavy_atoms_difference": 0.4675209898365002,
-      "eval_e3_heavy_atoms_difference_norm": 0.008863474235619222,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9900132567388422,
-      "eval_has_all_attachment_points": 0.9889527176314626,
-      "eval_has_three_substructures": 0.9999116217410517,
-      "eval_heavy_atoms_difference": 5.755987627043747,
-      "eval_heavy_atoms_difference_norm": 0.07290995804551238,
-      "eval_linker_equal": 0.8055678303137428,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9968183826778613,
-      "eval_linker_heavy_atoms_difference": 0.31842686699072026,
-      "eval_linker_heavy_atoms_difference_norm": 0.0018411549398138093,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9968183826778613,
-      "eval_loss": 0.31360945105552673,
-      "eval_num_fragments": 2.9999116217410515,
-      "eval_poi_equal": 0.7606716747680071,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.944675209898365,
-      "eval_poi_heavy_atoms_difference": 1.8359699513919576,
-      "eval_poi_heavy_atoms_difference_norm": 0.05083063503899151,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.944675209898365,
-      "eval_reassembly": 0.5662395050817499,
-      "eval_reassembly_nostereo": 0.5906319045514803,
-      "eval_runtime": 1537.5704,
-      "eval_samples_per_second": 7.359,
+      "eval_poi_valid": 0.9035793194874061,
+      "eval_reassembly": 0.008484312859036678,
+      "eval_reassembly_nostereo": 0.011489173663278833,
+      "eval_runtime": 1538.0562,
+      "eval_samples_per_second": 7.357,
       "eval_steps_per_second": 0.115,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9333627927529827,
-      "step": 10000
+      "eval_valid": 0.8996022978347327,
+      "step": 5000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=23-learning_rate=9.9e-05-warmup_ratio=0.070-min_lr=0.000-factor=0.120",
+  "trial_name": "trial-number=24-learning_rate=7.7e-06-warmup_ratio=0.040-min_lr=0.000-factor=0.570",
   "trial_params": {
-    "factor": 0.12000000000000001,
-    "learning_rate": 9.886132722858021e-05,
-    "min_lr": 4.9933665355601805e-12,
-    "warmup_ratio": 0.06999999999999999
+    "factor": 0.5700000000000001,
+    "learning_rate": 7.729384753693693e-06,
+    "min_lr": 3.4099230386483844e-10,
+    "warmup_ratio": 0.04
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:029e9519837cd9c702131829d4b707092d76dd219fffcb91e93c74c3cf895f50
+oid sha256:3834bc12963422092c70312d13bbac57fcc1945c5f1a3d131e77cedfa4502dec
 size 7544