Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2d498b6989586a4df92f1e0058ec1bced211ce2db37118da1fd2cf0da27376c
+oid sha256:f676607cd0397b1f5d8459ac2d3832aa6dc942f817980077bd2b7bdb170a2299
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cb16037b50c4447d44ee30bff4ea3eeccfdf9c924946a1bbbb713c2c41e61c56
+oid sha256:f7d11f3a46c7c00825c3eadc2dbdf53e3b820ad724200e9c0387872d22ba4248
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b2d04eddfc7b6b071b8d646acad56be885bfe31cd3f719a42e4e188063e0e48
+oid sha256:6de728555fabd8cf02c37f2038f990b0194c12700f952b8a8fbf3c13da0cdf9d
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:415f69ad45e35e2c658f3eb70a29da7d1c48b1525f93317c1722ac741ba10d64
+oid sha256:850c0f67bc5e910c481d6d749e2efd06031ffcdc3209d834462db0427281420d
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.5517454706142289,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=18-learning_rate=4.6e-05-warmup_ratio=0.040-min_lr=0.000-factor=0.820/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.5199292973928413,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25/trial-number=19-learning_rate=8.0e-05-warmup_ratio=0.060-min_lr=0.000-factor=0.970/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.6691174507141113,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.9843,
+      "grad_norm": 0.48272669315338135,
+      "learning_rate": 8.027431614642607e-05,
+      "loss": 0.8167,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.477989137172699,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.1813,
+      "grad_norm": 0.35677370429039,
+      "learning_rate": 8.027431614642607e-05,
+      "loss": 0.1099,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.28420594334602356,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0718,
+      "grad_norm": 0.28323549032211304,
+      "learning_rate": 8.027431614642607e-05,
+      "loss": 0.0399,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.27486521005630493,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0395,
+      "grad_norm": 0.18444126844406128,
+      "learning_rate": 8.027431614642607e-05,
+      "loss": 0.022,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.2175276130437851,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0251,
+      "grad_norm": 0.1738751381635666,
+      "learning_rate": 8.027431614642607e-05,
+      "loss": 0.0141,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.1922473907470703,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0177,
+      "grad_norm": 0.14796946942806244,
+      "learning_rate": 8.027431614642607e-05,
+      "loss": 0.0107,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.18892481923103333,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0139,
+      "grad_norm": 0.13383328914642334,
+      "learning_rate": 8.027431614642607e-05,
+      "loss": 0.0089,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.1161535456776619,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0111,
+      "grad_norm": 0.10535630583763123,
+      "learning_rate": 8.027431614642607e-05,
+      "loss": 0.0073,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.16154514253139496,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0088,
+      "grad_norm": 0.11387604475021362,
+      "learning_rate": 8.027431614642607e-05,
+      "loss": 0.0062,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.16088169813156128,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0078,
+      "grad_norm": 0.11770522594451904,
+      "learning_rate": 8.027431614642607e-05,
+      "loss": 0.0059,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.5058771542200619,
-      "eval_e3_equal": 0.7961113566062749,
+      "eval_all_ligands_equal": 0.5199292973928413,
+      "eval_e3_equal": 0.7980556783031374,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9840919133893062,
-      "eval_e3_heavy_atoms_difference": 0.6767123287671233,
-      "eval_e3_heavy_atoms_difference_norm": 0.015499968999404092,
+      "eval_e3_has_attachment_point(s)": 0.976049491825011,
+      "eval_e3_heavy_atoms_difference": 0.9379584622182943,
+      "eval_e3_heavy_atoms_difference_norm": 0.02296914272883557,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9840919133893062,
-      "eval_has_all_attachment_points": 0.9769332744144941,
-      "eval_has_three_substructures": 0.9985859478568272,
-      "eval_heavy_atoms_difference": 7.977021652673442,
-      "eval_heavy_atoms_difference_norm": 0.105252234591077,
-      "eval_linker_equal": 0.7311533362792753,
+      "eval_e3_valid": 0.976049491825011,
+      "eval_has_all_attachment_points": 0.9775519222271322,
+      "eval_has_three_substructures": 0.9992045956694653,
+      "eval_heavy_atoms_difference": 8.915245249668581,
+      "eval_heavy_atoms_difference_norm": 0.11359321330061124,
+      "eval_linker_equal": 0.7530711444984534,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9948740609809986,
-      "eval_linker_heavy_atoms_difference": 0.3779938135218736,
-      "eval_linker_heavy_atoms_difference_norm": 0.0009991271739602798,
+      "eval_linker_has_attachment_point(s)": 0.9946973044631021,
+      "eval_linker_heavy_atoms_difference": 0.3158638974812196,
+      "eval_linker_heavy_atoms_difference_norm": -0.005294061039033342,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9948740609809986,
-      "eval_loss": 0.2940564453601837,
-      "eval_num_fragments": 3.0001767565178965,
-      "eval_poi_equal": 0.7464427750773309,
+      "eval_linker_valid": 0.9946973044631021,
+      "eval_loss": 0.29772570729255676,
+      "eval_num_fragments": 2.9995581087052585,
+      "eval_poi_equal": 0.7513919575784357,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9214317277949624,
-      "eval_poi_heavy_atoms_difference": 2.363411400795404,
-      "eval_poi_heavy_atoms_difference_norm": 0.07635871742465364,
+      "eval_poi_has_attachment_point(s)": 0.9204595669465312,
+      "eval_poi_heavy_atoms_difference": 2.561201944321697,
+      "eval_poi_heavy_atoms_difference_norm": 0.07901894293442624,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9214317277949624,
-      "eval_reassembly": 0.5125939019001325,
-      "eval_reassembly_nostereo": 0.5362792752982766,
-      "eval_runtime": 1595.0598,
-      "eval_samples_per_second": 7.094,
-      "eval_steps_per_second": 0.111,
+      "eval_poi_valid": 0.9204595669465312,
+      "eval_reassembly": 0.5268228015908086,
+      "eval_reassembly_nostereo": 0.5501546619531595,
+      "eval_runtime": 1487.2402,
+      "eval_samples_per_second": 7.608,
+      "eval_steps_per_second": 0.119,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9042863455589925,
+      "eval_valid": 0.8945647370746797,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.14987365901470184,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0065,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.09952245652675629,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0059,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.09798336774110794,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0055,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.09167756885290146,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0048,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.0671759694814682,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0048,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.5443216968625718,
-      "eval_e3_equal": 0.8132567388422448,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9893946089262041,
-      "eval_e3_heavy_atoms_difference": 0.5114449845338047,
-      "eval_e3_heavy_atoms_difference_norm": 0.013358856534083323,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9893946089262041,
-      "eval_has_all_attachment_points": 0.982235969951392,
-      "eval_has_three_substructures": 0.9985859478568272,
-      "eval_heavy_atoms_difference": 8.227043747238179,
-      "eval_heavy_atoms_difference_norm": 0.10852013342695678,
-      "eval_linker_equal": 0.7702165267344233,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9939019001325674,
-      "eval_linker_heavy_atoms_difference": 0.434025629695095,
-      "eval_linker_heavy_atoms_difference_norm": 0.006870356807576514,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9939019001325674,
-      "eval_loss": 0.30229106545448303,
-      "eval_num_fragments": 3.0007070260715865,
-      "eval_poi_equal": 0.7606716747680071,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9114449845338047,
-      "eval_poi_heavy_atoms_difference": 2.584003535130358,
-      "eval_poi_heavy_atoms_difference_norm": 0.08180244640712389,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9114449845338047,
-      "eval_reassembly": 0.552364118426867,
-      "eval_reassembly_nostereo": 0.5768448961555457,
-      "eval_runtime": 1501.0589,
-      "eval_samples_per_second": 7.538,
-      "eval_steps_per_second": 0.118,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8990720282810428,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.10520212352275848,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0036,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.11396338790655136,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0036,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.09909546375274658,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0034,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.08407113701105118,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0032,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.0803075060248375,
-      "learning_rate": 4.606381292021688e-05,
-      "loss": 0.0033,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.5517454706142289,
-      "eval_e3_equal": 0.8123729562527618,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9821475916924437,
-      "eval_e3_heavy_atoms_difference": 0.7959346000883782,
-      "eval_e3_heavy_atoms_difference_norm": 0.0223010786971663,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9821475916924437,
-      "eval_has_all_attachment_points": 0.9794962439239947,
-      "eval_has_three_substructures": 0.9982324348210341,
-      "eval_heavy_atoms_difference": 8.843128590366769,
-      "eval_heavy_atoms_difference_norm": 0.1164947202784641,
-      "eval_linker_equal": 0.7893062306672558,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9941670349094123,
-      "eval_linker_heavy_atoms_difference": 0.4202386212991604,
-      "eval_linker_heavy_atoms_difference_norm": 0.007723105851183938,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9941670349094123,
-      "eval_loss": 0.31081581115722656,
-      "eval_num_fragments": 2.9998232434821035,
-      "eval_poi_equal": 0.7584622182942996,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9136544410075121,
-      "eval_poi_heavy_atoms_difference": 2.7370746796288112,
-      "eval_poi_heavy_atoms_difference_norm": 0.08879554023759309,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9136544410075121,
-      "eval_reassembly": 0.5607600530269554,
-      "eval_reassembly_nostereo": 0.5870967741935483,
-      "eval_runtime": 1541.0576,
-      "eval_samples_per_second": 7.342,
-      "eval_steps_per_second": 0.115,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8950950066283694,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=18-learning_rate=4.6e-05-warmup_ratio=0.040-min_lr=0.000-factor=0.820",
+  "trial_name": "trial-number=19-learning_rate=8.0e-05-warmup_ratio=0.060-min_lr=0.000-factor=0.970",
   "trial_params": {
-    "factor": 0.8200000000000001,
-    "learning_rate": 4.606381292021688e-05,
-    "min_lr": 9.570957186046748e-10,
-    "warmup_ratio": 0.04
+    "factor": 0.9700000000000001,
+    "learning_rate": 8.027431614642607e-05,
+    "min_lr": 7.655589150964243e-09,
+    "warmup_ratio": 0.06
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e04b41249f2af71631a96b3a173f413442176c6aa411f0cd997e7794a68d4b4
+oid sha256:8fd9e4a725db2ab30f7bf911ea9e6be239c00d1942a0bc3faad39f344c30b894
 size 7480