Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6190de4856e64d34d586b21681ba9266183ad59856a4b97b3789dc5b36e69d5
+oid sha256:acca90dc85a7dcb0ba251a8e8cac6eea7ccdca04e38b2e500a1f591ed64b1478
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:947f1171e40a48f03fbbc59ec258a1515761f14874494cbd31ad30c6941a095b
+oid sha256:d678f0ed7e583beb3f95a509225c01d02fd13d7385926edd383f52c78e4c86f2
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5370136260b11eacaa32121ff0058da294c89a8bf8b55e542cc23877768e4b55
+oid sha256:5b83e9a85ec588125dfd8f4697c82345d8aacf0f84a2d2063373a3801fdc77c4
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8012529a3e966d06ac12c1db15f08af55229902300d42ae95c7b048e773619c1
+oid sha256:d925c5fe16ac296f1170b611f090ec75101b814c00f93070017194f62dd64053
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=7-learning_rate=7.8e-04-warmup_ratio=0.090-num_cycles=2.000/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 8.837825894829872e-05,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=8-learning_rate=6.7e-06-warmup_ratio=0.040-num_cycles=6.500/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.7592541575431824,
-      "learning_rate": 0.0005572089662714916,
-      "loss": 1.0846,
+      "grad_norm": 1.5104084014892578,
+      "learning_rate": 4.764287234716251e-06,
+      "loss": 4.424,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.3683110773563385,
-      "learning_rate": 0.0007472119550121175,
-      "loss": 0.2071,
+      "grad_norm": 1.2274954319000244,
+      "learning_rate": 4.151261952929334e-06,
+      "loss": 1.1134,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.21385785937309265,
-      "learning_rate": 0.0005723056787112626,
-      "loss": 0.1756,
+      "grad_norm": 1.3203201293945312,
+      "learning_rate": 2.321578953844436e-07,
+      "loss": 0.7825,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.24777494370937347,
-      "learning_rate": 0.00031709545642021487,
-      "loss": 0.1621,
+      "grad_norm": 1.239029049873352,
+      "learning_rate": 6.133081099363204e-06,
+      "loss": 0.7182,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.36386048793792725,
-      "learning_rate": 9.368482054050638e-05,
-      "loss": 0.1542,
+      "grad_norm": 1.1526330709457397,
+      "learning_rate": 3.1497864683804228e-06,
+      "loss": 0.5425,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.40356898307800293,
-      "learning_rate": 2.0903083311035306e-07,
-      "loss": 0.1496,
+      "grad_norm": 1.0190362930297852,
+      "learning_rate": 7.384678968603421e-07,
+      "loss": 0.4601,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.3880619704723358,
-      "learning_rate": 7.772822245308111e-05,
-      "loss": 0.1487,
+      "grad_norm": 0.932418167591095,
+      "learning_rate": 6.542318154539015e-06,
+      "loss": 0.4294,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.23959334194660187,
-      "learning_rate": 0.0002921913486535547,
-      "loss": 0.1535,
+      "grad_norm": 0.9513956308364868,
+      "learning_rate": 2.164785723897086e-06,
+      "loss": 0.3502,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.24188180267810822,
-      "learning_rate": 0.0005484331729874041,
-      "loss": 0.185,
+      "grad_norm": 0.7861459851264954,
+      "learning_rate": 1.481401038093183e-06,
+      "loss": 0.3145,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.34928059577941895,
-      "learning_rate": 0.0007358760132057919,
-      "loss": 0.197,
+      "grad_norm": 0.8846765160560608,
+      "learning_rate": 6.658298821578928e-06,
+      "loss": 0.2958,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.0207688908528502,
+      "eval_all_ligands_equal": 8.837825894829872e-05,
+      "eval_e3_equal": 0.1581970835174547,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": 9.30446310207689,
-      "eval_e3_heavy_atoms_difference_norm": 0.26547885791232195,
+      "eval_e3_has_attachment_point(s)": 0.991427308882015,
+      "eval_e3_heavy_atoms_difference": -1.1548387096774193,
+      "eval_e3_heavy_atoms_difference_norm": -0.07234321264327499,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": 20.35103844454264,
-      "eval_heavy_atoms_difference_norm": 0.2403502406440708,
-      "eval_linker_equal": 0.0014140521431727795,
+      "eval_e3_valid": 0.991427308882015,
+      "eval_has_all_attachment_points": 0.9049049933716306,
+      "eval_has_three_substructures": 0.9924878479893946,
+      "eval_heavy_atoms_difference": 32.90561201944322,
+      "eval_heavy_atoms_difference_norm": 0.4163487360428623,
+      "eval_linker_equal": 0.0030048608042421566,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": 2.574193548387097,
-      "eval_linker_heavy_atoms_difference_norm": -0.04056603815230894,
+      "eval_linker_has_attachment_point(s)": 0.9836500220945648,
+      "eval_linker_heavy_atoms_difference": 5.734423331860363,
+      "eval_linker_heavy_atoms_difference_norm": 0.2366558550935315,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.6105839610099792,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0022978347326557667,
+      "eval_linker_valid": 0.9836500220945648,
+      "eval_loss": 0.42221882939338684,
+      "eval_num_fragments": 2.9930181175430843,
+      "eval_poi_equal": 0.09889527176314626,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9997348652231551,
-      "eval_poi_heavy_atoms_difference": 8.464074237737517,
-      "eval_poi_heavy_atoms_difference_norm": 0.16873850826283157,
+      "eval_poi_has_attachment_point(s)": 0.6477242598320813,
+      "eval_poi_heavy_atoms_difference": 13.377110030932391,
+      "eval_poi_heavy_atoms_difference_norm": 0.38139117011179696,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9997348652231551,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1030.7859,
-      "eval_samples_per_second": 10.977,
-      "eval_steps_per_second": 0.172,
+      "eval_poi_valid": 0.6477242598320813,
+      "eval_reassembly": 8.837825894829872e-05,
+      "eval_reassembly_nostereo": 8.837825894829872e-05,
+      "eval_runtime": 1551.651,
+      "eval_samples_per_second": 7.292,
+      "eval_steps_per_second": 0.114,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9997348652231551,
+      "eval_valid": 0.6431285903667697,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.27507615089416504,
-      "learning_rate": 0.000771165072704306,
-      "loss": 0.1772,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.23323027789592743,
-      "learning_rate": 0.0006387992943959788,
-      "loss": 0.165,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.23379911482334137,
-      "learning_rate": 0.000396921613090527,
-      "loss": 0.1559,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.31352663040161133,
-      "learning_rate": 0.00015177911497445075,
-      "loss": 0.15,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.30255958437919617,
-      "learning_rate": 1.1052988261315712e-05,
-      "loss": 0.1463,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.004860804242156429,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": -5.131506849315069,
-      "eval_e3_heavy_atoms_difference_norm": -0.299063499116105,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": -20.02129916040654,
-      "eval_heavy_atoms_difference_norm": -0.314300300668607,
-      "eval_linker_equal": 0.0016791869200176756,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 1.0,
-      "eval_linker_heavy_atoms_difference": -3.839681838267786,
-      "eval_linker_heavy_atoms_difference_norm": -0.5210048047300356,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 1.0,
-      "eval_loss": 0.49704626202583313,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0027397260273972603,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9993813521873619,
-      "eval_poi_heavy_atoms_difference": -11.08157313300928,
-      "eval_poi_heavy_atoms_difference_norm": -0.5551499346711933,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9993813521873619,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1804.3653,
-      "eval_samples_per_second": 6.271,
-      "eval_steps_per_second": 0.098,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9993813521873619,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.2914905250072479,
-      "learning_rate": 3.6558530802854075e-05,
-      "loss": 0.1451,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.2859417796134949,
-      "learning_rate": 0.00021709218895981126,
-      "loss": 0.1466,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.3361692726612091,
-      "learning_rate": 0.00047335282575153254,
-      "loss": 0.1634,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 1.0404810905456543,
-      "learning_rate": 0.0006927755041550175,
-      "loss": 0.1679,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.3088153898715973,
-      "learning_rate": 0.0007789767128938879,
-      "loss": 0.1925,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.008572691117984976,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 1.0,
-      "eval_e3_heavy_atoms_difference": -5.5740167918692,
-      "eval_e3_heavy_atoms_difference_norm": -0.3162344909098186,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 1.0,
-      "eval_has_all_attachment_points": 1.0,
-      "eval_has_three_substructures": 1.0,
-      "eval_heavy_atoms_difference": -0.8163499779054353,
-      "eval_heavy_atoms_difference_norm": -0.049597444879744636,
-      "eval_linker_equal": 0.001590808661069377,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9997348652231551,
-      "eval_linker_heavy_atoms_difference": -0.4775961113566063,
-      "eval_linker_heavy_atoms_difference_norm": -0.27010033853887316,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9997348652231551,
-      "eval_loss": 0.5193294286727905,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.0016791869200176756,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9573133009279717,
-      "eval_poi_heavy_atoms_difference": 3.002032699955811,
-      "eval_poi_heavy_atoms_difference_norm": -0.03282844379746269,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9573133009279717,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 1317.9182,
-      "eval_samples_per_second": 8.586,
-      "eval_steps_per_second": 0.134,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9570481661511269,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=7-learning_rate=7.8e-04-warmup_ratio=0.090-num_cycles=2.000",
+  "trial_name": "trial-number=8-learning_rate=6.7e-06-warmup_ratio=0.040-num_cycles=6.500",
   "trial_params": {
-    "learning_rate": 0.0007789781348475454,
-    "num_cycles": 2.0,
-    "warmup_ratio": 0.09
+    "learning_rate": 6.660473554133319e-06,
+    "num_cycles": 6.5,
+    "warmup_ratio": 0.04
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f9f38c124dc6cf9341743e890aae643afe44ee573d89c2473cc804da1310414
+oid sha256:3b0609760c80d39fd60b4b9a6b6bc7aba41bb55723c3dd76d04162efbf7717fa
 size 7480