Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:398c6b7be6121f3fdc7fda4d60dadc4583e04c06766d912d43ead10e99f89865
+oid sha256:706ecf94177e6cf87fcb75027a006510b226d68f19af7327e5a6bcf27f844cd7
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c65568942d1a55d07888753e1f4d9d5f45ec82e65c0b098610cef6040e7dfe61
+oid sha256:2de826b33b1d8e4ca29542d63625334da5026f6f5ce41f5652afded35508837e
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8dda156ceca660034353fdd5cae9f313ca49257e66482ec74171dea25debd7b6
+oid sha256:18b6e8e623a242f7505772e9983f2c11de7bdaf5d3816334a6ac31ff220c0a62
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16cf5f20a28dfa325b9f37e9a560f2177d69ef5b38261444003d24ebf40416c1
+oid sha256:7d0f18d37de5901dacb817a2bed2d4e6c9e6637a57fb3719a7fbb81d9922296e
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.005391073795846222,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=8-learning_rate=6.7e-06-warmup_ratio=0.040-num_cycles=6.500/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.0007070260715863897,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=9-learning_rate=9.1e-06-warmup_ratio=0.060-num_cycles=2.000/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 1.5104084014892578,
-      "learning_rate": 4.764287234716251e-06,
-      "loss": 4.424,
+      "grad_norm": 1.2336422204971313,
+      "learning_rate": 6.485415158644236e-06,
+      "loss": 4.0227,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.2274954319000244,
-      "learning_rate": 4.151261952929334e-06,
-      "loss": 1.1134,
+      "grad_norm": 1.1131938695907593,
+      "learning_rate": 8.696880404100767e-06,
+      "loss": 0.9587,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 1.3203201293945312,
-      "learning_rate": 2.321578953844436e-07,
-      "loss": 0.7825,
+      "grad_norm": 1.1662838459014893,
+      "learning_rate": 6.661127420343339e-06,
+      "loss": 0.5616,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 1.239029049873352,
-      "learning_rate": 6.133081099363204e-06,
-      "loss": 0.7182,
+      "grad_norm": 0.6989096999168396,
+      "learning_rate": 3.6907081620845236e-06,
+      "loss": 0.4128,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 1.1526330709457397,
-      "learning_rate": 3.1497864683804228e-06,
-      "loss": 0.5425,
+      "grad_norm": 0.7815684676170349,
+      "learning_rate": 1.0904077142438307e-06,
+      "loss": 0.3544,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 1.0190362930297852,
-      "learning_rate": 7.384678968603421e-07,
-      "loss": 0.4601,
+      "grad_norm": 0.8438536524772644,
+      "learning_rate": 2.4329323749923227e-09,
+      "loss": 0.3379,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.932418167591095,
-      "learning_rate": 6.542318154539015e-06,
-      "loss": 0.4294,
+      "grad_norm": 0.66645348072052,
+      "learning_rate": 9.046871508992705e-07,
+      "loss": 0.3344,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.9513956308364868,
-      "learning_rate": 2.164785723897086e-06,
-      "loss": 0.3502,
+      "grad_norm": 0.9242984652519226,
+      "learning_rate": 3.4008465701163275e-06,
+      "loss": 0.3231,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.7861459851264954,
-      "learning_rate": 1.481401038093183e-06,
-      "loss": 0.3145,
+      "grad_norm": 0.8741541504859924,
+      "learning_rate": 6.394449277883536e-06,
+      "loss": 0.2861,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.8846765160560608,
-      "learning_rate": 6.658298821578928e-06,
-      "loss": 0.2958,
+      "grad_norm": 0.7810369729995728,
+      "learning_rate": 8.570526637228443e-06,
+      "loss": 0.2387,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 8.837825894829872e-05,
-      "eval_e3_equal": 0.1581970835174547,
+      "eval_all_ligands_equal": 0.0007070260715863897,
+      "eval_e3_equal": 0.21502430402121078,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.991427308882015,
-      "eval_e3_heavy_atoms_difference": -1.1548387096774193,
-      "eval_e3_heavy_atoms_difference_norm": -0.07234321264327499,
+      "eval_e3_has_attachment_point(s)": 0.9926646045072912,
+      "eval_e3_heavy_atoms_difference": -1.1060539107379586,
+      "eval_e3_heavy_atoms_difference_norm": -0.06672245151567276,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.991427308882015,
-      "eval_has_all_attachment_points": 0.9049049933716306,
-      "eval_has_three_substructures": 0.9924878479893946,
-      "eval_heavy_atoms_difference": 32.90561201944322,
-      "eval_heavy_atoms_difference_norm": 0.4163487360428623,
-      "eval_linker_equal": 0.0030048608042421566,
+      "eval_e3_valid": 0.9926646045072912,
+      "eval_has_all_attachment_points": 0.9172779496243924,
+      "eval_has_three_substructures": 0.994078656650464,
+      "eval_heavy_atoms_difference": 26.261069376933275,
+      "eval_heavy_atoms_difference_norm": 0.33227872726986946,
+      "eval_linker_equal": 0.006981882456915599,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
       "eval_linker_has_attachment_point(s)": 0.9836500220945648,
-      "eval_linker_heavy_atoms_difference": 5.734423331860363,
-      "eval_linker_heavy_atoms_difference_norm": 0.2366558550935315,
+      "eval_linker_heavy_atoms_difference": 4.850022094564737,
+      "eval_linker_heavy_atoms_difference_norm": 0.18214354219664547,
       "eval_linker_tanimoto_similarity": 0.0,
       "eval_linker_valid": 0.9836500220945648,
-      "eval_loss": 0.42221882939338684,
-      "eval_num_fragments": 2.9930181175430843,
-      "eval_poi_equal": 0.09889527176314626,
+      "eval_loss": 0.3841758370399475,
+      "eval_num_fragments": 2.9944321696862573,
+      "eval_poi_equal": 0.18444542642509942,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.6477242598320813,
-      "eval_poi_heavy_atoms_difference": 13.377110030932391,
-      "eval_poi_heavy_atoms_difference_norm": 0.38139117011179696,
+      "eval_poi_has_attachment_point(s)": 0.7239946973044631,
+      "eval_poi_heavy_atoms_difference": 10.227397260273973,
+      "eval_poi_heavy_atoms_difference_norm": 0.2855137049452793,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.6477242598320813,
-      "eval_reassembly": 8.837825894829872e-05,
-      "eval_reassembly_nostereo": 8.837825894829872e-05,
-      "eval_runtime": 1551.651,
-      "eval_samples_per_second": 7.292,
-      "eval_steps_per_second": 0.114,
+      "eval_poi_valid": 0.7239946973044631,
+      "eval_reassembly": 0.0007070260715863897,
+      "eval_reassembly_nostereo": 0.0009721608484312859,
+      "eval_runtime": 1530.6545,
+      "eval_samples_per_second": 7.392,
+      "eval_steps_per_second": 0.116,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.6431285903667697,
+      "eval_valid": 0.7176314626601856,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.6100590825080872,
-      "learning_rate": 1.286188250762569e-06,
-      "loss": 0.2497,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.5636773109436035,
-      "learning_rate": 2.393129059452807e-06,
-      "loss": 0.2339,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.5719656348228455,
-      "learning_rate": 6.4704343053956755e-06,
-      "loss": 0.2191,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.5350475311279297,
-      "learning_rate": 5.942081815962343e-07,
-      "loss": 0.192,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.5770079493522644,
-      "learning_rate": 3.3904130777710025e-06,
-      "loss": 0.1836,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.001237295625276182,
-      "eval_e3_equal": 0.2624834290764472,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9961997348652232,
-      "eval_e3_heavy_atoms_difference": -1.1247017233760495,
-      "eval_e3_heavy_atoms_difference_norm": -0.062236374339759794,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9961997348652232,
-      "eval_has_all_attachment_points": 0.946442775077331,
-      "eval_has_three_substructures": 0.9971718957136544,
-      "eval_heavy_atoms_difference": 20.501016349977906,
-      "eval_heavy_atoms_difference_norm": 0.261852483412523,
-      "eval_linker_equal": 0.008572691117984976,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9900132567388422,
-      "eval_linker_heavy_atoms_difference": 5.009456473707468,
-      "eval_linker_heavy_atoms_difference_norm": 0.20222795268141094,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9900132567388422,
-      "eval_loss": 0.3643175959587097,
-      "eval_num_fragments": 2.9984091913389306,
-      "eval_poi_equal": 0.2521431727794962,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.7975254087494477,
-      "eval_poi_heavy_atoms_difference": 7.319664162615997,
-      "eval_poi_heavy_atoms_difference_norm": 0.20421401046509144,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.7975254087494477,
-      "eval_reassembly": 0.0013256738842244808,
-      "eval_reassembly_nostereo": 0.0017675651789659744,
-      "eval_runtime": 1504.2054,
-      "eval_samples_per_second": 7.522,
-      "eval_steps_per_second": 0.118,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.7926646045072913,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.7667906880378723,
-      "learning_rate": 5.995876247868925e-06,
-      "loss": 0.1739,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.5333065986633301,
-      "learning_rate": 1.520218659043423e-07,
-      "loss": 0.1568,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.5492430925369263,
-      "learning_rate": 4.382203124261553e-06,
-      "loss": 0.1523,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.5843077301979065,
-      "learning_rate": 5.2779508184062e-06,
-      "loss": 0.1435,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.5402572751045227,
-      "learning_rate": 0.0,
-      "loss": 0.1315,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.005391073795846222,
-      "eval_e3_equal": 0.373663278833407,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9975254087494476,
-      "eval_e3_heavy_atoms_difference": -1.0522315510384446,
-      "eval_e3_heavy_atoms_difference_norm": -0.057307141416027135,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9975254087494476,
-      "eval_has_all_attachment_points": 0.9695095006628369,
-      "eval_has_three_substructures": 0.9980556783031375,
-      "eval_heavy_atoms_difference": 11.881926646045073,
-      "eval_heavy_atoms_difference_norm": 0.1534738333404604,
-      "eval_linker_equal": 0.0205921343349536,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9942554131683606,
-      "eval_linker_heavy_atoms_difference": 3.2428634555899247,
-      "eval_linker_heavy_atoms_difference_norm": 0.11490263193193657,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9942554131683606,
-      "eval_loss": 0.33794277906417847,
-      "eval_num_fragments": 2.9984091913389306,
-      "eval_poi_equal": 0.4138753866548829,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.880159080866107,
-      "eval_poi_heavy_atoms_difference": 3.95687140963323,
-      "eval_poi_heavy_atoms_difference_norm": 0.11301100363494641,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.880159080866107,
-      "eval_reassembly": 0.005479452054794521,
-      "eval_reassembly_nostereo": 0.00821917808219178,
-      "eval_runtime": 1545.1229,
-      "eval_samples_per_second": 7.323,
-      "eval_steps_per_second": 0.115,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8769774635439682,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=8-learning_rate=6.7e-06-warmup_ratio=0.040-num_cycles=6.500",
+  "trial_name": "trial-number=9-learning_rate=9.1e-06-warmup_ratio=0.060-num_cycles=2.000",
   "trial_params": {
-    "learning_rate": 6.660473554133319e-06,
-    "num_cycles": 6.5,
-    "warmup_ratio": 0.04
+    "learning_rate": 9.066610391784642e-06,
+    "num_cycles": 2.0,
+    "warmup_ratio": 0.060000000000000005
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b0609760c80d39fd60b4b9a6b6bc7aba41bb55723c3dd76d04162efbf7717fa
+oid sha256:324a4f6c8065b68520160d68813ed488ec8981224f51c7a464e07a1b4642182a
 size 7480