Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76fc34c86d8554bd2361ea21a010789be9b490bbcfa6e38eb77672d65e19d935
+oid sha256:ffa8acab7764e950b683335670e01005f8a1f787969a07baadf5823fb145b494
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49785842d067d82889de1c9dedd9e9260f50e76b8895d564d83a1b6c47482b74
+oid sha256:4d904010911abb693412a356c5e1ecd2f34f5969fd107be293eb5d016da628f9
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b9c0ee8e7add866808757347d4514f80ef8ce2fa43fc33ed1a223fcb228aec3
+oid sha256:04a8122517fa82ad6ed387e7da241075ff40248f2aee9802f1c51edc013df433
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3fd1611e4ea9d20890ab10928d6c76fdd332d1eb1cce70fd9c7927e50946b909
+oid sha256:e5abb4045b7a90cdf6ddf50a8c38ea41405bd80989036a2b83584448ef66bb42
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.00017675651789659743,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=4-learning_rate=2.6e-06-warmup_ratio=0.030-num_cycles=0.500/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.0,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25/trial-number=5-learning_rate=1.3e-06-warmup_ratio=0.080-num_cycles=4.000/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 3.954847574234009,
-      "learning_rate": 1.84116374394636e-06,
-      "loss": 5.7773,
+      "grad_norm": 4.2125678062438965,
+      "learning_rate": 9.653298476903908e-07,
+      "loss": 6.6359,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.564314842224121,
-      "learning_rate": 2.567301241779092e-06,
-      "loss": 1.7772,
+      "grad_norm": 2.408616542816162,
+      "learning_rate": 1.1383762270122183e-06,
+      "loss": 2.6568,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 1.3318532705307007,
-      "learning_rate": 2.5271309141899734e-06,
-      "loss": 1.1257,
+      "grad_norm": 1.875552773475647,
+      "learning_rate": 2.973196899345272e-07,
+      "loss": 1.588,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 1.318094253540039,
-      "learning_rate": 2.4516727688213163e-06,
-      "loss": 0.887,
+      "grad_norm": 2.1072449684143066,
+      "learning_rate": 4.662220845232186e-08,
+      "loss": 1.4699,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 1.4123839139938354,
-      "learning_rate": 2.343073914710247e-06,
-      "loss": 0.7328,
+      "grad_norm": 1.682381510734558,
+      "learning_rate": 7.783970331251584e-07,
+      "loss": 1.438,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 1.3738270998001099,
-      "learning_rate": 2.204424456708191e-06,
-      "loss": 0.6289,
+      "grad_norm": 1.580583095550537,
+      "learning_rate": 1.3480829841452437e-06,
+      "loss": 1.294,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 1.1391628980636597,
-      "learning_rate": 2.0396695686968327e-06,
-      "loss": 0.5524,
+      "grad_norm": 1.402235746383667,
+      "learning_rate": 8.646403560799816e-07,
+      "loss": 1.1103,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 1.1100925207138062,
-      "learning_rate": 1.853497236409607e-06,
-      "loss": 0.497,
+      "grad_norm": 1.4020792245864868,
+      "learning_rate": 8.421659496257073e-08,
+      "loss": 1.0136,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 1.0999289751052856,
-      "learning_rate": 1.6512048640586581e-06,
-      "loss": 0.4504,
+      "grad_norm": 1.4105451107025146,
+      "learning_rate": 2.2746416198410507e-07,
+      "loss": 0.9933,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 1.2520828247070312,
-      "learning_rate": 1.4385485403829063e-06,
-      "loss": 0.4177,
+      "grad_norm": 1.4410276412963867,
+      "learning_rate": 1.0703310690373619e-06,
+      "loss": 0.9689,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 8.837825894829872e-05,
-      "eval_e3_equal": 0.10764471939902784,
+      "eval_all_ligands_equal": 0.0,
+      "eval_e3_equal": 0.02421564295183385,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.983738400353513,
-      "eval_e3_heavy_atoms_difference": -1.0652231551038445,
-      "eval_e3_heavy_atoms_difference_norm": -0.07217928093884596,
+      "eval_e3_has_attachment_point(s)": 0.7856827220503756,
+      "eval_e3_heavy_atoms_difference": 7.01767565178966,
+      "eval_e3_heavy_atoms_difference_norm": 0.193812682287618,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.983738400353513,
-      "eval_has_all_attachment_points": 0.8426866990720283,
-      "eval_has_three_substructures": 0.9863013698630136,
-      "eval_heavy_atoms_difference": 47.09703932832523,
-      "eval_heavy_atoms_difference_norm": 0.6033113620483537,
-      "eval_linker_equal": 0.002209456473707468,
+      "eval_e3_valid": 0.7856827220503756,
+      "eval_has_all_attachment_points": 0.47989394608926206,
+      "eval_has_three_substructures": 0.7911621741051701,
+      "eval_heavy_atoms_difference": 71.59911621741051,
+      "eval_heavy_atoms_difference_norm": 0.9386096937863235,
+      "eval_linker_equal": 0.0007954043305346885,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9756959787892179,
-      "eval_linker_heavy_atoms_difference": 7.105788775961114,
-      "eval_linker_heavy_atoms_difference_norm": 0.32299607171762157,
+      "eval_linker_has_attachment_point(s)": 0.6998674326115776,
+      "eval_linker_heavy_atoms_difference": 11.983208130799824,
+      "eval_linker_heavy_atoms_difference_norm": 0.6700703059837816,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9756959787892179,
-      "eval_loss": 0.5187373757362366,
-      "eval_num_fragments": 2.9990278391515686,
-      "eval_poi_equal": 0.025187803800265134,
+      "eval_linker_valid": 0.6998674326115776,
+      "eval_loss": 0.9760982990264893,
+      "eval_num_fragments": 3.1617322138753865,
+      "eval_poi_equal": 0.0,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.4813079982324348,
-      "eval_poi_heavy_atoms_difference": 19.630490499337164,
-      "eval_poi_heavy_atoms_difference_norm": 0.5756228960447303,
+      "eval_poi_has_attachment_point(s)": 0.15687140963323024,
+      "eval_poi_heavy_atoms_difference": 29.274502872293414,
+      "eval_poi_heavy_atoms_difference_norm": 0.9270241865988441,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.4813079982324348,
-      "eval_reassembly": 8.837825894829872e-05,
-      "eval_reassembly_nostereo": 8.837825894829872e-05,
-      "eval_runtime": 1427.669,
-      "eval_samples_per_second": 7.926,
-      "eval_steps_per_second": 0.124,
+      "eval_poi_valid": 0.15687140963323024,
+      "eval_reassembly": 0.0,
+      "eval_reassembly_nostereo": 0.0,
+      "eval_runtime": 2252.5623,
+      "eval_samples_per_second": 5.023,
+      "eval_steps_per_second": 0.079,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.4770658418029165,
+      "eval_valid": 0.1456473707467963,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.7651128172874451,
-      "learning_rate": 1.2215792531469616e-06,
-      "loss": 0.3907,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.7874689102172852,
-      "learning_rate": 1.006470712493407e-06,
-      "loss": 0.3697,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.7223606109619141,
-      "learning_rate": 7.993436823151736e-07,
-      "loss": 0.3538,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.9319483637809753,
-      "learning_rate": 6.060918181766256e-07,
-      "loss": 0.3417,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.7011885046958923,
-      "learning_rate": 4.32213967444154e-07,
-      "loss": 0.3325,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.00017675651789659743,
-      "eval_e3_equal": 0.13433495360141406,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9863013698630136,
-      "eval_e3_heavy_atoms_difference": -1.321873619089704,
-      "eval_e3_heavy_atoms_difference_norm": -0.07822395872484526,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9863013698630136,
-      "eval_has_all_attachment_points": 0.8727353071144498,
-      "eval_has_three_substructures": 0.9881573133009279,
-      "eval_heavy_atoms_difference": 40.0600088378259,
-      "eval_heavy_atoms_difference_norm": 0.5112885776661789,
-      "eval_linker_equal": 0.002474591250552364,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9775519222271322,
-      "eval_linker_heavy_atoms_difference": 6.490057445868317,
-      "eval_linker_heavy_atoms_difference_norm": 0.28522627017102503,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9775519222271322,
-      "eval_loss": 0.4637558162212372,
-      "eval_num_fragments": 2.9939902783915158,
-      "eval_poi_equal": 0.05647370746796288,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.5658859920459567,
-      "eval_poi_heavy_atoms_difference": 16.55006628369421,
-      "eval_poi_heavy_atoms_difference_norm": 0.4834426488801326,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.5658859920459567,
-      "eval_reassembly": 0.00017675651789659743,
-      "eval_reassembly_nostereo": 0.00035351303579319487,
-      "eval_runtime": 1467.0417,
-      "eval_samples_per_second": 7.713,
-      "eval_steps_per_second": 0.121,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.5610251878038003,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.7916216254234314,
-      "learning_rate": 2.8265770340941057e-07,
-      "loss": 0.3276,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.8538663387298584,
-      "learning_rate": 1.616785455328715e-07,
-      "loss": 0.3228,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.8023797869682312,
-      "learning_rate": 7.271887159776171e-08,
-      "loss": 0.3209,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.9218820929527283,
-      "learning_rate": 1.8309967244785057e-08,
-      "loss": 0.3196,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.887701690196991,
-      "learning_rate": 0.0,
-      "loss": 0.319,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.00017675651789659743,
-      "eval_e3_equal": 0.1391957578435705,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9885992045956694,
-      "eval_e3_heavy_atoms_difference": -1.5863013698630137,
-      "eval_e3_heavy_atoms_difference_norm": -0.0880508148671753,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9885992045956694,
-      "eval_has_all_attachment_points": 0.8707909854175873,
-      "eval_has_three_substructures": 0.9902783915156872,
-      "eval_heavy_atoms_difference": 39.54228899690676,
-      "eval_heavy_atoms_difference_norm": 0.5050256554007249,
-      "eval_linker_equal": 0.0030932390631904553,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.980468404772426,
-      "eval_linker_heavy_atoms_difference": 6.626955368979231,
-      "eval_linker_heavy_atoms_difference_norm": 0.29765534741668487,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.980468404772426,
-      "eval_loss": 0.45504653453826904,
-      "eval_num_fragments": 2.9961113566062747,
-      "eval_poi_equal": 0.06177640300486081,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.5744586831639417,
-      "eval_poi_heavy_atoms_difference": 15.996199734865224,
-      "eval_poi_heavy_atoms_difference_norm": 0.46355078401344835,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.5744586831639417,
-      "eval_reassembly": 0.00017675651789659743,
-      "eval_reassembly_nostereo": 0.00017675651789659743,
-      "eval_runtime": 1454.9215,
-      "eval_samples_per_second": 7.777,
-      "eval_steps_per_second": 0.122,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.569244365885992,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=4-learning_rate=2.6e-06-warmup_ratio=0.030-num_cycles=0.500",
+  "trial_name": "trial-number=5-learning_rate=1.3e-06-warmup_ratio=0.080-num_cycles=4.000",
   "trial_params": {
-    "learning_rate": 2.5739469140370113e-06,
-    "num_cycles": 0.5,
-    "warmup_ratio": 0.03
+    "learning_rate": 1.3495311270711663e-06,
+    "num_cycles": 4.0,
+    "warmup_ratio": 0.08
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e7efa8e579f6d53e77e7125e1bb01c2acc7fe22652127303c6aa1edac03766d3
+oid sha256:71d2586a39a8b7d80743d047b8033a47d036f0685212eab19f9ffbf7918ea36d
 size 7416