Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f15bc6899e10e2875d0e58ef068fabe8068ed092ffaaccf564e13f45aa5e223
+oid sha256:449d7f97eb3c0683bef5d41093e565229353e342beb26e31331a2378bee5ad2a
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fb14749e522ed4c74afc370e250f0ceafeda5e9584b713160889a43634e3c31
+oid sha256:3414a4eb86fcbb753564f4f678d1e33ed8fc47f8f1be57e7869653405ca6bcb0
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fdd21bd9c09685281fbf6c62ade7d74e50289f375c871724c2edc8c81e406942
+oid sha256:f2b76396262859c6e6ec96eb86f20cc83f8178654790af038e7a29c44321d24b
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:397c886399d45556b57f80d83f9cc295715699f48ae5732744568b7ce4e12ddf
+oid sha256:97292eb1d0bbf93658e37ef08ce944568285f7894259d71b828585ab2258d7f4
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0007070260715863897,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=12-learning_rate=2.1e-06-warmup_ratio=0.030-min_lr=0.000-factor=0.760/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.00017675651789659743,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=13-learning_rate=3.2e-06-warmup_ratio=0.090-min_lr=0.000-factor=0.620/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 1.9846965074539185,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 3.3119,
+      "grad_norm": 1.4082831144332886,
+      "learning_rate": 3.193326729069682e-06,
+      "loss": 2.7622,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.4285236597061157,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 1.3895,
+      "grad_norm": 1.2825632095336914,
+      "learning_rate": 3.193326729069682e-06,
+      "loss": 1.1427,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 1.320665955543518,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 1.0707,
+      "grad_norm": 1.2571899890899658,
+      "learning_rate": 3.193326729069682e-06,
+      "loss": 0.8533,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 1.3279285430908203,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.8836,
+      "grad_norm": 1.1884210109710693,
+      "learning_rate": 3.193326729069682e-06,
+      "loss": 0.681,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 1.3615200519561768,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.7519,
+      "grad_norm": 1.2224940061569214,
+      "learning_rate": 3.193326729069682e-06,
+      "loss": 0.5664,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 1.389214277267456,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.6564,
+      "grad_norm": 1.2473653554916382,
+      "learning_rate": 3.193326729069682e-06,
+      "loss": 0.4851,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 1.147653579711914,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.5797,
+      "grad_norm": 0.927285373210907,
+      "learning_rate": 3.193326729069682e-06,
+      "loss": 0.4205,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 1.351214051246643,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.5204,
+      "grad_norm": 1.2039693593978882,
+      "learning_rate": 3.193326729069682e-06,
+      "loss": 0.3708,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 1.181339979171753,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.4675,
+      "grad_norm": 0.9429658651351929,
+      "learning_rate": 3.193326729069682e-06,
+      "loss": 0.3277,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 1.1129992008209229,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.4263,
+      "grad_norm": 0.7795262336730957,
+      "learning_rate": 3.193326729069682e-06,
+      "loss": 0.2952,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 8.837825894829872e-05,
-      "eval_e3_equal": 0.10755634114007954,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9873619089703933,
-      "eval_e3_heavy_atoms_difference": -0.9413168360583296,
-      "eval_e3_heavy_atoms_difference_norm": -0.0685199058037525,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9873619089703933,
-      "eval_has_all_attachment_points": 0.8654882898806894,
-      "eval_has_three_substructures": 0.9889527176314626,
-      "eval_heavy_atoms_difference": 44.94865223155104,
-      "eval_heavy_atoms_difference_norm": 0.5755157169598037,
-      "eval_linker_equal": 0.0023862129916040653,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9806451612903225,
-      "eval_linker_heavy_atoms_difference": 7.377286787450287,
-      "eval_linker_heavy_atoms_difference_norm": 0.341241279313658,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9806451612903225,
-      "eval_loss": 0.5191501975059509,
-      "eval_num_fragments": 2.999381352187362,
-      "eval_poi_equal": 0.02262483429076447,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.5184268669907203,
-      "eval_poi_heavy_atoms_difference": 18.761201944321698,
-      "eval_poi_heavy_atoms_difference_norm": 0.5490757149518393,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.5184268669907203,
-      "eval_reassembly": 8.837825894829872e-05,
-      "eval_reassembly_nostereo": 8.837825894829872e-05,
-      "eval_runtime": 1410.3215,
-      "eval_samples_per_second": 8.023,
-      "eval_steps_per_second": 0.126,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.5147149801148917,
-      "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.8510923981666565,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.3895,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.8566566705703735,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.3578,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.7617588639259338,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.3306,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 1.0278968811035156,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.3068,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.8128023147583008,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.2855,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
       "eval_all_ligands_equal": 0.00017675651789659743,
-      "eval_e3_equal": 0.14732655766681396,
+      "eval_e3_equal": 0.14918250110472825,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9926646045072912,
-      "eval_e3_heavy_atoms_difference": -1.3951391957578436,
-      "eval_e3_heavy_atoms_difference_norm": -0.08085056605381094,
+      "eval_e3_has_attachment_point(s)": 0.9908970393283252,
+      "eval_e3_heavy_atoms_difference": -1.160053026955369,
+      "eval_e3_heavy_atoms_difference_norm": -0.07128999673551212,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9926646045072912,
-      "eval_has_all_attachment_points": 0.9148033583738401,
-      "eval_has_three_substructures": 0.9933716305788776,
-      "eval_heavy_atoms_difference": 30.79673000441891,
-      "eval_heavy_atoms_difference_norm": 0.39236787791885963,
-      "eval_linker_equal": 0.003977021652673442,
+      "eval_e3_valid": 0.9908970393283252,
+      "eval_has_all_attachment_points": 0.9206363234644277,
+      "eval_has_three_substructures": 0.9915156871409633,
+      "eval_heavy_atoms_difference": 31.817852408307555,
+      "eval_heavy_atoms_difference_norm": 0.40802498981968244,
+      "eval_linker_equal": 0.003535130357931949,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9880689350419797,
-      "eval_linker_heavy_atoms_difference": 5.820503756076005,
-      "eval_linker_heavy_atoms_difference_norm": 0.24138745220741845,
+      "eval_linker_has_attachment_point(s)": 0.9851524524966858,
+      "eval_linker_heavy_atoms_difference": 6.225629695095007,
+      "eval_linker_heavy_atoms_difference_norm": 0.27139980807463093,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9880689350419797,
-      "eval_loss": 0.42061758041381836,
-      "eval_num_fragments": 2.99584622182943,
-      "eval_poi_equal": 0.09288555015466195,
+      "eval_linker_valid": 0.9851524524966858,
+      "eval_loss": 0.42596012353897095,
+      "eval_num_fragments": 2.9932832523199293,
+      "eval_poi_equal": 0.09155987627043748,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.6764471939902784,
-      "eval_poi_heavy_atoms_difference": 12.387715422006186,
-      "eval_poi_heavy_atoms_difference_norm": 0.35498857497941394,
+      "eval_poi_has_attachment_point(s)": 0.6665488289880689,
+      "eval_poi_heavy_atoms_difference": 12.666548828988068,
+      "eval_poi_heavy_atoms_difference_norm": 0.36623285252137916,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.6764471939902784,
-      "eval_reassembly": 0.0002651347768448962,
-      "eval_reassembly_nostereo": 0.00035351303579319487,
-      "eval_runtime": 1510.8447,
-      "eval_samples_per_second": 7.489,
-      "eval_steps_per_second": 0.117,
+      "eval_poi_valid": 0.6665488289880689,
+      "eval_reassembly": 0.00017675651789659743,
+      "eval_reassembly_nostereo": 0.00017675651789659743,
+      "eval_runtime": 1535.5989,
+      "eval_samples_per_second": 7.368,
+      "eval_steps_per_second": 0.115,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.6732655766681397,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.9169577956199646,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.268,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.8658037781715393,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.2512,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.8686316609382629,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.2369,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 1.007585883140564,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.2234,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.6619876623153687,
-      "learning_rate": 2.079905465346972e-06,
-      "loss": 0.2112,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0007070260715863897,
-      "eval_e3_equal": 0.2,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9937251436146708,
-      "eval_e3_heavy_atoms_difference": -1.2498453380468404,
-      "eval_e3_heavy_atoms_difference_norm": -0.07127699571538457,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9937251436146708,
-      "eval_has_all_attachment_points": 0.9333627927529827,
-      "eval_has_three_substructures": 0.994078656650464,
-      "eval_heavy_atoms_difference": 24.293680954485197,
-      "eval_heavy_atoms_difference_norm": 0.31102480232191276,
-      "eval_linker_equal": 0.006981882456915599,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9891294741493593,
-      "eval_linker_heavy_atoms_difference": 5.449580203269996,
-      "eval_linker_heavy_atoms_difference_norm": 0.23420443958910875,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9891294741493593,
-      "eval_loss": 0.3799628019332886,
-      "eval_num_fragments": 2.99584622182943,
-      "eval_poi_equal": 0.18877596111356607,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.7513919575784357,
-      "eval_poi_heavy_atoms_difference": 9.262218294299602,
-      "eval_poi_heavy_atoms_difference_norm": 0.265899326545649,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.7513919575784357,
-      "eval_reassembly": 0.0007070260715863897,
-      "eval_reassembly_nostereo": 0.0009721608484312859,
-      "eval_runtime": 1478.0556,
-      "eval_samples_per_second": 7.655,
-      "eval_steps_per_second": 0.12,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.7479452054794521,
-      "step": 10000
+      "eval_valid": 0.6627485638532921,
+      "step": 5000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=12-learning_rate=2.1e-06-warmup_ratio=0.030-min_lr=0.000-factor=0.760",
+  "trial_name": "trial-number=13-learning_rate=3.2e-06-warmup_ratio=0.090-min_lr=0.000-factor=0.620",
   "trial_params": {
-    "factor": 0.76,
-    "learning_rate": 2.079905465346972e-06,
-    "min_lr": 9.98020021796813e-10,
-    "warmup_ratio": 0.03
+    "factor": 0.62,
+    "learning_rate": 3.193326729069682e-06,
+    "min_lr": 1.511578228908639e-10,
+    "warmup_ratio": 0.09
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ff9a1e82668889c8fa0127b33263c5e1e57338688ea312e289631509f2b117d
+oid sha256:097beebc9b628f0762b91a7bc00cc15508a178b9c11e8d8b1d896af9ff8e3924
 size 7544