Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c13c01c39666e440188e24c1739ec3a1675fcb597743cbc9ffd8ca8f944a83e
+oid sha256:d4a48f39b7125cfa8a984a75d887fd19104dd137b9ec7191793bd58060b5290b
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c2316716331e7298f49329d3c6a26a0569ede77f26a578d906a2a1cbd6a2375
+oid sha256:331f621f3a2ac982719cc5ea3629865dd6be1354abb9d6be16adcb180d07c7ff
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:579d5adf9d4f20ef508e49955208505217498071c2fb8d9254c11be579cc291c
+oid sha256:7dfd5d53703ab1584eb06ee8f467b51b3a92071c4a1363f6f80813fb12af192a
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4b7b070ada8df300c245e6daa26454039d86686f1bc260ce873c14c0e22ed613
+oid sha256:d65a73d223c88a085b2cae33e1a56325451289961eb434afb802c07ddabfc907
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.4693769332744145,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=17-learning_rate=2.3e-05-warmup_ratio=0.070-num_cycles=6/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.5108263367211666,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=18-learning_rate=2.0e-04-warmup_ratio=0.010-num_cycles=3/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.9457626938819885,
-      "learning_rate": 2.0006150861219307e-05,
-      "loss": 1.7641,
+      "grad_norm": 0.5011103749275208,
+      "learning_rate": 0.00019333014963385616,
+      "loss": 0.9833,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.7090033888816833,
-      "learning_rate": 9.95453959695377e-06,
-      "loss": 0.382,
+      "grad_norm": 0.30493640899658203,
+      "learning_rate": 0.00016595114383716005,
+      "loss": 0.1736,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.6167610287666321,
-      "learning_rate": 1.2902076164106024e-06,
-      "loss": 0.2555,
+      "grad_norm": 0.20391501486301422,
+      "learning_rate": 0.00012396207622088705,
+      "loss": 0.065,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.5820146799087524,
-      "learning_rate": 2.1922365111122713e-05,
-      "loss": 0.2229,
+      "grad_norm": 0.12541060149669647,
+      "learning_rate": 7.66983385109593e-05,
+      "loss": 0.0227,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.5265125036239624,
-      "learning_rate": 1.325803313057954e-05,
-      "loss": 0.1442,
+      "grad_norm": 0.10875938832759857,
+      "learning_rate": 3.466803510220776e-05,
+      "loss": 0.0121,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.3760647475719452,
-      "learning_rate": 3.206421866314002e-06,
-      "loss": 0.1059,
+      "grad_norm": 0.10840030014514923,
+      "learning_rate": 7.215725637295107e-06,
+      "loss": 0.0084,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.4465866684913635,
-      "learning_rate": 2.3002832012997965e-05,
-      "loss": 0.0951,
+      "grad_norm": 0.28355872631073,
+      "learning_rate": 0.0002000300854243022,
+      "loss": 0.0154,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.31987500190734863,
-      "learning_rate": 1.642771196447441e-05,
-      "loss": 0.0765,
+      "grad_norm": 0.1145474836230278,
+      "learning_rate": 0.00018461415129199327,
+      "loss": 0.0275,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.3740421235561371,
-      "learning_rate": 5.803143181883335e-06,
-      "loss": 0.0549,
+      "grad_norm": 0.1480787992477417,
+      "learning_rate": 0.00015043882563917473,
+      "loss": 0.0093,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.30238306522369385,
-      "learning_rate": 5.255416329408794e-08,
-      "loss": 0.0476,
+      "grad_norm": 0.08729447424411774,
+      "learning_rate": 0.00010510227795238147,
+      "loss": 0.0059,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.18824569155987628,
-      "eval_e3_equal": 0.6958020326999558,
+      "eval_all_ligands_equal": 0.5108263367211666,
+      "eval_e3_equal": 0.7976137870083959,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9968183826778613,
-      "eval_e3_heavy_atoms_difference": -0.253557224922669,
-      "eval_e3_heavy_atoms_difference_norm": -0.024097480185388778,
+      "eval_e3_has_attachment_point(s)": 0.9916924436588599,
+      "eval_e3_heavy_atoms_difference": 0.4874944763588157,
+      "eval_e3_heavy_atoms_difference_norm": 0.006469055588544779,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9968183826778613,
-      "eval_has_all_attachment_points": 0.9860362350861688,
-      "eval_has_three_substructures": 0.9985859478568272,
-      "eval_heavy_atoms_difference": 7.789659743703049,
-      "eval_heavy_atoms_difference_norm": 0.10390847735933036,
-      "eval_linker_equal": 0.2976579761378701,
+      "eval_e3_valid": 0.9916924436588599,
+      "eval_has_all_attachment_points": 0.9919575784357049,
+      "eval_has_three_substructures": 0.9999116217410517,
+      "eval_heavy_atoms_difference": 2.0157313300927973,
+      "eval_heavy_atoms_difference_norm": 0.02104730460463736,
+      "eval_linker_equal": 0.7404330534688467,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9957578435704817,
-      "eval_linker_heavy_atoms_difference": 1.1033141847105612,
-      "eval_linker_heavy_atoms_difference_norm": 0.022649674013351273,
+      "eval_linker_has_attachment_point(s)": 0.9988510826336722,
+      "eval_linker_heavy_atoms_difference": 0.546442775077331,
+      "eval_linker_heavy_atoms_difference_norm": 0.006276838082081687,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9957578435704817,
-      "eval_loss": 0.290093332529068,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.6439239946973044,
+      "eval_linker_valid": 0.9988510826336722,
+      "eval_loss": 0.32633018493652344,
+      "eval_num_fragments": 2.9999116217410515,
+      "eval_poi_equal": 0.744675209898365,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9087936367653557,
-      "eval_poi_heavy_atoms_difference": 2.6612461334511712,
-      "eval_poi_heavy_atoms_difference_norm": 0.08440871186102332,
+      "eval_poi_has_attachment_point(s)": 0.9913389306230668,
+      "eval_poi_heavy_atoms_difference": -0.07856827220503756,
+      "eval_poi_heavy_atoms_difference_norm": -0.022628294753132334,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9087936367653557,
-      "eval_reassembly": 0.19133893062306673,
-      "eval_reassembly_nostereo": 0.21307998232434822,
-      "eval_runtime": 1572.4469,
-      "eval_samples_per_second": 7.196,
-      "eval_steps_per_second": 0.113,
+      "eval_poi_valid": 0.9913389306230668,
+      "eval_reassembly": 0.5181617322138754,
+      "eval_reassembly_nostereo": 0.5433495360141405,
+      "eval_runtime": 1536.1175,
+      "eval_samples_per_second": 7.366,
+      "eval_steps_per_second": 0.115,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9050817498895272,
+      "eval_valid": 0.982235969951392,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.3234905004501343,
-      "learning_rate": 1.9206787570322047e-05,
-      "loss": 0.046,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.2600933909416199,
-      "learning_rate": 8.870000645427467e-06,
-      "loss": 0.0344,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.23982013761997223,
-      "learning_rate": 8.313822822439741e-07,
-      "loss": 0.0282,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.2609094977378845,
-      "learning_rate": 2.137011577033572e-05,
-      "loss": 0.0283,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.22073349356651306,
-      "learning_rate": 1.2158535704553633e-05,
-      "loss": 0.0236,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.38400353513035795,
-      "eval_e3_equal": 0.7617322138753867,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9939019001325674,
-      "eval_e3_heavy_atoms_difference": 0.27927529827662395,
-      "eval_e3_heavy_atoms_difference_norm": 0.0007151575670521906,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9939019001325674,
-      "eval_has_all_attachment_points": 0.9831197525408749,
-      "eval_has_three_substructures": 0.9979673000441891,
-      "eval_heavy_atoms_difference": 7.73194874060981,
-      "eval_heavy_atoms_difference_norm": 0.10313332504564382,
-      "eval_linker_equal": 0.5638532920901458,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9953159522757402,
-      "eval_linker_heavy_atoms_difference": 0.5695095006628369,
-      "eval_linker_heavy_atoms_difference_norm": 0.005042392788121972,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9953159522757402,
-      "eval_loss": 0.2872321903705597,
-      "eval_num_fragments": 3.001148917366328,
-      "eval_poi_equal": 0.6979231109147149,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.911533362792753,
-      "eval_poi_heavy_atoms_difference": 2.6250110472823684,
-      "eval_poi_heavy_atoms_difference_norm": 0.08379373724944911,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.911533362792753,
-      "eval_reassembly": 0.3897481219619974,
-      "eval_reassembly_nostereo": 0.41555457357490055,
-      "eval_runtime": 1517.5446,
-      "eval_samples_per_second": 7.456,
-      "eval_steps_per_second": 0.117,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9062306672558551,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.26592886447906494,
-      "learning_rate": 2.483129049056858e-06,
-      "loss": 0.0188,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.24544061720371246,
-      "learning_rate": 2.2742436579558516e-05,
-      "loss": 0.0182,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.28115570545196533,
-      "learning_rate": 1.540233080644256e-05,
-      "loss": 0.0176,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.18894816935062408,
-      "learning_rate": 4.873979764001823e-06,
-      "loss": 0.0136,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.1895386427640915,
-      "learning_rate": 0.0,
-      "loss": 0.0117,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.4693769332744145,
-      "eval_e3_equal": 0.7873619089703933,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9937251436146708,
-      "eval_e3_heavy_atoms_difference": 0.17021652673442333,
-      "eval_e3_heavy_atoms_difference_norm": -0.0029225608928936555,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9937251436146708,
-      "eval_has_all_attachment_points": 0.9855943437914273,
-      "eval_has_three_substructures": 0.9990278391515687,
-      "eval_heavy_atoms_difference": 7.027308882015024,
-      "eval_heavy_atoms_difference_norm": 0.09441348364885087,
-      "eval_linker_equal": 0.6623950508174989,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9961113566062748,
-      "eval_linker_heavy_atoms_difference": 0.4987185152452497,
-      "eval_linker_heavy_atoms_difference_norm": 0.00498684362683764,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9961113566062748,
-      "eval_loss": 0.2913830280303955,
-      "eval_num_fragments": 3.0000883782589485,
-      "eval_poi_equal": 0.7327441449403447,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9178965974370304,
-      "eval_poi_heavy_atoms_difference": 2.328767123287671,
-      "eval_poi_heavy_atoms_difference_norm": 0.07470090895462175,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9178965974370304,
-      "eval_reassembly": 0.4764471939902784,
-      "eval_reassembly_nostereo": 0.5030490499337164,
-      "eval_runtime": 1495.0802,
-      "eval_samples_per_second": 7.568,
-      "eval_steps_per_second": 0.118,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9112682280159081,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=17-learning_rate=2.3e-05-warmup_ratio=0.070-num_cycles=6",
+  "trial_name": "trial-number=18-learning_rate=2.0e-04-warmup_ratio=0.010-num_cycles=3",
   "trial_params": {
-    "learning_rate": 2.321257272753331e-05,
-    "num_cycles": 6,
-    "warmup_ratio": 0.06999999999999999
+    "learning_rate": 0.0002004749428706299,
+    "num_cycles": 3,
+    "warmup_ratio": 0.01
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28afeabe39a09cec3f84c61857ff0a7aa70d81b71ad820e515f0d02a64926893
+oid sha256:3d4dc2f1997c6fb2b97d75f7386d588e59cbc1631d18eba4831404dccc7583fc
 size 7480