Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31c731ab87ef520c3117511325377a97ad15dae205398f02dd514384b602391c
+oid sha256:b82a301955ba2f9aa82bc36a51700c2c354faa17ac4f30bc0aa4339fd6c6cc84
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f71cd535304c2b198acff97c548e4ee56f03d5adfcbd904755b7b5913a41ff61
+oid sha256:679f7566bcb90cff383d136ac5615185bbb042c5fdd60a6a54161b0486810ca0
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a81971f92b5a7d51fd883bb2a9836249c0ca486f06f6b05ff9ab1e0e69716d27
+oid sha256:2270c457ff62ab3539634227f4be5896e72a1b0aeb7296f63fc6037e55d6aafd
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4b7b070ada8df300c245e6daa26454039d86686f1bc260ce873c14c0e22ed613
+oid sha256:efc7b8261dc9f0cd1aff039568c605f93169128d3740989947948ba3d1781e01
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.41882456915598765,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=17-learning_rate=2.3e-05-warmup_ratio=0.070-num_cycles=5.500/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.5033141847105612,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine-opt25-rand-smiles/trial-number=18-learning_rate=2.0e-04-warmup_ratio=0.010-num_cycles=2.500/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 1.1214300394058228,
-      "learning_rate": 1.66041292757749e-05,
-      "loss": 3.0065,
+      "grad_norm": 0.6453943848609924,
+      "learning_rate": 0.00014340124668857647,
+      "loss": 1.519,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.7123772501945496,
-      "learning_rate": 1.6680127157646984e-05,
-      "loss": 0.577,
+      "grad_norm": 0.30024901032447815,
+      "learning_rate": 0.0001878001438351963,
+      "loss": 0.102,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.728777289390564,
-      "learning_rate": 1.586004103408713e-07,
-      "loss": 0.3074,
+      "grad_norm": 0.15960834920406342,
+      "learning_rate": 0.00012191951719773646,
+      "loss": 0.0292,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.6285669803619385,
-      "learning_rate": 1.3012154594856692e-05,
-      "loss": 0.2741,
+      "grad_norm": 0.10984167456626892,
+      "learning_rate": 4.1475601830154605e-05,
+      "loss": 0.0142,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.5516284704208374,
-      "learning_rate": 2.2258211750211904e-05,
-      "loss": 0.2059,
+      "grad_norm": 0.09457601606845856,
+      "learning_rate": 5.005669712982349e-07,
+      "loss": 0.0092,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.5238842964172363,
-      "learning_rate": 4.171131747579263e-06,
-      "loss": 0.1392,
+      "grad_norm": 0.12721996009349823,
+      "learning_rate": 2.6516320374474144e-05,
+      "loss": 0.0085,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.3185393214225769,
-      "learning_rate": 5.162865531391757e-06,
-      "loss": 0.1204,
+      "grad_norm": 0.17860917747020721,
+      "learning_rate": 0.00010204873010150722,
+      "loss": 0.0108,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.4352491497993469,
-      "learning_rate": 2.2688596596301756e-05,
-      "loss": 0.1114,
+      "grad_norm": 0.1712469458580017,
+      "learning_rate": 0.00017636456261024047,
+      "loss": 0.0198,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.4045301377773285,
-      "learning_rate": 1.1776811003545574e-05,
-      "loss": 0.0844,
+      "grad_norm": 0.16711823642253876,
+      "learning_rate": 0.00019954772889767774,
+      "loss": 0.0159,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.33092254400253296,
-      "learning_rate": 4.274544366418573e-07,
-      "loss": 0.0672,
+      "grad_norm": 0.11587321758270264,
+      "learning_rate": 0.00015602667506056428,
+      "loss": 0.0109,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.11011931064958021,
-      "eval_e3_equal": 0.5928413610251878,
+      "eval_all_ligands_equal": 0.5033141847105612,
+      "eval_e3_equal": 0.7938135218736191,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9980556783031375,
-      "eval_e3_heavy_atoms_difference": -0.3526292532037119,
-      "eval_e3_heavy_atoms_difference_norm": -0.028914826481674143,
+      "eval_e3_has_attachment_point(s)": 0.9905435262925321,
+      "eval_e3_heavy_atoms_difference": 0.4669023420238621,
+      "eval_e3_heavy_atoms_difference_norm": 0.007575271177021908,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9980556783031375,
-      "eval_has_all_attachment_points": 0.9870083959346001,
-      "eval_has_three_substructures": 0.9987627043747238,
-      "eval_heavy_atoms_difference": 5.611400795404331,
-      "eval_heavy_atoms_difference_norm": 0.07280718278132618,
-      "eval_linker_equal": 0.20132567388422448,
+      "eval_e3_valid": 0.9905435262925321,
+      "eval_has_all_attachment_points": 0.9913389306230668,
+      "eval_has_three_substructures": 0.9996464869642068,
+      "eval_heavy_atoms_difference": 6.135572249226691,
+      "eval_heavy_atoms_difference_norm": 0.07836621490514804,
+      "eval_linker_equal": 0.7340698188245691,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9957578435704817,
-      "eval_linker_heavy_atoms_difference": 1.1542200618647813,
-      "eval_linker_heavy_atoms_difference_norm": 0.015000989286050285,
+      "eval_linker_has_attachment_point(s)": 0.9969951391957579,
+      "eval_linker_heavy_atoms_difference": 0.7690676093680955,
+      "eval_linker_heavy_atoms_difference_norm": 0.025383089814494973,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9957578435704817,
-      "eval_loss": 0.301456481218338,
-      "eval_num_fragments": 2.999646486964207,
-      "eval_poi_equal": 0.5993813521873619,
+      "eval_linker_valid": 0.9969951391957579,
+      "eval_loss": 0.29212433099746704,
+      "eval_num_fragments": 2.9995581087052585,
+      "eval_poi_equal": 0.7510384445426425,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9392841361025188,
-      "eval_poi_heavy_atoms_difference": 1.6953601414052144,
-      "eval_poi_heavy_atoms_difference_norm": 0.04866969285000904,
+      "eval_poi_has_attachment_point(s)": 0.9475033141847106,
+      "eval_poi_heavy_atoms_difference": 1.743791427308882,
+      "eval_poi_heavy_atoms_difference_norm": 0.050082263417517286,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9392841361025188,
-      "eval_reassembly": 0.11259390190013256,
-      "eval_reassembly_nostereo": 0.13442333186036234,
-      "eval_runtime": 1600.9107,
-      "eval_samples_per_second": 7.068,
-      "eval_steps_per_second": 0.111,
+      "eval_poi_valid": 0.9475033141847106,
+      "eval_reassembly": 0.5104728236853734,
+      "eval_reassembly_nostereo": 0.5452938577110031,
+      "eval_runtime": 1512.4575,
+      "eval_samples_per_second": 7.481,
+      "eval_steps_per_second": 0.117,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9366327883340698,
+      "eval_valid": 0.9362792752982766,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.4414580762386322,
-      "learning_rate": 1.776329188349048e-05,
-      "loss": 0.0644,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.4248468279838562,
-      "learning_rate": 1.9300081717101823e-05,
-      "loss": 0.0569,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.2549741566181183,
-      "learning_rate": 1.0943784305675876e-06,
-      "loss": 0.0439,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.27541837096214294,
-      "learning_rate": 9.862523618028005e-06,
-      "loss": 0.0401,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.4759804904460907,
-      "learning_rate": 2.3105212568338626e-05,
-      "loss": 0.0393,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.2690234202386213,
-      "eval_e3_equal": 0.7056120194432169,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.997348652231551,
-      "eval_e3_heavy_atoms_difference": 0.10172337604949183,
-      "eval_e3_heavy_atoms_difference_norm": -0.0070542327648459024,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.997348652231551,
-      "eval_has_all_attachment_points": 0.9858594785682722,
-      "eval_has_three_substructures": 0.9992045956694653,
-      "eval_heavy_atoms_difference": 5.296155545735749,
-      "eval_heavy_atoms_difference_norm": 0.06857629473680947,
-      "eval_linker_equal": 0.4202386212991604,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9968183826778613,
-      "eval_linker_heavy_atoms_difference": 0.6022978347326557,
-      "eval_linker_heavy_atoms_difference_norm": -0.0013174337936367362,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9968183826778613,
-      "eval_loss": 0.29956623911857605,
-      "eval_num_fragments": 3.0004418912947415,
-      "eval_poi_equal": 0.6509058771542201,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.943791427308882,
-      "eval_poi_heavy_atoms_difference": 1.6980998674326115,
-      "eval_poi_heavy_atoms_difference_norm": 0.04522592378769132,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.943791427308882,
-      "eval_reassembly": 0.27353071144498453,
-      "eval_reassembly_nostereo": 0.297039328325232,
-      "eval_runtime": 1495.624,
-      "eval_samples_per_second": 7.565,
-      "eval_steps_per_second": 0.118,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9398144056562086,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.21357761323451996,
-      "learning_rate": 6.84133667553644e-06,
-      "loss": 0.0326,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.25558289885520935,
-      "learning_rate": 2.8044543687991375e-06,
-      "loss": 0.0273,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.29299396276474,
-      "learning_rate": 2.135331805932135e-05,
-      "loss": 0.0275,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.2609122097492218,
-      "learning_rate": 1.4891027000593144e-05,
-      "loss": 0.026,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.20302577316761017,
-      "learning_rate": 0.0,
-      "loss": 0.0204,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.41882456915598765,
-      "eval_e3_equal": 0.759169244365886,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9969067609368095,
-      "eval_e3_heavy_atoms_difference": 0.032523199292973926,
-      "eval_e3_heavy_atoms_difference_norm": -0.008456533785678375,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9969067609368095,
-      "eval_has_all_attachment_points": 0.9897481219619974,
-      "eval_has_three_substructures": 0.9994697304463102,
-      "eval_heavy_atoms_difference": 4.253910737958463,
-      "eval_heavy_atoms_difference_norm": 0.05558554950808673,
-      "eval_linker_equal": 0.6033583738400353,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9972602739726028,
-      "eval_linker_heavy_atoms_difference": 0.4562969509500663,
-      "eval_linker_heavy_atoms_difference_norm": -0.0010081340677265851,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9972602739726028,
-      "eval_loss": 0.3004947006702423,
-      "eval_num_fragments": 2.9998232434821035,
-      "eval_poi_equal": 0.7078214759169245,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9555457357490057,
-      "eval_poi_heavy_atoms_difference": 1.2387980556783031,
-      "eval_poi_heavy_atoms_difference_norm": 0.035008881003771114,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9555457357490057,
-      "eval_reassembly": 0.42492266902342024,
-      "eval_reassembly_nostereo": 0.45258506407423776,
-      "eval_runtime": 1502.872,
-      "eval_samples_per_second": 7.529,
-      "eval_steps_per_second": 0.118,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9508616880247459,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=17-learning_rate=2.3e-05-warmup_ratio=0.070-num_cycles=5.500",
+  "trial_name": "trial-number=18-learning_rate=2.0e-04-warmup_ratio=0.010-num_cycles=2.500",
   "trial_params": {
-    "learning_rate": 2.321257272753331e-05,
-    "num_cycles": 5.5,
-    "warmup_ratio": 0.06999999999999999
+    "learning_rate": 0.0002004749428706299,
+    "num_cycles": 2.5,
+    "warmup_ratio": 0.01
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b261af858adb9cfe0d2c1a61e005cf32851d1b6c12b154cbe6a390caaae3353b
+oid sha256:f6d827f97d112aeca59a21ac48c9a4653f744b8e6a9bd8e942e1f99049559f88
 size 7480