Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bcdb3969386fa38fd6adc71ea7e9f7b4b777da6d435448e1643c0757a5cecbcb
+oid sha256:27c49c291cc5bc668eaefd667c7a5489f7ec7947ffcb5bf41328ef0f29e70d19
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6279370528b3f60c4f30ba71f95a93e2c8f5bc7e7f6a818c6afc8689f76340f1
+oid sha256:fc292a6f249b5bc55817d47b8cdcc963df12ea9e87b121a35f9dd2b4d5ec0fd4
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac938fc6b38a53610f05ced08e522126f1a61c094b4aedb5a6dc9ff5ced12fc0
+oid sha256:30601a1a451d7e39285941af5bd86e6869f62dfe2eed0251ea3a96f241145dcc
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff4b36dddeb8a6b98fa508682c8e66b736535bad766232baf8629362d1598122
+oid sha256:03dbbcdb5e205947327bbaa309e87beb06940f7673f4e0865bfa4e81834bd029
 size 1064

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.49403446752098984,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=16-learning_rate=2.7e-05-warmup_ratio=0.020-num_cycles=4/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.18824569155987628,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=17-learning_rate=2.3e-05-warmup_ratio=0.070-num_cycles=6/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.9225704073905945,
-      "learning_rate": 2.4950994668529118e-05,
-      "loss": 1.6862,
+      "grad_norm": 0.9457626938819885,
+      "learning_rate": 2.0006150861219307e-05,
+      "loss": 1.7641,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.6497099995613098,
-      "learning_rate": 1.8846772251455575e-05,
-      "loss": 0.3201,
+      "grad_norm": 0.7090033888816833,
+      "learning_rate": 9.95453959695377e-06,
+      "loss": 0.382,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.5509499907493591,
-      "learning_rate": 1.058829211586969e-05,
-      "loss": 0.1784,
+      "grad_norm": 0.6167610287666321,
+      "learning_rate": 1.2902076164106024e-06,
+      "loss": 0.2555,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.3397536873817444,
-      "learning_rate": 3.391897681746473e-06,
-      "loss": 0.135,
+      "grad_norm": 0.5820146799087524,
+      "learning_rate": 2.1922365111122713e-05,
+      "loss": 0.2229,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.4305669665336609,
-      "learning_rate": 6.029301880940131e-08,
-      "loss": 0.1211,
+      "grad_norm": 0.5265125036239624,
+      "learning_rate": 1.325803313057954e-05,
+      "loss": 0.1442,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.35494357347488403,
-      "learning_rate": 2.4739733516006596e-05,
-      "loss": 0.1049,
+      "grad_norm": 0.3760647475719452,
+      "learning_rate": 3.206421866314002e-06,
+      "loss": 0.1059,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.2973407208919525,
-      "learning_rate": 1.8459695994350122e-05,
-      "loss": 0.0658,
+      "grad_norm": 0.4465866684913635,
+      "learning_rate": 2.3002832012997965e-05,
+      "loss": 0.0951,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.3330534100532532,
-      "learning_rate": 1.0176151273905278e-05,
-      "loss": 0.0475,
+      "grad_norm": 0.31987500190734863,
+      "learning_rate": 1.642771196447441e-05,
+      "loss": 0.0765,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.2717476785182953,
-      "learning_rate": 3.115204414956426e-06,
-      "loss": 0.0391,
+      "grad_norm": 0.3740421235561371,
+      "learning_rate": 5.803143181883335e-06,
+      "loss": 0.0549,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.26711612939834595,
-      "learning_rate": 2.680814436355032e-08,
-      "loss": 0.0362,
+      "grad_norm": 0.30238306522369385,
+      "learning_rate": 5.255416329408794e-08,
+      "loss": 0.0476,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.2691117984975696,
-      "eval_e3_equal": 0.7353071144498453,
+      "eval_all_ligands_equal": 0.18824569155987628,
+      "eval_e3_equal": 0.6958020326999558,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9956694653115333,
-      "eval_e3_heavy_atoms_difference": -0.16137870083959346,
-      "eval_e3_heavy_atoms_difference_norm": -0.018238988976364858,
+      "eval_e3_has_attachment_point(s)": 0.9968183826778613,
+      "eval_e3_heavy_atoms_difference": -0.253557224922669,
+      "eval_e3_heavy_atoms_difference_norm": -0.024097480185388778,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9956694653115333,
-      "eval_has_all_attachment_points": 0.9850640742377376,
-      "eval_has_three_substructures": 0.999116217410517,
-      "eval_heavy_atoms_difference": 7.798585947856827,
-      "eval_heavy_atoms_difference_norm": 0.10333166546592974,
-      "eval_linker_equal": 0.39717189571365447,
+      "eval_e3_valid": 0.9968183826778613,
+      "eval_has_all_attachment_points": 0.9860362350861688,
+      "eval_has_three_substructures": 0.9985859478568272,
+      "eval_heavy_atoms_difference": 7.789659743703049,
+      "eval_heavy_atoms_difference_norm": 0.10390847735933036,
+      "eval_linker_equal": 0.2976579761378701,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9960229783473266,
-      "eval_linker_heavy_atoms_difference": 0.8761820592134335,
-      "eval_linker_heavy_atoms_difference_norm": 0.013908912707481406,
+      "eval_linker_has_attachment_point(s)": 0.9957578435704817,
+      "eval_linker_heavy_atoms_difference": 1.1033141847105612,
+      "eval_linker_heavy_atoms_difference_norm": 0.022649674013351273,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9960229783473266,
-      "eval_loss": 0.28606870770454407,
-      "eval_num_fragments": 2.9998232434821035,
-      "eval_poi_equal": 0.671321254971277,
+      "eval_linker_valid": 0.9957578435704817,
+      "eval_loss": 0.290093332529068,
+      "eval_num_fragments": 3.0,
+      "eval_poi_equal": 0.6439239946973044,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9096774193548387,
-      "eval_poi_heavy_atoms_difference": 2.651524524966858,
-      "eval_poi_heavy_atoms_difference_norm": 0.0825384289930311,
+      "eval_poi_has_attachment_point(s)": 0.9087936367653557,
+      "eval_poi_heavy_atoms_difference": 2.6612461334511712,
+      "eval_poi_heavy_atoms_difference_norm": 0.08440871186102332,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9096774193548387,
-      "eval_reassembly": 0.2733539549270879,
-      "eval_reassembly_nostereo": 0.2933274414494034,
-      "eval_runtime": 1568.5175,
-      "eval_samples_per_second": 7.214,
+      "eval_poi_valid": 0.9087936367653557,
+      "eval_reassembly": 0.19133893062306673,
+      "eval_reassembly_nostereo": 0.21307998232434822,
+      "eval_runtime": 1572.4469,
+      "eval_samples_per_second": 7.196,
       "eval_steps_per_second": 0.113,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9043747238179408,
+      "eval_valid": 0.9050817498895272,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.28343096375465393,
-      "learning_rate": 2.451696899615336e-05,
-      "loss": 0.0365,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.22572530806064606,
-      "learning_rate": 1.806743983536228e-05,
-      "loss": 0.0269,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.20301209390163422,
-      "learning_rate": 9.767171357425924e-06,
-      "loss": 0.0204,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.1848202645778656,
-      "learning_rate": 2.8487818486075695e-06,
-      "loss": 0.0173,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.20325815677642822,
-      "learning_rate": 6.703723611243483e-09,
-      "loss": 0.0159,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.4345558992487848,
-      "eval_e3_equal": 0.7791427308882015,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.993813521873619,
-      "eval_e3_heavy_atoms_difference": 0.18974812196199736,
-      "eval_e3_heavy_atoms_difference_norm": -0.002278847997197217,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.993813521873619,
-      "eval_has_all_attachment_points": 0.9840919133893062,
-      "eval_has_three_substructures": 0.9990278391515687,
-      "eval_heavy_atoms_difference": 7.406186478126381,
-      "eval_heavy_atoms_difference_norm": 0.09853121268513322,
-      "eval_linker_equal": 0.6171453822359699,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9968183826778613,
-      "eval_linker_heavy_atoms_difference": 0.5403446752098984,
-      "eval_linker_heavy_atoms_difference_norm": 0.00343612583628965,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9968183826778613,
-      "eval_loss": 0.2865717113018036,
-      "eval_num_fragments": 2.999734865223155,
-      "eval_poi_equal": 0.7204595669465311,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9159522757401679,
-      "eval_poi_heavy_atoms_difference": 2.4522315510384445,
-      "eval_poi_heavy_atoms_difference_norm": 0.07822232883011575,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9159522757401679,
-      "eval_reassembly": 0.4412726469288555,
-      "eval_reassembly_nostereo": 0.46531153336279274,
-      "eval_runtime": 1494.4607,
-      "eval_samples_per_second": 7.571,
-      "eval_steps_per_second": 0.118,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.909147149801149,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.2400677502155304,
-      "learning_rate": 2.428292541395795e-05,
-      "loss": 0.0189,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.1835109442472458,
-      "learning_rate": 1.7670398743193993e-05,
-      "loss": 0.0142,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.21109040081501007,
-      "learning_rate": 9.361764174539816e-06,
-      "loss": 0.011,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.15360859036445618,
-      "learning_rate": 2.5928982476405618e-06,
-      "loss": 0.0095,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.17562519013881683,
-      "learning_rate": 0.0,
-      "loss": 0.0087,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.49403446752098984,
-      "eval_e3_equal": 0.7933716305788776,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9946973044631021,
-      "eval_e3_heavy_atoms_difference": 0.1994697304463102,
-      "eval_e3_heavy_atoms_difference_norm": -0.0015405930907161603,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9946973044631021,
-      "eval_has_all_attachment_points": 0.9829429960229783,
-      "eval_has_three_substructures": 0.9992929739284137,
-      "eval_heavy_atoms_difference": 7.228988068935042,
-      "eval_heavy_atoms_difference_norm": 0.09704291267150084,
-      "eval_linker_equal": 0.6980114891736633,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9960229783473266,
-      "eval_linker_heavy_atoms_difference": 0.3879805567830314,
-      "eval_linker_heavy_atoms_difference_norm": -0.0004028315220320436,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9960229783473266,
-      "eval_loss": 0.29365620017051697,
-      "eval_num_fragments": 3.0001767565178965,
-      "eval_poi_equal": 0.7405214317277949,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.916570923552806,
-      "eval_poi_heavy_atoms_difference": 2.436411842686699,
-      "eval_poi_heavy_atoms_difference_norm": 0.08075788602113727,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.916570923552806,
-      "eval_reassembly": 0.5017233760494918,
-      "eval_reassembly_nostereo": 0.5281484754750332,
-      "eval_runtime": 1493.4927,
-      "eval_samples_per_second": 7.576,
-      "eval_steps_per_second": 0.119,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.910384445426425,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,17 +133,17 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.663036348367667e+16,
+  "total_flos": 4.33139477262336e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=16-learning_rate=2.7e-05-warmup_ratio=0.020-num_cycles=4",
+  "trial_name": "trial-number=17-learning_rate=2.3e-05-warmup_ratio=0.070-num_cycles=6",
   "trial_params": {
-    "learning_rate": 2.663073667910919e-05,
-    "num_cycles": 4,
-    "warmup_ratio": 0.02
+    "learning_rate": 2.321257272753331e-05,
+    "num_cycles": 6,
+    "warmup_ratio": 0.06999999999999999
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:142086f5e312a0f106951e266b96c414d0c14b22cdec248257e38aa9958859c4
+oid sha256:28afeabe39a09cec3f84c61857ff0a7aa70d81b71ad820e515f0d02a64926893
 size 7480